From 70b95d8f9314ebcddee15e5ac2f4bdb03143846d Mon Sep 17 00:00:00 2001 From: Graeme Winter Date: Thu, 23 Mar 2023 05:20:46 +0000 Subject: [PATCH 001/110] samd/machine_dac: Fix SAMD51 DAC for two channels. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Improvements to DAC support for SAMD51: - properly validate DAC id - correctly use dac_init flag, as a 2-ple for A0, A1 channels - disable DAC before adjusting settings, see SAMD5x data sheet ยง47.6.2.3 Co-authored-by: robert-hh Signed-off-by: Graeme Winter --- ports/samd/machine_dac.c | 52 +++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/ports/samd/machine_dac.c b/ports/samd/machine_dac.c index c611f95e653d0..7dcc654644d79 100644 --- a/ports/samd/machine_dac.c +++ b/ports/samd/machine_dac.c @@ -72,7 +72,8 @@ static uint8_t dac_vref_table[] = { #define MAX_DAC_VALUE (4095) #define DEFAULT_DAC_VREF (2) #define MAX_DAC_VREF (3) -static bool dac_init = false; +static bool dac_init[2] = {false, false}; + #endif @@ -91,10 +92,10 @@ static mp_obj_t dac_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_ uint8_t id = args[ARG_id].u_int; dac_obj_t *self = NULL; - if (0 <= id && id <= MP_ARRAY_SIZE(dac_obj)) { + if (0 <= id && id < MP_ARRAY_SIZE(dac_obj)) { self = &dac_obj[id]; } else { - mp_raise_ValueError(MP_ERROR_TEXT("invalid Pin for DAC")); + mp_raise_ValueError(MP_ERROR_TEXT("invalid id for DAC")); } uint8_t vref = args[ARG_vref].u_int; @@ -102,9 +103,10 @@ static mp_obj_t dac_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_ self->vref = vref; } - Dac *dac = dac_bases[0]; // Just one DAC + Dac *dac = dac_bases[0]; // Just one DAC register block + + // initialize DAC - // Init DAC #if defined(MCU_SAMD21) // Configuration SAMD21 @@ -127,21 +129,39 @@ static mp_obj_t dac_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_ // Configuration SAMD51 // Enable APBD clocks and PCHCTRL clocks; GCLK3 at 8 MHz - dac_init = true; - MCLK->APBDMASK.reg |= MCLK_APBDMASK_DAC; - GCLK->PCHCTRL[DAC_GCLK_ID].reg = GCLK_PCHCTRL_GEN_GCLK3 | GCLK_PCHCTRL_CHEN; - // Reset DAC registers - dac->CTRLA.bit.SWRST = 1; - while (dac->CTRLA.bit.SWRST) { + if (!(dac_init[0] | dac_init[1])) { + MCLK->APBDMASK.reg |= MCLK_APBDMASK_DAC; + GCLK->PCHCTRL[DAC_GCLK_ID].reg = GCLK_PCHCTRL_GEN_GCLK3 | \ + GCLK_PCHCTRL_CHEN; + + // Reset DAC registers + dac->CTRLA.bit.SWRST = 1; + while (dac->CTRLA.bit.SWRST) { + } + dac->CTRLB.reg = DAC_CTRLB_REFSEL(dac_vref_table[self->vref]); + } - dac->CTRLB.reg = DAC_CTRLB_REFSEL(dac_vref_table[self->vref]); - dac->DACCTRL[self->id].reg = DAC_DACCTRL_ENABLE | DAC_DACCTRL_REFRESH(2) | DAC_DACCTRL_CCTRL_CC12M; - // Enable DAC and wait to be ready - dac->CTRLA.bit.ENABLE = 1; - while (dac->SYNCBUSY.bit.ENABLE) { + // Modify DAC config - requires disabling see Section 47.6.2.3 of data sheet + if (!dac_init[self->id]) { + // Disable DAC and wait + dac->CTRLA.bit.ENABLE = 0; + while (dac->SYNCBUSY.bit.ENABLE) { + } + + // Modify configuration + dac->DACCTRL[self->id].reg = DAC_DACCTRL_ENABLE | \ + DAC_DACCTRL_REFRESH(2) | DAC_DACCTRL_CCTRL_CC12M; + dac->DATA[self->id].reg = 0; + dac_init[self->id] = true; + + // Enable DAC and wait + dac->CTRLA.bit.ENABLE = 1; + while (dac->SYNCBUSY.bit.ENABLE) { + } } + #endif // Set the port as given in self->gpio_id as DAC From c69f0e4eeeb1058ac4a4b1b5b5d1896371e0ecd2 Mon Sep 17 00:00:00 2001 From: "I. Tomita" Date: Thu, 19 Oct 2023 12:54:37 +0300 Subject: [PATCH 002/110] samd/samd_qspiflash: Correct QSPI baud calculation. The QSPI baud is derived from the AHB clock, not from the APB (peripheral) clock. Datasheet: The QSPI Baud rate clock is generated by dividing the module clock (CLK_QSPI_AHB) by a value between 1 and 255. As previously implemented, all baudrates are 2.5 times greater than expected. Signed-off-by: I. Tomita --- ports/samd/samd_qspiflash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ports/samd/samd_qspiflash.c b/ports/samd/samd_qspiflash.c index d027a0495f3c0..e25ddf21efe8f 100644 --- a/ports/samd/samd_qspiflash.c +++ b/ports/samd/samd_qspiflash.c @@ -238,7 +238,7 @@ static void wait_for_flash_ready(void) { } static uint8_t get_baud(int32_t freq_mhz) { - int baud = get_peripheral_freq() / (freq_mhz * 1000000) - 1; + int baud = get_cpu_freq() / (freq_mhz * 1000000) - 1; if (baud < 1) { baud = 1; } From 865a4c8bf6868ce488347a76f5f3db8e0dabd11d Mon Sep 17 00:00:00 2001 From: IhorNehrutsa Date: Tue, 21 Jan 2025 08:48:25 +0200 Subject: [PATCH 003/110] esp32: Add support for IDF v5.4. Add WIFI_AUTH_WPA3_ENTERPRISE and WIFI_AUTH_WPA2_WPA3_ENTERPRISE, and update PPP callback signature for latest lwIP. Co-authored-by: Daniel van de Giessen Signed-off-by: IhorNehrutsa --- ports/esp32/README.md | 2 +- ports/esp32/network_ppp.c | 9 +++++++-- ports/esp32/network_wlan.c | 8 +++++++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/ports/esp32/README.md b/ports/esp32/README.md index bc24e5cd71ad6..8597c85ec3103 100644 --- a/ports/esp32/README.md +++ b/ports/esp32/README.md @@ -28,7 +28,7 @@ manage the ESP32 microcontroller, as well as a way to manage the required build environment and toolchains needed to build the firmware. The ESP-IDF changes quickly and MicroPython only supports certain versions. -Currently MicroPython supports v5.2, v5.2.2, and v5.3. +Currently MicroPython supports v5.2, v5.2.2, v5.3 and v5.4. To install the ESP-IDF the full instructions can be found at the [Espressif Getting Started guide](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/get-started/index.html#installation-step-by-step). diff --git a/ports/esp32/network_ppp.c b/ports/esp32/network_ppp.c index 5c41cf948c1ed..4dd5a3718c295 100644 --- a/ports/esp32/network_ppp.c +++ b/ports/esp32/network_ppp.c @@ -100,7 +100,12 @@ static mp_obj_t ppp_make_new(mp_obj_t stream) { } MP_DEFINE_CONST_FUN_OBJ_1(esp_network_ppp_make_new_obj, ppp_make_new); -static u32_t ppp_output_callback(ppp_pcb *pcb, u8_t *data, u32_t len, void *ctx) { +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 4, 0) +static u32_t ppp_output_callback(ppp_pcb *pcb, const void *data, u32_t len, void *ctx) +#else +static u32_t ppp_output_callback(ppp_pcb *pcb, u8_t *data, u32_t len, void *ctx) +#endif +{ ppp_if_obj_t *self = ctx; mp_obj_t stream = self->stream; @@ -109,7 +114,7 @@ static u32_t ppp_output_callback(ppp_pcb *pcb, u8_t *data, u32_t len, void *ctx) } int err; - return mp_stream_rw(stream, data, len, &err, MP_STREAM_RW_WRITE); + return mp_stream_rw(stream, (void *)data, len, &err, MP_STREAM_RW_WRITE); } static void pppos_client_task(void *self_in) { diff --git a/ports/esp32/network_wlan.c b/ports/esp32/network_wlan.c index fed81d28cb6a6..e85d1328fdc2b 100644 --- a/ports/esp32/network_wlan.c +++ b/ports/esp32/network_wlan.c @@ -763,6 +763,10 @@ static const mp_rom_map_elem_t wlan_if_locals_dict_table[] = { #if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 3, 0) { MP_ROM_QSTR(MP_QSTR_SEC_DPP), MP_ROM_INT(WIFI_AUTH_DPP) }, #endif + #if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 4, 0) + { MP_ROM_QSTR(MP_QSTR_SEC_WPA3_ENT), MP_ROM_INT(WIFI_AUTH_WPA3_ENTERPRISE) }, + { MP_ROM_QSTR(MP_QSTR_SEC_WPA2_WPA3_ENT), MP_ROM_INT(WIFI_AUTH_WPA2_WPA3_ENTERPRISE) }, + #endif { MP_ROM_QSTR(MP_QSTR_PM_NONE), MP_ROM_INT(WIFI_PS_NONE) }, { MP_ROM_QSTR(MP_QSTR_PM_PERFORMANCE), MP_ROM_INT(WIFI_PS_MIN_MODEM) }, @@ -770,7 +774,9 @@ static const mp_rom_map_elem_t wlan_if_locals_dict_table[] = { }; static MP_DEFINE_CONST_DICT(wlan_if_locals_dict, wlan_if_locals_dict_table); -#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 3, 0) +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 4, 0) +_Static_assert(WIFI_AUTH_MAX == 16, "Synchronize WIFI_AUTH_XXX constants with the ESP-IDF. Look at esp-idf/components/esp_wifi/include/esp_wifi_types_generic.h"); +#elif ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 3, 0) _Static_assert(WIFI_AUTH_MAX == 14, "Synchronize WIFI_AUTH_XXX constants with the ESP-IDF. Look at esp-idf/components/esp_wifi/include/esp_wifi_types_generic.h"); #elif ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 2, 0) _Static_assert(WIFI_AUTH_MAX == 13, "Synchronize WIFI_AUTH_XXX constants with the ESP-IDF. Look at esp-idf/components/esp_wifi/include/esp_wifi_types.h"); From a4ab847688406bb31eb531e07155bf173a8785aa Mon Sep 17 00:00:00 2001 From: IhorNehrutsa Date: Tue, 21 Jan 2025 09:55:55 +0200 Subject: [PATCH 004/110] py/persistentcode: Initialize prelude_ptr to prevent compiler warning. The esp32 IDF toolchain can give a "may be used uninitialized" warning, at least for ESP32-S3 with gcc 14.2.0. Silence that warning by initializing the variable with NULL. Co-authored-by: Daniel van de Giessen Signed-off-by: IhorNehrutsa --- py/persistentcode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/persistentcode.c b/py/persistentcode.c index 0843d1a2c52e6..840ee49d3e4eb 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -402,7 +402,7 @@ static mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co #if MICROPY_EMIT_MACHINE_CODE } else { - const uint8_t *prelude_ptr; + const uint8_t *prelude_ptr = NULL; #if MICROPY_EMIT_NATIVE_PRELUDE_SEPARATE_FROM_MACHINE_CODE if (kind == MP_CODE_NATIVE_PY) { // Executable code cannot be accessed byte-wise on this architecture, so copy From 7b3f189b1723fe642f122a3b7826d16fe32f801a Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Thu, 8 Feb 2024 16:56:52 -0800 Subject: [PATCH 005/110] tests/basics/nanbox_smallint.py: Fix incorrect use of int() in test. The literal is in base 16 but int()'s default radix in CPython is 10, not 0. Signed-off-by: Jeff Epler --- tests/basics/nanbox_smallint.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/basics/nanbox_smallint.py b/tests/basics/nanbox_smallint.py index b3a502e447e3a..9451ab3284661 100644 --- a/tests/basics/nanbox_smallint.py +++ b/tests/basics/nanbox_smallint.py @@ -23,17 +23,17 @@ raise SystemExit micropython.heap_lock() -print(int("0x80000000")) +print(int("0x80000000", 16)) micropython.heap_unlock() # This is the most positive small integer. micropython.heap_lock() -print(int("0x3fffffffffff")) +print(int("0x3fffffffffff", 16)) micropython.heap_unlock() # This is the most negative small integer. micropython.heap_lock() -print(int("-0x3fffffffffff") - 1) +print(int("-0x3fffffffffff", 16) - 1) micropython.heap_unlock() x = 1 From 13b13d1fdd05549d504eeded0b5aa8871d5e5dcf Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Wed, 3 Jan 2024 19:31:35 -0600 Subject: [PATCH 006/110] py/parsenum: Throw an exception for invalid int literals like "01". This includes making int("01") parse in base 10 like standard Python. When a base of 0 is specified it means auto-detect based on the prefix, and literals begining with 0 (except when the literal is all 0's) like "01" are then invalid and now throw an exception. The new error message is different from CPython. It says e.g., `SyntaxError: invalid syntax for integer with base 0: '09'` Additional test cases were added to cover the changed & added code. Co-authored-by: Damien George Signed-off-by: Jeff Epler --- py/objint.c | 2 +- py/parsenum.c | 4 ++-- py/parsenumbase.c | 29 +++++++++++------------------ tests/basics/int1.py | 15 +++++++++++++++ tests/basics/lexer.py | 8 ++++++++ 5 files changed, 37 insertions(+), 21 deletions(-) diff --git a/py/objint.c b/py/objint.c index 773e180343aeb..4be6009a440e4 100644 --- a/py/objint.c +++ b/py/objint.c @@ -55,7 +55,7 @@ static mp_obj_t mp_obj_int_make_new(const mp_obj_type_t *type_in, size_t n_args, return o; } else if (mp_get_buffer(args[0], &bufinfo, MP_BUFFER_READ)) { // a textual representation, parse it - return mp_parse_num_integer(bufinfo.buf, bufinfo.len, 0, NULL); + return mp_parse_num_integer(bufinfo.buf, bufinfo.len, 10, NULL); #if MICROPY_PY_BUILTINS_FLOAT } else if (mp_obj_is_float(args[0])) { return mp_obj_new_int_from_float(mp_obj_float_get(args[0])); diff --git a/py/parsenum.c b/py/parsenum.c index b33ffb6ff23e0..27d6641198730 100644 --- a/py/parsenum.c +++ b/py/parsenum.c @@ -151,13 +151,13 @@ mp_obj_t mp_parse_num_integer(const char *restrict str_, size_t len, int base, m raise_exc(exc, lex); #elif MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_NORMAL mp_obj_t exc = mp_obj_new_exception_msg_varg(&mp_type_ValueError, - MP_ERROR_TEXT("invalid syntax for integer with base %d"), base); + MP_ERROR_TEXT("invalid syntax for integer with base %d"), base == 1 ? 0 : base); raise_exc(exc, lex); #else vstr_t vstr; mp_print_t print; vstr_init_print(&vstr, 50, &print); - mp_printf(&print, "invalid syntax for integer with base %d: ", base); + mp_printf(&print, "invalid syntax for integer with base %d: ", base == 1 ? 0 : base); mp_str_print_quoted(&print, str_val_start, top - str_val_start, true); mp_obj_t exc = mp_obj_new_exception_arg1(&mp_type_ValueError, mp_obj_new_str_from_utf8_vstr(&vstr)); diff --git a/py/parsenumbase.c b/py/parsenumbase.c index 94523a666d325..cc3275c456e40 100644 --- a/py/parsenumbase.c +++ b/py/parsenumbase.c @@ -30,35 +30,28 @@ // find real radix base, and strip preceding '0x', '0o' and '0b' // puts base in *base, and returns number of bytes to skip the prefix +// in base-0, puts 1 in *base to indicate a number that starts with 0, to provoke a +// ValueError if it's not all-digits-zero. size_t mp_parse_num_base(const char *str, size_t len, int *base) { const byte *p = (const byte *)str; if (len <= 1) { goto no_prefix; } unichar c = *(p++); - if ((*base == 0 || *base == 16) && c == '0') { - c = *(p++); - if ((c | 32) == 'x') { + if (c == '0') { + c = *(p++) | 32; + int b = *base; + if (c == 'x' && !(b & ~16)) { *base = 16; - } else if (*base == 0 && (c | 32) == 'o') { + } else if (c == 'o' && !(b & ~8)) { *base = 8; - } else if (*base == 0 && (c | 32) == 'b') { + } else if (c == 'b' && !(b & ~2)) { *base = 2; } else { - if (*base == 0) { - *base = 10; - } - p -= 2; - } - } else if (*base == 8 && c == '0') { - c = *(p++); - if ((c | 32) != 'o') { - p -= 2; - } - } else if (*base == 2 && c == '0') { - c = *(p++); - if ((c | 32) != 'b') { p -= 2; + if (b == 0) { + *base = 1; + } } } else { p--; diff --git a/tests/basics/int1.py b/tests/basics/int1.py index 2d92105c73e88..94723af4d00b4 100644 --- a/tests/basics/int1.py +++ b/tests/basics/int1.py @@ -13,6 +13,7 @@ print(int('+1')) print(int('-1')) print(int('01')) +print(int('00')) print(int('9')) print(int('10')) print(int('+10')) @@ -31,6 +32,7 @@ print(int('0', 10)) print(int('1', 10)) print(int(' \t 1 \t ', 10)) +print(int(' \t 00 \t ', 10)) print(int('11', 10)) print(int('11', 16)) print(int('11', 8)) @@ -52,6 +54,17 @@ print(int('0o12 \t ', 8)) print(int(b"12", 10)) print(int(b"12")) +print(int('000 ', 0)) +print(int('000 ', 2)) +print(int('000 ', 8)) +print(int('000 ', 10)) +print(int('000 ', 16)) +print(int('000 ', 36)) +print(int('010 ', 2)) +print(int('010 ', 8)) +print(int('010 ', 10)) +print(int('010 ', 16)) +print(int('010 ', 36)) def test(value, base): @@ -79,6 +92,8 @@ def test(value, base): test('0xg', 16) test('1 1', 16) test('123', 37) +test('01', 0) +test('01 ', 0) # check that we don't parse this as a floating point number print(0x1e+1) diff --git a/tests/basics/lexer.py b/tests/basics/lexer.py index 181d62db1aadb..addb8a13df36a 100644 --- a/tests/basics/lexer.py +++ b/tests/basics/lexer.py @@ -83,3 +83,11 @@ def a(x): exec(r"'\U0000000'") except SyntaxError: print("SyntaxError") + +# Properly formed integer literals +print(eval("00")) +# badly formed integer literals +try: + eval("01") +except SyntaxError: + print("SyntaxError") From bfc0d7b0b97cbf52db892775b0f371dd06d2ba08 Mon Sep 17 00:00:00 2001 From: Alessandro Gatti Date: Thu, 23 Jan 2025 23:55:57 +0100 Subject: [PATCH 007/110] py/emitnative: Optimise Viper register offset load/stores on Xtensa. This commit improves the emitted code sequences for address generation in the Viper subsystem when loading/storing 16 and 32 bit values via a register offset. The Xtensa opcodes ADDX2 and ADDX4 are used to avoid performing the extra shifts to align the final operation offset. Those opcodes are available on both xtensa and xtensawin MicroPython architectures. Signed-off-by: Alessandro Gatti --- py/asmxtensa.h | 8 ++++++++ py/emitnative.c | 17 +++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/py/asmxtensa.h b/py/asmxtensa.h index f226624a82631..d2f37bf828e86 100644 --- a/py/asmxtensa.h +++ b/py/asmxtensa.h @@ -143,6 +143,14 @@ static inline void asm_xtensa_op_addi(asm_xtensa_t *as, uint reg_dest, uint reg_ asm_xtensa_op24(as, ASM_XTENSA_ENCODE_RRI8(2, 12, reg_src, reg_dest, imm8 & 0xff)); } +static inline void asm_xtensa_op_addx2(asm_xtensa_t *as, uint reg_dest, uint reg_src_a, uint reg_src_b) { + asm_xtensa_op24(as, ASM_XTENSA_ENCODE_RRR(0, 0, 9, reg_dest, reg_src_a, reg_src_b)); +} + +static inline void asm_xtensa_op_addx4(asm_xtensa_t *as, uint reg_dest, uint reg_src_a, uint reg_src_b) { + asm_xtensa_op24(as, ASM_XTENSA_ENCODE_RRR(0, 0, 10, reg_dest, reg_src_a, reg_src_b)); +} + static inline void asm_xtensa_op_and(asm_xtensa_t *as, uint reg_dest, uint reg_src_a, uint reg_src_b) { asm_xtensa_op24(as, ASM_XTENSA_ENCODE_RRR(0, 0, 1, reg_dest, reg_src_a, reg_src_b)); } diff --git a/py/emitnative.c b/py/emitnative.c index 66c345b233d76..7d856e13f753a 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -1625,6 +1625,11 @@ static void emit_native_load_subscr(emit_t *emit) { } case VTYPE_PTR16: { // pointer to 16-bit memory + #if N_XTENSA || N_XTENSAWIN + asm_xtensa_op_addx2(emit->as, REG_ARG_1, reg_index, REG_ARG_1); + asm_xtensa_op_l16ui(emit->as, REG_RET, REG_ARG_1, 0); + break; + #endif ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_LOAD16_REG_REG(emit->as, REG_RET, REG_ARG_1); // load from (base+2*index) @@ -1637,6 +1642,10 @@ static void emit_native_load_subscr(emit_t *emit) { asm_rv32_opcode_cadd(emit->as, REG_ARG_1, REG_TEMP2); asm_rv32_opcode_lw(emit->as, REG_RET, REG_ARG_1, 0); break; + #elif N_XTENSA || N_XTENSAWIN + asm_xtensa_op_addx4(emit->as, REG_ARG_1, reg_index, REG_ARG_1); + asm_xtensa_op_l32i_n(emit->as, REG_RET, REG_ARG_1, 0); + break; #endif ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base @@ -1900,6 +1909,10 @@ static void emit_native_store_subscr(emit_t *emit) { #if N_ARM asm_arm_strh_reg_reg_reg(emit->as, reg_value, REG_ARG_1, reg_index); break; + #elif N_XTENSA || N_XTENSAWIN + asm_xtensa_op_addx2(emit->as, REG_ARG_1, reg_index, REG_ARG_1); + asm_xtensa_op_s16i(emit->as, reg_value, REG_ARG_1, 0); + break; #endif ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base @@ -1916,6 +1929,10 @@ static void emit_native_store_subscr(emit_t *emit) { asm_rv32_opcode_cadd(emit->as, REG_ARG_1, REG_TEMP2); asm_rv32_opcode_sw(emit->as, reg_value, REG_ARG_1, 0); break; + #elif N_XTENSA || N_XTENSAWIN + asm_xtensa_op_addx4(emit->as, REG_ARG_1, reg_index, REG_ARG_1); + asm_xtensa_op_s32i_n(emit->as, reg_value, REG_ARG_1, 0); + break; #endif ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base ASM_ADD_REG_REG(emit->as, REG_ARG_1, reg_index); // add index to base From 40585eaa8f1b603f0094b73764e8ce5623812ecf Mon Sep 17 00:00:00 2001 From: Alessandro Gatti Date: Fri, 24 Jan 2025 00:20:39 +0100 Subject: [PATCH 008/110] py/emitnative: Emit shorter exception handler entry code on RV32. This commit improves the RV32 code sequence that is emitted if a function needs to set up an exception handler as its prologue. The old code would clear a temporary register and then copy that value to places that needed to be initialised with zero values. On RV32 there's a dedicated register that's hardwired to be equal to zero, which allows us to bypass the extra register clear and use the zero register to initialise values. Signed-off-by: Alessandro Gatti --- py/asmrv32.h | 2 ++ py/emitnative.c | 15 ++++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/py/asmrv32.h b/py/asmrv32.h index 584e3ffd22436..b09f48eb12f66 100644 --- a/py/asmrv32.h +++ b/py/asmrv32.h @@ -694,6 +694,7 @@ static inline void asm_rv32_opcode_xori(asm_rv32_t *state, mp_uint_t rd, mp_uint #define REG_LOCAL_1 ASM_RV32_REG_S3 #define REG_LOCAL_2 ASM_RV32_REG_S4 #define REG_LOCAL_3 ASM_RV32_REG_S5 +#define REG_ZERO ASM_RV32_REG_ZERO void asm_rv32_meta_comparison_eq(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t rd); void asm_rv32_meta_comparison_ne(asm_rv32_t *state, mp_uint_t rs1, mp_uint_t rs2, mp_uint_t rd); @@ -756,6 +757,7 @@ void asm_rv32_emit_store_reg_reg_offset(asm_rv32_t *state, mp_uint_t source, mp_ #define ASM_STORE_REG_REG(state, rs1, rs2) ASM_STORE32_REG_REG(state, rs1, rs2) #define ASM_SUB_REG_REG(state, rd, rs) asm_rv32_opcode_sub(state, rd, rd, rs) #define ASM_XOR_REG_REG(state, rd, rs) asm_rv32_emit_optimised_xor(state, rd, rs) +#define ASM_CLR_REG(state, rd) #endif diff --git a/py/emitnative.c b/py/emitnative.c index 7d856e13f753a..f846e8bb4e2cb 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -288,6 +288,11 @@ struct _emit_t { ASM_T *as; }; +#ifndef REG_ZERO +#define REG_ZERO REG_TEMP0 +#define ASM_CLR_REG(state, rd) ASM_XOR_REG_REG(state, rd, rd) +#endif + static void emit_load_reg_with_object(emit_t *emit, int reg, mp_obj_t obj); static void emit_native_global_exc_entry(emit_t *emit); static void emit_native_global_exc_exit(emit_t *emit); @@ -1200,12 +1205,12 @@ static void emit_native_global_exc_entry(emit_t *emit) { ASM_JUMP_IF_REG_ZERO(emit->as, REG_RET, start_label, true); } else { // Clear the unwind state - ASM_XOR_REG_REG(emit->as, REG_TEMP0, REG_TEMP0); - ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_HANDLER_UNWIND(emit), REG_TEMP0); + ASM_CLR_REG(emit->as, REG_ZERO); + ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_HANDLER_UNWIND(emit), REG_ZERO); // clear nlr.ret_val, because it's passed to mp_native_raise regardless // of whether there was an exception or not - ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_VAL(emit), REG_TEMP0); + ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_VAL(emit), REG_ZERO); // Put PC of start code block into REG_LOCAL_1 ASM_MOV_REG_PCREL(emit->as, REG_LOCAL_1, start_label); @@ -1221,8 +1226,8 @@ static void emit_native_global_exc_entry(emit_t *emit) { ASM_JUMP_IF_REG_NONZERO(emit->as, REG_RET, global_except_label, true); // Clear PC of current code block, and jump there to resume execution - ASM_XOR_REG_REG(emit->as, REG_TEMP0, REG_TEMP0); - ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_HANDLER_PC(emit), REG_TEMP0); + ASM_CLR_REG(emit->as, REG_ZERO); + ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_EXC_HANDLER_PC(emit), REG_ZERO); ASM_JUMP_REG(emit->as, REG_LOCAL_1); // Global exception handler: check for valid exception handler From 55ca3fd67512555707304c6b68b836eb89f09d1c Mon Sep 17 00:00:00 2001 From: Alessandro Gatti Date: Fri, 24 Jan 2025 09:22:36 +0100 Subject: [PATCH 009/110] py/emitnative: Optimise Viper immediate offset load/stores on Xtensa. This commit introduces the ability to emit optimised code paths on Xtensa for load/store operations indexed via an immediate offset. If an immediate offset for a load/store operation is within a certain range that allows it to be embedded into an available opcode then said opcode is emitted instead of the generic code sequence. Signed-off-by: Alessandro Gatti --- py/emitnative.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/py/emitnative.c b/py/emitnative.c index f846e8bb4e2cb..82ee729d3d50f 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -1550,6 +1550,11 @@ static void emit_native_load_subscr(emit_t *emit) { asm_rv32_opcode_lbu(emit->as, REG_RET, reg_base, index_value); break; } + #elif N_XTENSA || N_XTENSAWIN + if (index_value > 0 && index_value < 256) { + asm_xtensa_op_l8ui(emit->as, REG_RET, reg_base, index_value); + break; + } #endif need_reg_single(emit, reg_index, 0); ASM_MOV_REG_IMM(emit->as, reg_index, index_value); @@ -1573,6 +1578,11 @@ static void emit_native_load_subscr(emit_t *emit) { asm_rv32_opcode_lhu(emit->as, REG_RET, reg_base, index_value << 1); break; } + #elif N_XTENSA || N_XTENSAWIN + if (index_value > 0 && index_value < 256) { + asm_xtensa_op_l16ui(emit->as, REG_RET, reg_base, index_value); + break; + } #endif need_reg_single(emit, reg_index, 0); ASM_MOV_REG_IMM(emit->as, reg_index, index_value << 1); @@ -1596,6 +1606,10 @@ static void emit_native_load_subscr(emit_t *emit) { asm_rv32_opcode_lw(emit->as, REG_RET, reg_base, index_value << 2); break; } + #elif N_XTENSA || N_XTENSAWIN + if (index_value > 0 && index_value < 256) { + asm_xtensa_l32i_optimised(emit->as, REG_RET, reg_base, index_value); + } #endif need_reg_single(emit, reg_index, 0); ASM_MOV_REG_IMM(emit->as, reg_index, index_value << 2); @@ -1812,6 +1826,11 @@ static void emit_native_store_subscr(emit_t *emit) { asm_rv32_opcode_sb(emit->as, reg_value, reg_base, index_value); break; } + #elif N_XTENSA || N_XTENSAWIN + if (index_value > 0 && index_value < 256) { + asm_xtensa_op_s8i(emit->as, REG_RET, reg_base, index_value); + break; + } #endif ASM_MOV_REG_IMM(emit->as, reg_index, index_value); #if N_ARM @@ -1838,6 +1857,11 @@ static void emit_native_store_subscr(emit_t *emit) { asm_rv32_opcode_sh(emit->as, reg_value, reg_base, index_value << 1); break; } + #elif N_XTENSA || N_XTENSAWIN + if (index_value > 0 && index_value < 256) { + asm_xtensa_op_s16i(emit->as, REG_RET, reg_base, index_value); + break; + } #endif ASM_MOV_REG_IMM(emit->as, reg_index, index_value << 1); ASM_ADD_REG_REG(emit->as, reg_index, reg_base); // add 2*index to base @@ -1860,6 +1884,10 @@ static void emit_native_store_subscr(emit_t *emit) { asm_rv32_opcode_sw(emit->as, reg_value, reg_base, index_value << 2); break; } + #elif N_XTENSA || N_XTENSAWIN + if (index_value > 0 && index_value < 256) { + asm_xtensa_s32i_optimised(emit->as, REG_RET, reg_base, index_value); + } #elif N_ARM ASM_MOV_REG_IMM(emit->as, reg_index, index_value); asm_arm_str_reg_reg_reg(emit->as, reg_value, reg_base, reg_index); From 0d46e45a1f72ee61a153c41aaaf6c63818ccffb0 Mon Sep 17 00:00:00 2001 From: Hans Maerki Date: Sun, 26 Jan 2025 18:00:38 +0100 Subject: [PATCH 010/110] tools/mpremote: Avoid initial blocking read in read_until(). If the target does not return any data then `read_until()` will block indefinitely. Fix this by making the initial read part of the general read look, which always checks `inWaiting() > 0` before reading from the serial device. Also added the UART timeout to the constructor. This is not currently used but may be used as an additional safeguard. Signed-off-by: Hans Maerki --- tools/mpremote/mpremote/transport_serial.py | 30 ++++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/tools/mpremote/mpremote/transport_serial.py b/tools/mpremote/mpremote/transport_serial.py index 28ccaf6d8c907..449708a5ee43d 100644 --- a/tools/mpremote/mpremote/transport_serial.py +++ b/tools/mpremote/mpremote/transport_serial.py @@ -42,7 +42,7 @@ class SerialTransport(Transport): - def __init__(self, device, baudrate=115200, wait=0, exclusive=True): + def __init__(self, device, baudrate=115200, wait=0, exclusive=True, timeout=None): self.in_raw_repl = False self.use_raw_paste = True self.device_name = device @@ -52,7 +52,11 @@ def __init__(self, device, baudrate=115200, wait=0, exclusive=True): import serial.tools.list_ports # Set options, and exclusive if pyserial supports it - serial_kwargs = {"baudrate": baudrate, "interCharTimeout": 1} + serial_kwargs = { + "baudrate": baudrate, + "timeout": timeout, + "interCharTimeout": 1, + } if serial.__version__ >= "3.3": serial_kwargs["exclusive"] = exclusive @@ -95,13 +99,20 @@ def close(self): self.serial.close() def read_until(self, min_num_bytes, ending, timeout=10, data_consumer=None): - # if data_consumer is used then data is not accumulated and the ending must be 1 byte long + """ + min_num_bytes: Obsolete. + ending: Return if 'ending' matches. + timeout [s]: Return if timeout between characters. None: Infinite timeout. + data_consumer: Use callback for incoming characters. + If data_consumer is used then data is not accumulated and the ending must be 1 byte long + + It is not visible to the caller why the function returned. It could be ending or timeout. + """ assert data_consumer is None or len(ending) == 1 + assert isinstance(timeout, (type(None), int, float)) - data = self.serial.read(min_num_bytes) - if data_consumer: - data_consumer(data) - timeout_count = 0 + data = b"" + begin_char_s = time.monotonic() while True: if data.endswith(ending): break @@ -112,10 +123,9 @@ def read_until(self, min_num_bytes, ending, timeout=10, data_consumer=None): data = new_data else: data = data + new_data - timeout_count = 0 + begin_char_s = time.monotonic() else: - timeout_count += 1 - if timeout is not None and timeout_count >= 100 * timeout: + if timeout is not None and time.monotonic() >= begin_char_s + timeout: break time.sleep(0.01) return data From 03fe9c55ea8a3ef8bede5547476f37d7fca20119 Mon Sep 17 00:00:00 2001 From: Hans Maerki Date: Mon, 27 Jan 2025 07:29:44 +0100 Subject: [PATCH 011/110] tools/mpremote: Introduce timeout_overall for read_until(). And use it in `enter_raw_repl()`. This prevents waiting forever for a serial device that does not respond to the Ctrl-C/Ctrl-D/etc commands and is constantly outputting data. Signed-off-by: Hans Maerki --- tools/mpremote/mpremote/transport_serial.py | 23 +++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tools/mpremote/mpremote/transport_serial.py b/tools/mpremote/mpremote/transport_serial.py index 449708a5ee43d..d3148f9ace318 100644 --- a/tools/mpremote/mpremote/transport_serial.py +++ b/tools/mpremote/mpremote/transport_serial.py @@ -98,11 +98,14 @@ def __init__(self, device, baudrate=115200, wait=0, exclusive=True, timeout=None def close(self): self.serial.close() - def read_until(self, min_num_bytes, ending, timeout=10, data_consumer=None): + def read_until( + self, min_num_bytes, ending, timeout=10, data_consumer=None, timeout_overall=None + ): """ min_num_bytes: Obsolete. ending: Return if 'ending' matches. timeout [s]: Return if timeout between characters. None: Infinite timeout. + timeout_overall [s]: Return not later than timeout_overall. None: Infinite timeout. data_consumer: Use callback for incoming characters. If data_consumer is used then data is not accumulated and the ending must be 1 byte long @@ -110,9 +113,10 @@ def read_until(self, min_num_bytes, ending, timeout=10, data_consumer=None): """ assert data_consumer is None or len(ending) == 1 assert isinstance(timeout, (type(None), int, float)) + assert isinstance(timeout_overall, (type(None), int, float)) data = b"" - begin_char_s = time.monotonic() + begin_overall_s = begin_char_s = time.monotonic() while True: if data.endswith(ending): break @@ -127,10 +131,15 @@ def read_until(self, min_num_bytes, ending, timeout=10, data_consumer=None): else: if timeout is not None and time.monotonic() >= begin_char_s + timeout: break + if ( + timeout_overall is not None + and time.monotonic() >= begin_overall_s + timeout_overall + ): + break time.sleep(0.01) return data - def enter_raw_repl(self, soft_reset=True): + def enter_raw_repl(self, soft_reset=True, timeout_overall=10): self.serial.write(b"\r\x03") # ctrl-C: interrupt any running program # flush input (without relying on serial.flushInput()) @@ -142,7 +151,9 @@ def enter_raw_repl(self, soft_reset=True): self.serial.write(b"\r\x01") # ctrl-A: enter raw REPL if soft_reset: - data = self.read_until(1, b"raw REPL; CTRL-B to exit\r\n>") + data = self.read_until( + 1, b"raw REPL; CTRL-B to exit\r\n>", timeout_overall=timeout_overall + ) if not data.endswith(b"raw REPL; CTRL-B to exit\r\n>"): print(data) raise TransportError("could not enter raw repl") @@ -152,12 +163,12 @@ def enter_raw_repl(self, soft_reset=True): # Waiting for "soft reboot" independently to "raw REPL" (done below) # allows boot.py to print, which will show up after "soft reboot" # and before "raw REPL". - data = self.read_until(1, b"soft reboot\r\n") + data = self.read_until(1, b"soft reboot\r\n", timeout_overall=timeout_overall) if not data.endswith(b"soft reboot\r\n"): print(data) raise TransportError("could not enter raw repl") - data = self.read_until(1, b"raw REPL; CTRL-B to exit\r\n") + data = self.read_until(1, b"raw REPL; CTRL-B to exit\r\n", timeout_overall=timeout_overall) if not data.endswith(b"raw REPL; CTRL-B to exit\r\n"): print(data) raise TransportError("could not enter raw repl") From f29bd5a65e9688222474e54b4f5d0fb127a23db3 Mon Sep 17 00:00:00 2001 From: Jared Hancock Date: Sun, 19 Jan 2025 15:37:55 -0600 Subject: [PATCH 012/110] extmod/modlwip: Fix incorrect peer address for IPv6. For IPv6 connections, the peer address was previously defined as only the first four bytes of the IP address. For IPv6 addresses, this resulted in an incorrect IPv4 address. For instance, receiving a packet via `::recvfrom` from `'fe80::87:e7ff:fe48:629a'` is returned as having a peer address of `'254.128.0.0'` Signed-off-by: Jared Hancock --- extmod/modlwip.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/extmod/modlwip.c b/extmod/modlwip.c index ce70627c31c60..bcccd363e673c 100644 --- a/extmod/modlwip.c +++ b/extmod/modlwip.c @@ -37,7 +37,6 @@ #if MICROPY_PY_LWIP -#include "shared/netutils/netutils.h" #include "modnetwork.h" #include "lwip/init.h" @@ -308,7 +307,7 @@ typedef struct _lwip_socket_obj_t { } connection; } incoming; mp_obj_t callback; - byte peer[4]; + ip_addr_t peer; mp_uint_t peer_port; mp_uint_t timeout; uint16_t recv_offset; @@ -638,7 +637,7 @@ static mp_uint_t lwip_raw_udp_send(lwip_socket_obj_t *socket, const byte *buf, m } // Helper function for recv/recvfrom to handle raw/UDP packets -static mp_uint_t lwip_raw_udp_receive(lwip_socket_obj_t *socket, byte *buf, mp_uint_t len, byte *ip, mp_uint_t *port, int *_errno) { +static mp_uint_t lwip_raw_udp_receive(lwip_socket_obj_t *socket, byte *buf, mp_uint_t len, ip_addr_t *ip, mp_uint_t *port, int *_errno) { if (socket->incoming.pbuf == NULL) { if (socket->timeout == 0) { @@ -1132,7 +1131,7 @@ static mp_obj_t lwip_socket_connect(mp_obj_t self_in, mp_obj_t addr_in) { mp_raise_OSError(error_lookup_table[-err]); } socket->peer_port = (mp_uint_t)port; - memcpy(socket->peer, &dest, sizeof(socket->peer)); + memcpy(&socket->peer, &dest, sizeof(socket->peer)); MICROPY_PY_LWIP_EXIT // And now we wait... @@ -1294,13 +1293,13 @@ static mp_obj_t lwip_socket_recvfrom(mp_obj_t self_in, mp_obj_t len_in) { mp_int_t len = mp_obj_get_int(len_in); vstr_t vstr; vstr_init_len(&vstr, len); - byte ip[4]; + ip_addr_t ip; mp_uint_t port; mp_uint_t ret = 0; switch (socket->type) { case MOD_NETWORK_SOCK_STREAM: { - memcpy(ip, &socket->peer, sizeof(socket->peer)); + memcpy(&ip, &socket->peer, sizeof(socket->peer)); port = (mp_uint_t)socket->peer_port; ret = lwip_tcp_receive(socket, (byte *)vstr.buf, len, &_errno); break; @@ -1309,7 +1308,7 @@ static mp_obj_t lwip_socket_recvfrom(mp_obj_t self_in, mp_obj_t len_in) { #if MICROPY_PY_LWIP_SOCK_RAW case MOD_NETWORK_SOCK_RAW: #endif - ret = lwip_raw_udp_receive(socket, (byte *)vstr.buf, len, ip, &port, &_errno); + ret = lwip_raw_udp_receive(socket, (byte *)vstr.buf, len, &ip, &port, &_errno); break; } if (ret == -1) { @@ -1323,7 +1322,8 @@ static mp_obj_t lwip_socket_recvfrom(mp_obj_t self_in, mp_obj_t len_in) { vstr.len = ret; tuple[0] = mp_obj_new_bytes_from_vstr(&vstr); } - tuple[1] = netutils_format_inet_addr(ip, port, NETUTILS_BIG); + tuple[1] = lwip_format_inet_addr(&ip, port); + return mp_obj_new_tuple(2, tuple); } static MP_DEFINE_CONST_FUN_OBJ_2(lwip_socket_recvfrom_obj, lwip_socket_recvfrom); From ec527a11136e1c83ddf51099a418ea785215284f Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 14 Jan 2025 11:38:43 +1100 Subject: [PATCH 013/110] esp32: Disable component manager when running 'make submodules'. - ECHO_SUBMODULES=1 exits CMake early. With idf_component_manager 1.x this seems to leave the managed_components directory in a state that causes later builds to fail. - Looks like the component manager isn't needed for this step, so disable it. This invocation logs a warning (not visible in normal output) but completes successfully and returns the correct list of submodules. Signed-off-by: Angus Gratton --- ports/esp32/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ports/esp32/Makefile b/ports/esp32/Makefile index 8888180766b79..1c2fa82ce1265 100644 --- a/ports/esp32/Makefile +++ b/ports/esp32/Makefile @@ -112,6 +112,6 @@ size-files: # output and passes the list of submodules to py/mkrules.mk which does the # `git submodule init` on each. submodules: - @GIT_SUBMODULES=$$(idf.py $(IDFPY_FLAGS) -B $(BUILD)/submodules -D ECHO_SUBMODULES=1 build 2>&1 | \ + @GIT_SUBMODULES=$$(IDF_COMPONENT_MANAGER=0 idf.py $(IDFPY_FLAGS) -B $(BUILD)/submodules -D ECHO_SUBMODULES=1 build 2>&1 | \ grep '^GIT_SUBMODULES=' | cut -d= -f2); \ $(MAKE) -f ../../py/mkrules.mk GIT_SUBMODULES="$${GIT_SUBMODULES}" submodules From 43e3ab6131b11d7fb7409a35fe5127317ba27397 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 14 Jan 2025 11:39:32 +1100 Subject: [PATCH 014/110] esp32: Don't add TinyUSB files to an ECHO_SUBMODULES build. Similar to other places, CMake will error out if this file doesn't exist yet but we don't want this if we're only getting the list of submodules. This work was funded through GitHub Sponsors. Signed-off-by: Angus Gratton --- ports/esp32/esp32_common.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ports/esp32/esp32_common.cmake b/ports/esp32/esp32_common.cmake index b6bfcdacc3cb4..e03d2ff11aada 100644 --- a/ports/esp32/esp32_common.cmake +++ b/ports/esp32/esp32_common.cmake @@ -60,7 +60,7 @@ list(APPEND MICROPY_SOURCE_DRIVERS ) string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/tinyusb) -if(MICROPY_PY_TINYUSB) +if(MICROPY_PY_TINYUSB AND NOT ECHO_SUBMODULES) set(TINYUSB_SRC "${MICROPY_DIR}/lib/tinyusb/src") string(TOUPPER OPT_MCU_${IDF_TARGET} tusb_mcu) From 22353e9e1ed5bf8bc5bad179896ca2877f7511e2 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 14 Jan 2025 11:50:27 +1100 Subject: [PATCH 015/110] py/mkrules: Add GIT_SUBMODULES_FAIL_IF_EMPTY flag for CMake ports. The way CMake gathers the submodule list, it can quietly be empty if the previous step fails. This makes it an explicit error. This work was funded through GitHub Sponsors. Signed-off-by: Angus Gratton --- ports/esp32/Makefile | 2 +- ports/rp2/Makefile | 2 +- py/mkrules.mk | 7 +++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ports/esp32/Makefile b/ports/esp32/Makefile index 1c2fa82ce1265..1ce4d97208da7 100644 --- a/ports/esp32/Makefile +++ b/ports/esp32/Makefile @@ -114,4 +114,4 @@ size-files: submodules: @GIT_SUBMODULES=$$(IDF_COMPONENT_MANAGER=0 idf.py $(IDFPY_FLAGS) -B $(BUILD)/submodules -D ECHO_SUBMODULES=1 build 2>&1 | \ grep '^GIT_SUBMODULES=' | cut -d= -f2); \ - $(MAKE) -f ../../py/mkrules.mk GIT_SUBMODULES="$${GIT_SUBMODULES}" submodules + $(MAKE) -f ../../py/mkrules.mk GIT_SUBMODULES="$${GIT_SUBMODULES}" GIT_SUBMODULES_FAIL_IF_EMPTY=1 submodules diff --git a/ports/rp2/Makefile b/ports/rp2/Makefile index afa21cc7a4383..200899d338e6a 100644 --- a/ports/rp2/Makefile +++ b/ports/rp2/Makefile @@ -76,4 +76,4 @@ submodules: $(MAKE) -f ../../py/mkrules.mk GIT_SUBMODULES="lib/pico-sdk" submodules @GIT_SUBMODULES=$$(cmake -B $(BUILD)/submodules -DECHO_SUBMODULES=1 ${CMAKE_ARGS} -S . 2>&1 | \ grep '^GIT_SUBMODULES=' | cut -d= -f2); \ - $(MAKE) -f ../../py/mkrules.mk GIT_SUBMODULES="$${GIT_SUBMODULES}" submodules + $(MAKE) -f ../../py/mkrules.mk GIT_SUBMODULES="$${GIT_SUBMODULES}" GIT_SUBMODULES_FAIL_IF_EMPTY=1 submodules diff --git a/py/mkrules.mk b/py/mkrules.mk index 74978b0d15bf2..373bda8996f19 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -257,6 +257,13 @@ ifneq ($(GIT_SUBMODULES),) $(Q)cd $(TOP) && git submodule sync $(GIT_SUBMODULES) $(Q)cd $(TOP) && git submodule update --init --filter=blob:none $(GIT_SUBMODULES) || \ git submodule update --init $(GIT_SUBMODULES) +else +ifeq ($(GIT_SUBMODULES_FAIL_IF_EMPTY),1) + # If you see this error, it may mean the internal step run by the port's build + # system to find git submodules has failed. Double-check dependencies are set correctly. + $(ECHO) "Internal build error: The submodule list should not be empty." + exit 1 +endif endif .PHONY: submodules From abb13b1e1e37c42876f69c83b96c9ead3285a9a7 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 25 Jan 2025 17:45:40 +1100 Subject: [PATCH 016/110] extmod/lwip-include: Factor common lwIP config into lwipopts_common.h. This lwIP configuration file has options that are common to all ports, and the ports are updated to use this file. This change is a no-op, the lwIP configuration remains the same for the four ports using this common file. This reduces code duplication, keeps the ports in sync, and makes it easier to update the configuration for all ports at once. Signed-off-by: Damien George --- extmod/lwip-include/lwipopts_common.h | 109 ++++++++++++++++++++++++++ ports/mimxrt/lwip_inc/lwipopts.h | 47 +---------- ports/renesas-ra/lwip_inc/lwipopts.h | 42 +--------- ports/rp2/lwip_inc/lwipopts.h | 56 +------------ ports/stm32/lwip_inc/lwipopts.h | 77 +----------------- 5 files changed, 122 insertions(+), 209 deletions(-) create mode 100644 extmod/lwip-include/lwipopts_common.h diff --git a/extmod/lwip-include/lwipopts_common.h b/extmod/lwip-include/lwipopts_common.h new file mode 100644 index 0000000000000..8688db064ef6f --- /dev/null +++ b/extmod/lwip-include/lwipopts_common.h @@ -0,0 +1,109 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2025 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef MICROPY_INCLUDED_LWIPOPTS_COMMON_H +#define MICROPY_INCLUDED_LWIPOPTS_COMMON_H + +#include "py/mpconfig.h" + +// This sys-arch protection is not needed. +// Ports either protect lwIP code with flags, or run it at PendSV priority. +#define SYS_ARCH_DECL_PROTECT(lev) do { } while (0) +#define SYS_ARCH_PROTECT(lev) do { } while (0) +#define SYS_ARCH_UNPROTECT(lev) do { } while (0) + +#define NO_SYS 1 +#define SYS_LIGHTWEIGHT_PROT 1 +#define MEM_ALIGNMENT 4 + +#define LWIP_CHKSUM_ALGORITHM 3 +#define LWIP_CHECKSUM_CTRL_PER_NETIF 1 + +#define LWIP_ARP 1 +#define LWIP_ETHERNET 1 +#define LWIP_RAW 1 +#define LWIP_NETCONN 0 +#define LWIP_SOCKET 0 +#define LWIP_STATS 0 +#define LWIP_NETIF_HOSTNAME 1 + +#define LWIP_DHCP 1 +#define LWIP_DHCP_CHECK_LINK_UP 1 +#define LWIP_DHCP_DOES_ACD_CHECK 0 // to speed DHCP up +#define LWIP_DNS 1 +#define LWIP_DNS_SUPPORT_MDNS_QUERIES 1 +#define LWIP_MDNS_RESPONDER 1 +#define LWIP_IGMP 1 + +#if MICROPY_PY_LWIP_PPP +#define PPP_SUPPORT 1 +#define PAP_SUPPORT 1 +#define CHAP_SUPPORT 1 +#endif + +#define LWIP_NUM_NETIF_CLIENT_DATA LWIP_MDNS_RESPONDER +#define MEMP_NUM_UDP_PCB (4 + LWIP_MDNS_RESPONDER) +#define MEMP_NUM_SYS_TIMEOUT (LWIP_NUM_SYS_TIMEOUT_INTERNAL + LWIP_MDNS_RESPONDER) + +#define SO_REUSE 1 +#define TCP_LISTEN_BACKLOG 1 + +// TCP memory settings. +// Default lwIP settings takes 15800 bytes; TCP d/l: 380k/s local, 7.2k/s remote; TCP u/l is very slow. +#ifndef MEM_SIZE + +#if 0 +// lwIP takes 19159 bytes; TCP d/l and u/l are around 320k/s on local network. +#define MEM_SIZE (5000) +#define TCP_WND (4 * TCP_MSS) +#define TCP_SND_BUF (4 * TCP_MSS) +#endif + +#if 1 +// lwIP takes 26700 bytes; TCP dl/ul are around 750/600 k/s on local network. +#define MEM_SIZE (8000) +#define TCP_MSS (800) +#define TCP_WND (8 * TCP_MSS) +#define TCP_SND_BUF (8 * TCP_MSS) +#define MEMP_NUM_TCP_SEG (32) +#endif + +#if 0 +// lwIP takes 45600 bytes; TCP dl/ul are around 1200/1000 k/s on local network. +#define MEM_SIZE (16000) +#define TCP_MSS (1460) +#define TCP_WND (8 * TCP_MSS) +#define TCP_SND_BUF (8 * TCP_MSS) +#define MEMP_NUM_TCP_SEG (32) +#endif + +#endif // MEM_SIZE + +// Needed for PPP. +#define sys_jiffies sys_now + +typedef uint32_t sys_prot_t; + +#endif // MICROPY_INCLUDED_LWIPOPTS_COMMON_H diff --git a/ports/mimxrt/lwip_inc/lwipopts.h b/ports/mimxrt/lwip_inc/lwipopts.h index 411ca4b60f342..cf25597f95c1f 100644 --- a/ports/mimxrt/lwip_inc/lwipopts.h +++ b/ports/mimxrt/lwip_inc/lwipopts.h @@ -1,56 +1,15 @@ #ifndef MICROPY_INCLUDED_MIMXRT_LWIP_LWIPOPTS_H #define MICROPY_INCLUDED_MIMXRT_LWIP_LWIPOPTS_H -#include - -// This protection is not needed, instead we execute all lwIP code at PendSV priority -#define SYS_ARCH_DECL_PROTECT(lev) do { } while (0) -#define SYS_ARCH_PROTECT(lev) do { } while (0) -#define SYS_ARCH_UNPROTECT(lev) do { } while (0) - -#define NO_SYS 1 -#define SYS_LIGHTWEIGHT_PROT 1 -#define MEM_ALIGNMENT 4 - -#define LWIP_CHKSUM_ALGORITHM 3 -// The checksum flags are set in eth.c -#define LWIP_CHECKSUM_CTRL_PER_NETIF 1 - -#define LWIP_ARP 1 -#define LWIP_ETHERNET 1 -#define LWIP_RAW 1 -#define LWIP_NETCONN 0 -#define LWIP_SOCKET 0 -#define LWIP_STATS 0 -#define LWIP_NETIF_HOSTNAME 1 #define LWIP_NETIF_EXT_STATUS_CALLBACK 1 #define LWIP_IPV6 0 -#define LWIP_DHCP 1 -#define LWIP_DHCP_CHECK_LINK_UP 1 -#define LWIP_DHCP_DOES_ACD_CHECK 0 // to speed DHCP up -#define LWIP_DNS 1 -#define LWIP_DNS_SUPPORT_MDNS_QUERIES 1 -#define LWIP_MDNS_RESPONDER 1 -#define LWIP_IGMP 1 -#define LWIP_NUM_NETIF_CLIENT_DATA LWIP_MDNS_RESPONDER -#define MEMP_NUM_UDP_PCB (4 + LWIP_MDNS_RESPONDER) -#define MEMP_NUM_SYS_TIMEOUT (LWIP_NUM_SYS_TIMEOUT_INTERNAL + LWIP_MDNS_RESPONDER) - -#define SO_REUSE 1 -#define TCP_LISTEN_BACKLOG 1 - -extern uint32_t trng_random_u32(void); #define LWIP_RAND() trng_random_u32() -// lwip takes 26700 bytes -#define MEM_SIZE (8000) -#define TCP_MSS (800) -#define TCP_WND (8 * TCP_MSS) -#define TCP_SND_BUF (8 * TCP_MSS) -#define MEMP_NUM_TCP_SEG (32) +// Include common lwIP configuration. +#include "extmod/lwip-include/lwipopts_common.h" -typedef uint32_t sys_prot_t; +extern uint32_t trng_random_u32(void); #endif // MICROPY_INCLUDED_MIMXRT_LWIP_LWIPOPTS_H diff --git a/ports/renesas-ra/lwip_inc/lwipopts.h b/ports/renesas-ra/lwip_inc/lwipopts.h index 74e39022a6a85..77645ae6c3c3d 100644 --- a/ports/renesas-ra/lwip_inc/lwipopts.h +++ b/ports/renesas-ra/lwip_inc/lwipopts.h @@ -1,45 +1,8 @@ #ifndef MICROPY_INCLUDED_RA_LWIP_LWIPOPTS_H #define MICROPY_INCLUDED_RA_LWIP_LWIPOPTS_H -#include - -// This protection is not needed, instead protect lwIP code with flags -#define SYS_ARCH_DECL_PROTECT(lev) do { } while (0) -#define SYS_ARCH_PROTECT(lev) do { } while (0) -#define SYS_ARCH_UNPROTECT(lev) do { } while (0) - -#define NO_SYS 1 -#define SYS_LIGHTWEIGHT_PROT 1 -#define MEM_ALIGNMENT 4 - -#define LWIP_CHKSUM_ALGORITHM 3 -#define LWIP_CHECKSUM_CTRL_PER_NETIF 1 - -#define LWIP_ARP 1 -#define LWIP_ETHERNET 1 -#define LWIP_RAW 1 -#define LWIP_NETCONN 0 -#define LWIP_SOCKET 0 -#define LWIP_STATS 0 -#define LWIP_NETIF_HOSTNAME 1 - #define LWIP_IPV6 0 -#define LWIP_DHCP 1 -#define LWIP_DHCP_CHECK_LINK_UP 1 -#define LWIP_DHCP_DOES_ACD_CHECK 0 // to speed DHCP up -#define LWIP_DNS 1 -#define LWIP_DNS_SUPPORT_MDNS_QUERIES 1 -#define LWIP_MDNS_RESPONDER 1 -#define LWIP_IGMP 1 -#define LWIP_NUM_NETIF_CLIENT_DATA LWIP_MDNS_RESPONDER -#define MEMP_NUM_UDP_PCB (4 + LWIP_MDNS_RESPONDER) -#define MEMP_NUM_SYS_TIMEOUT (LWIP_NUM_SYS_TIMEOUT_INTERNAL + LWIP_MDNS_RESPONDER) - -#define SO_REUSE 1 -#define TCP_LISTEN_BACKLOG 1 - -extern uint32_t rng_read(void); #define LWIP_RAND() rng_read() #define MEM_SIZE (16 * 1024) @@ -51,6 +14,9 @@ extern uint32_t rng_read(void); #define TCP_QUEUE_OOSEQ (1) #define MEMP_NUM_TCP_SEG (2 * TCP_SND_QUEUELEN) -typedef uint32_t sys_prot_t; +// Include common lwIP configuration. +#include "extmod/lwip-include/lwipopts_common.h" + +extern uint32_t rng_read(void); #endif // MICROPY_INCLUDED_RA_LWIP_LWIPOPTS_H diff --git a/ports/rp2/lwip_inc/lwipopts.h b/ports/rp2/lwip_inc/lwipopts.h index 15679e0cf018d..da45a5735b28c 100644 --- a/ports/rp2/lwip_inc/lwipopts.h +++ b/ports/rp2/lwip_inc/lwipopts.h @@ -1,28 +1,6 @@ #ifndef MICROPY_INCLUDED_RP2_LWIP_LWIPOPTS_H #define MICROPY_INCLUDED_RP2_LWIP_LWIPOPTS_H -#include "py/mpconfig.h" - -// This protection is not needed, instead protect lwIP code with flags -#define SYS_ARCH_DECL_PROTECT(lev) do { } while (0) -#define SYS_ARCH_PROTECT(lev) do { } while (0) -#define SYS_ARCH_UNPROTECT(lev) do { } while (0) - -#define NO_SYS 1 -#define SYS_LIGHTWEIGHT_PROT 1 -#define MEM_ALIGNMENT 4 - -#define LWIP_CHKSUM_ALGORITHM 3 -// The checksum flags are set in eth.c -#define LWIP_CHECKSUM_CTRL_PER_NETIF 1 - -#define LWIP_ARP 1 -#define LWIP_ETHERNET 1 -#define LWIP_RAW 1 -#define LWIP_NETCONN 0 -#define LWIP_SOCKET 0 -#define LWIP_STATS 0 -#define LWIP_NETIF_HOSTNAME 1 #define LWIP_NETIF_EXT_STATUS_CALLBACK 1 #define LWIP_NETIF_STATUS_CALLBACK 1 @@ -30,40 +8,12 @@ #define LWIP_IPV6 1 #define LWIP_ND6_NUM_DESTINATIONS 4 #define LWIP_ND6_QUEUEING 0 -#define LWIP_DHCP 1 -#define LWIP_DHCP_CHECK_LINK_UP 1 -#define LWIP_DHCP_DOES_ACD_CHECK 0 // to speed DHCP up -#define LWIP_DNS 1 -#define LWIP_DNS_SUPPORT_MDNS_QUERIES 1 -#define LWIP_MDNS_RESPONDER 1 -#define LWIP_IGMP 1 - -#if MICROPY_PY_LWIP_PPP -#define PPP_SUPPORT 1 -#define PAP_SUPPORT 1 -#define CHAP_SUPPORT 1 -#endif -#define LWIP_NUM_NETIF_CLIENT_DATA LWIP_MDNS_RESPONDER -#define MEMP_NUM_UDP_PCB (4 + LWIP_MDNS_RESPONDER) -#define MEMP_NUM_SYS_TIMEOUT (LWIP_NUM_SYS_TIMEOUT_INTERNAL + LWIP_MDNS_RESPONDER) - -#define SO_REUSE 1 -#define TCP_LISTEN_BACKLOG 1 - -extern uint32_t rosc_random_u32(void); #define LWIP_RAND() rosc_random_u32() -// lwip takes 26700 bytes -#define MEM_SIZE (8000) -#define TCP_MSS (800) -#define TCP_WND (8 * TCP_MSS) -#define TCP_SND_BUF (8 * TCP_MSS) -#define MEMP_NUM_TCP_SEG (32) +// Include common lwIP configuration. +#include "extmod/lwip-include/lwipopts_common.h" -typedef uint32_t sys_prot_t; - -// Needed for PPP. -#define sys_jiffies sys_now +extern uint32_t rosc_random_u32(void); #endif // MICROPY_INCLUDED_RP2_LWIP_LWIPOPTS_H diff --git a/ports/stm32/lwip_inc/lwipopts.h b/ports/stm32/lwip_inc/lwipopts.h index 9e1aa8d0f14c0..9e2402c8dc5dc 100644 --- a/ports/stm32/lwip_inc/lwipopts.h +++ b/ports/stm32/lwip_inc/lwipopts.h @@ -1,89 +1,18 @@ #ifndef MICROPY_INCLUDED_STM32_LWIP_LWIPOPTS_H #define MICROPY_INCLUDED_STM32_LWIP_LWIPOPTS_H -#include "py/mpconfig.h" - -// This protection is not needed, instead we execute all lwIP code at PendSV priority -#define SYS_ARCH_DECL_PROTECT(lev) do { } while (0) -#define SYS_ARCH_PROTECT(lev) do { } while (0) -#define SYS_ARCH_UNPROTECT(lev) do { } while (0) - -#define NO_SYS 1 -#define SYS_LIGHTWEIGHT_PROT 1 -#define MEM_ALIGNMENT 4 - -#define LWIP_CHKSUM_ALGORITHM 3 -#define LWIP_CHECKSUM_CTRL_PER_NETIF 1 - -#define LWIP_ARP 1 -#define LWIP_ETHERNET 1 -#define LWIP_RAW 1 -#define LWIP_NETCONN 0 -#define LWIP_SOCKET 0 -#define LWIP_STATS 0 -#define LWIP_NETIF_HOSTNAME 1 #define LWIP_NETIF_EXT_STATUS_CALLBACK 1 #define LWIP_LOOPIF_MULTICAST 1 #define LWIP_LOOPBACK_MAX_PBUFS 8 #define LWIP_IPV6 0 -#define LWIP_DHCP 1 -#define LWIP_DHCP_CHECK_LINK_UP 1 -#define LWIP_DHCP_DOES_ACD_CHECK 0 // to speed DHCP up -#define LWIP_DNS 1 -#define LWIP_DNS_SUPPORT_MDNS_QUERIES 1 -#define LWIP_MDNS_RESPONDER 1 -#define LWIP_IGMP 1 - -#if MICROPY_PY_LWIP_PPP -#define PPP_SUPPORT 1 -#define PAP_SUPPORT 1 -#define CHAP_SUPPORT 1 -#endif - -#define LWIP_NUM_NETIF_CLIENT_DATA LWIP_MDNS_RESPONDER -#define MEMP_NUM_UDP_PCB (4 + LWIP_MDNS_RESPONDER) -#define MEMP_NUM_SYS_TIMEOUT (LWIP_NUM_SYS_TIMEOUT_INTERNAL + LWIP_MDNS_RESPONDER) - -#define SO_REUSE 1 -#define TCP_LISTEN_BACKLOG 1 -extern uint32_t rng_get(void); #define LWIP_RAND() rng_get() -// default -// lwip takes 15800 bytes; TCP d/l: 380k/s local, 7.2k/s remote -// TCP u/l is very slow - -#if 0 -// lwip takes 19159 bytes; TCP d/l and u/l are around 320k/s on local network -#define MEM_SIZE (5000) -#define TCP_WND (4 * TCP_MSS) -#define TCP_SND_BUF (4 * TCP_MSS) -#endif - -#if 1 -// lwip takes 26700 bytes; TCP dl/ul are around 750/600 k/s on local network -#define MEM_SIZE (8000) -#define TCP_MSS (800) -#define TCP_WND (8 * TCP_MSS) -#define TCP_SND_BUF (8 * TCP_MSS) -#define MEMP_NUM_TCP_SEG (32) -#endif +// Include common lwIP configuration. +#include "extmod/lwip-include/lwipopts_common.h" -#if 0 -// lwip takes 45600 bytes; TCP dl/ul are around 1200/1000 k/s on local network -#define MEM_SIZE (16000) -#define TCP_MSS (1460) -#define TCP_WND (8 * TCP_MSS) -#define TCP_SND_BUF (8 * TCP_MSS) -#define MEMP_NUM_TCP_SEG (32) -#endif - -typedef uint32_t sys_prot_t; - -// Needed for PPP. -#define sys_jiffies sys_now +extern uint32_t rng_get(void); #endif // MICROPY_INCLUDED_STM32_LWIP_LWIPOPTS_H From bfb1bee6feba9f06452a0e4b572ec4d15df325cf Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sun, 26 Jan 2025 20:54:14 -0600 Subject: [PATCH 017/110] py/parsenumbase: Favor clarity of code over manual optimisation. Follow up to 13b13d1fdd05549d504eeded0b5aa8871d5e5dcf, based on some testing on godbolt, the manual code optimisation seems unnecessary for code size, at least on gcc x86_64 and ARM, and it's definitely not good for clarity. Signed-off-by: Jeff Epler --- py/parsenumbase.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/py/parsenumbase.c b/py/parsenumbase.c index cc3275c456e40..fbf07a119584f 100644 --- a/py/parsenumbase.c +++ b/py/parsenumbase.c @@ -41,11 +41,11 @@ size_t mp_parse_num_base(const char *str, size_t len, int *base) { if (c == '0') { c = *(p++) | 32; int b = *base; - if (c == 'x' && !(b & ~16)) { + if (c == 'x' && (b == 0 || b == 16)) { *base = 16; - } else if (c == 'o' && !(b & ~8)) { + } else if (c == 'o' && (b == 0 || b == 8)) { *base = 8; - } else if (c == 'b' && !(b & ~2)) { + } else if (c == 'b' && (b == 0 || b == 2)) { *base = 2; } else { p -= 2; From 195bf051153cff1d49b3645c51099e9ff6e1d80d Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 15 Oct 2024 12:23:28 +1100 Subject: [PATCH 018/110] tests: Add a test for SSL socket memory leaks. Test is for an issue reported on the micropython-lib Discord as effecting the rp2 port umqtt.simple interface when reconnecting with TLS, however it's a more generic problem. Currently this test fails on RPI_PICO_W and ESP32_GENERIC_C3 (and no doubt others). Fixes are in the subsequent commits. This work was funded through GitHub Sponsors. Signed-off-by: Angus Gratton --- tests/extmod/ssl_noleak.py | 50 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tests/extmod/ssl_noleak.py diff --git a/tests/extmod/ssl_noleak.py b/tests/extmod/ssl_noleak.py new file mode 100644 index 0000000000000..870032d58e63e --- /dev/null +++ b/tests/extmod/ssl_noleak.py @@ -0,0 +1,50 @@ +# Ensure that SSLSockets can be allocated sequentially +# without running out of available memory. +try: + import io + import tls +except ImportError: + print("SKIP") + raise SystemExit + +import unittest + + +class TestSocket(io.IOBase): + def write(self, buf): + return len(buf) + + def readinto(self, buf): + return 0 + + def ioctl(self, cmd, arg): + return 0 + + def setblocking(self, value): + pass + + +ITERS = 128 + + +class TLSNoLeaks(unittest.TestCase): + def test_unique_context(self): + for n in range(ITERS): + print(n) + s = TestSocket() + ctx = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT) + ctx.verify_mode = tls.CERT_NONE + s = ctx.wrap_socket(s, do_handshake_on_connect=False) + + def test_shared_context(self): + # Single SSLContext, multiple sockets + ctx = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT) + ctx.verify_mode = tls.CERT_NONE + for n in range(ITERS): + print(n) + s = TestSocket() + s = ctx.wrap_socket(s, do_handshake_on_connect=False) + + +if __name__ == "__main__": + unittest.main() From 97f444bfa04408dd510f118baaea5760b93bd892 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 28 Jan 2025 15:31:18 +1100 Subject: [PATCH 019/110] extmod/mbedtls: Try GC before failing to setup socket on esp32, unix. On mbedTLS ports with non-baremetal configs (mostly esp32, technically also unix port), mbedTLS memory is allocated from the libc heap. This means an old SSL socket may be holding large SSL buffers and preventing a new SSL socket from being allocated. As a workaround, trigger a GC pass and retry before failing outright. This was originally implemented as a global mbedTLS calloc function, but there is complexity around the possibility of C user modules calling into mbedTLS without holding the GIL. It would be interesting to try making a generic version for any malloc which fails, but this would require checking for a Python thread and probably making the GIL recursive. This work was funded through GitHub Sponsors. Signed-off-by: Angus Gratton --- extmod/modtls_mbedtls.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/extmod/modtls_mbedtls.c b/extmod/modtls_mbedtls.c index 0a1b8828af5ca..3fd416d72f5ef 100644 --- a/extmod/modtls_mbedtls.c +++ b/extmod/modtls_mbedtls.c @@ -37,6 +37,7 @@ #include "py/stream.h" #include "py/objstr.h" #include "py/reader.h" +#include "py/gc.h" #include "extmod/vfs.h" // mbedtls_time_t @@ -58,6 +59,10 @@ #include "mbedtls/asn1.h" #endif +#ifndef MICROPY_MBEDTLS_CONFIG_BARE_METAL +#define MICROPY_MBEDTLS_CONFIG_BARE_METAL (0) +#endif + #define MP_STREAM_POLL_RDWR (MP_STREAM_POLL_RD | MP_STREAM_POLL_WR) // This corresponds to an SSLContext object. @@ -545,6 +550,16 @@ static mp_obj_t ssl_socket_make_new(mp_obj_ssl_context_t *ssl_context, mp_obj_t mbedtls_ssl_init(&o->ssl); ret = mbedtls_ssl_setup(&o->ssl, &ssl_context->conf); + #if !MICROPY_MBEDTLS_CONFIG_BARE_METAL + if (ret == MBEDTLS_ERR_SSL_ALLOC_FAILED) { + // If mbedTLS relies on platform libc heap for buffers (i.e. esp32 + // port), then run a GC pass and then try again. This is useful because + // it may free a Python object (like an old SSL socket) whose finaliser + // frees some platform-level heap. + gc_collect(); + ret = mbedtls_ssl_setup(&o->ssl, &ssl_context->conf); + } + #endif if (ret != 0) { goto cleanup; } From d642cce27a9a07e043211f099c31fca390f96f1a Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Wed, 16 Oct 2024 16:43:13 +1100 Subject: [PATCH 020/110] unix: Use the bare metal mbedTLS config in the coverage buiid. This allows coverage to test MicroPython-specific features such as the tracked alloc cleanups added in the parent commit. This work was funded through GitHub Sponsors. Signed-off-by: Angus Gratton --- ports/unix/mbedtls/mbedtls_config_port.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ports/unix/mbedtls/mbedtls_config_port.h b/ports/unix/mbedtls/mbedtls_config_port.h index c619de9b8b1b9..aec65e6581e73 100644 --- a/ports/unix/mbedtls/mbedtls_config_port.h +++ b/ports/unix/mbedtls/mbedtls_config_port.h @@ -32,7 +32,18 @@ // Enable mbedtls modules #define MBEDTLS_TIMING_C +#if defined(MICROPY_UNIX_COVERAGE) +// Test the "bare metal" memory management in the coverage build +#define MICROPY_MBEDTLS_CONFIG_BARE_METAL (1) +#endif + // Include common mbedtls configuration. #include "extmod/mbedtls/mbedtls_config_common.h" +#if defined(MICROPY_UNIX_COVERAGE) +// See comment above, but fall back to the default platform entropy functions +#undef MBEDTLS_ENTROPY_HARDWARE_ALT +#undef MBEDTLS_NO_PLATFORM_ENTROPY +#endif + #endif /* MICROPY_INCLUDED_MBEDTLS_CONFIG_H */ From 8a2ff2ca7366f605dd55c93f6b393552b365cd10 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Wed, 4 Dec 2024 10:53:08 +1100 Subject: [PATCH 021/110] py/gc: Split out running finalizers to a separate pass. Currently a finalizer may run and access memory which has already been freed. (This happens mostly during gc_sweep_all() but could happen during any garbage collection pass.) Includes some speed improvement tweaks to skip empty FTB blocks. These help compensate for the inherent slowdown of having to walk the heap twice. Signed-off-by: Angus Gratton --- py/gc.c | 67 ++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 24 deletions(-) diff --git a/py/gc.c b/py/gc.c index bee44925076f0..297f4f85d3f6a 100644 --- a/py/gc.c +++ b/py/gc.c @@ -477,29 +477,20 @@ static void gc_deal_with_stack_overflow(void) { } } -static void gc_sweep(void) { - #if MICROPY_PY_GC_COLLECT_RETVAL - MP_STATE_MEM(gc_collected) = 0; - #endif - // free unmarked heads and their tails - int free_tail = 0; - #if MICROPY_GC_SPLIT_HEAP_AUTO - mp_state_mem_area_t *prev_area = NULL; - #endif - for (mp_state_mem_area_t *area = &MP_STATE_MEM(area); area != NULL; area = NEXT_AREA(area)) { - size_t end_block = area->gc_alloc_table_byte_len * BLOCKS_PER_ATB; - if (area->gc_last_used_block < end_block) { - end_block = area->gc_last_used_block + 1; - } - - size_t last_used_block = 0; - - for (size_t block = 0; block < end_block; block++) { - MICROPY_GC_HOOK_LOOP(block); - switch (ATB_GET_KIND(area, block)) { - case AT_HEAD: - #if MICROPY_ENABLE_FINALISER - if (FTB_GET(area, block)) { +// Run finalisers for all to-be-freed blocks +static void gc_sweep_run_finalisers(void) { + #if MICROPY_ENABLE_FINALISER + for (const mp_state_mem_area_t *area = &MP_STATE_MEM(area); area != NULL; area = NEXT_AREA(area)) { + assert(area->gc_last_used_block <= area->gc_alloc_table_byte_len * BLOCKS_PER_ATB); + // Small speed optimisation: skip over empty FTB blocks + size_t ftb_end = area->gc_last_used_block / BLOCKS_PER_FTB; // index is inclusive + for (size_t ftb_idx = 0; ftb_idx <= ftb_end; ftb_idx++) { + byte ftb = area->gc_finaliser_table_start[ftb_idx]; + size_t block = ftb_idx * BLOCKS_PER_FTB; + while (ftb) { + MICROPY_GC_HOOK_LOOP(block); + if (ftb & 1) { // FTB_GET(area, block) shortcut + if (ATB_GET_KIND(area, block) == AT_HEAD) { mp_obj_base_t *obj = (mp_obj_base_t *)PTR_FROM_BLOCK(area, block); if (obj->type != NULL) { // if the object has a type then see if it has a __del__ method @@ -519,7 +510,35 @@ static void gc_sweep(void) { // clear finaliser flag FTB_CLEAR(area, block); } - #endif + } + ftb >>= 1; + block++; + } + } + } + #endif // MICROPY_ENABLE_FINALISER +} + +static void gc_sweep(void) { + #if MICROPY_PY_GC_COLLECT_RETVAL + MP_STATE_MEM(gc_collected) = 0; + #endif + // free unmarked heads and their tails + int free_tail = 0; + #if MICROPY_GC_SPLIT_HEAP_AUTO + mp_state_mem_area_t *prev_area = NULL; + #endif + + gc_sweep_run_finalisers(); + + for (mp_state_mem_area_t *area = &MP_STATE_MEM(area); area != NULL; area = NEXT_AREA(area)) { + size_t last_used_block = 0; + assert(area->gc_last_used_block <= area->gc_alloc_table_byte_len * BLOCKS_PER_ATB); + + for (size_t block = 0; block <= area->gc_last_used_block; block++) { + MICROPY_GC_HOOK_LOOP(block); + switch (ATB_GET_KIND(area, block)) { + case AT_HEAD: free_tail = 1; DEBUG_printf("gc_sweep(%p)\n", (void *)PTR_FROM_BLOCK(area, block)); #if MICROPY_PY_GC_COLLECT_RETVAL From 40e1c111e17864044190596dff6d32955d11280c Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Wed, 4 Dec 2024 10:58:06 +1100 Subject: [PATCH 022/110] py/gc: Allow gc_free from inside a gc_sweep finalizer. Do this by tracking being inside gc collection with a separate flag, GC_COLLECT_FLAG. In gc_free(), ignore this flag when determining if the heap is locked. * For finalisers calling gc_free() when heap is otherwise unlocked, this allows memory to be immediately freed (potentially avoiding a MemoryError). * Hard IRQs still can't call gc_free(), as heap will be locked via gc_lock(). * If finalisers are disabled then all of this code can be compiled out to save some code size. Signed-off-by: Angus Gratton --- py/gc.c | 35 +++++++++++++++++++++-------------- py/modmicropython.c | 4 ++-- py/mpstate.h | 13 +++++++++++++ 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/py/gc.c b/py/gc.c index 297f4f85d3f6a..3d48e062494a8 100644 --- a/py/gc.c +++ b/py/gc.c @@ -334,12 +334,12 @@ void gc_lock(void) { // - each thread has its own gc_lock_depth so there are no races between threads; // - a hard interrupt will only change gc_lock_depth during its execution, and // upon return will restore the value of gc_lock_depth. - MP_STATE_THREAD(gc_lock_depth)++; + MP_STATE_THREAD(gc_lock_depth) += (1 << GC_LOCK_DEPTH_SHIFT); } void gc_unlock(void) { // This does not need to be atomic, See comment above in gc_lock. - MP_STATE_THREAD(gc_lock_depth)--; + MP_STATE_THREAD(gc_lock_depth) -= (1 << GC_LOCK_DEPTH_SHIFT); } bool gc_is_locked(void) { @@ -581,13 +581,18 @@ static void gc_sweep(void) { } } -void gc_collect_start(void) { +static void gc_collect_start_common(void) { GC_ENTER(); - MP_STATE_THREAD(gc_lock_depth)++; + assert((MP_STATE_THREAD(gc_lock_depth) & GC_COLLECT_FLAG) == 0); + MP_STATE_THREAD(gc_lock_depth) |= GC_COLLECT_FLAG; + MP_STATE_MEM(gc_stack_overflow) = 0; +} + +void gc_collect_start(void) { + gc_collect_start_common(); #if MICROPY_GC_ALLOC_THRESHOLD MP_STATE_MEM(gc_alloc_amount) = 0; #endif - MP_STATE_MEM(gc_stack_overflow) = 0; // Trace root pointers. This relies on the root pointers being organised // correctly in the mp_state_ctx structure. We scan nlr_top, dict_locals, @@ -658,14 +663,12 @@ void gc_collect_end(void) { for (mp_state_mem_area_t *area = &MP_STATE_MEM(area); area != NULL; area = NEXT_AREA(area)) { area->gc_last_free_atb_index = 0; } - MP_STATE_THREAD(gc_lock_depth)--; + MP_STATE_THREAD(gc_lock_depth) &= ~GC_COLLECT_FLAG; GC_EXIT(); } void gc_sweep_all(void) { - GC_ENTER(); - MP_STATE_THREAD(gc_lock_depth)++; - MP_STATE_MEM(gc_stack_overflow) = 0; + gc_collect_start_common(); gc_collect_end(); } @@ -902,10 +905,13 @@ void *gc_alloc(size_t n_bytes, unsigned int alloc_flags) { // force the freeing of a piece of memory // TODO: freeing here does not call finaliser void gc_free(void *ptr) { - if (MP_STATE_THREAD(gc_lock_depth) > 0) { - // Cannot free while the GC is locked. However free is an optimisation - // to reclaim the memory immediately, this means it will now be left - // until the next collection. + // Cannot free while the GC is locked, unless we're only doing a gc sweep. + // However free is an optimisation to reclaim the memory immediately, this + // means it will now be left until the next collection. + // + // (We have the optimisation to free immediately from inside a gc sweep so + // that finalisers can free more memory when trying to avoid MemoryError.) + if (MP_STATE_THREAD(gc_lock_depth) & ~GC_COLLECT_FLAG) { return; } @@ -930,7 +936,8 @@ void gc_free(void *ptr) { #endif size_t block = BLOCK_FROM_PTR(area, ptr); - assert(ATB_GET_KIND(area, block) == AT_HEAD); + assert(ATB_GET_KIND(area, block) == AT_HEAD + || (ATB_GET_KIND(area, block) == AT_MARK && (MP_STATE_THREAD(gc_lock_depth) & GC_COLLECT_FLAG))); #if MICROPY_ENABLE_FINALISER FTB_CLEAR(area, block); diff --git a/py/modmicropython.c b/py/modmicropython.c index 1bf0a000c2026..d1a687f10e14e 100644 --- a/py/modmicropython.c +++ b/py/modmicropython.c @@ -132,13 +132,13 @@ static MP_DEFINE_CONST_FUN_OBJ_0(mp_micropython_heap_lock_obj, mp_micropython_he static mp_obj_t mp_micropython_heap_unlock(void) { gc_unlock(); - return MP_OBJ_NEW_SMALL_INT(MP_STATE_THREAD(gc_lock_depth)); + return MP_OBJ_NEW_SMALL_INT(MP_STATE_THREAD(gc_lock_depth) >> GC_LOCK_DEPTH_SHIFT); } static MP_DEFINE_CONST_FUN_OBJ_0(mp_micropython_heap_unlock_obj, mp_micropython_heap_unlock); #if MICROPY_PY_MICROPYTHON_HEAP_LOCKED static mp_obj_t mp_micropython_heap_locked(void) { - return MP_OBJ_NEW_SMALL_INT(MP_STATE_THREAD(gc_lock_depth)); + return MP_OBJ_NEW_SMALL_INT(MP_STATE_THREAD(gc_lock_depth) >> GC_LOCK_DEPTH_SHIFT); } static MP_DEFINE_CONST_FUN_OBJ_0(mp_micropython_heap_locked_obj, mp_micropython_heap_locked); #endif diff --git a/py/mpstate.h b/py/mpstate.h index 54eca596daa28..51f290b55b5ed 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -77,6 +77,18 @@ typedef struct _mp_sched_item_t { mp_obj_t arg; } mp_sched_item_t; +// gc_lock_depth field is a combination of the GC_COLLECT_FLAG +// bit and a lock depth shifted GC_LOCK_DEPTH_SHIFT bits left. +#if MICROPY_ENABLE_FINALISER +#define GC_COLLECT_FLAG 1 +#define GC_LOCK_DEPTH_SHIFT 1 +#else +// If finalisers are disabled then this check doesn't matter, as gc_lock() +// is called anywhere else that heap can't be changed. So save some code size. +#define GC_COLLECT_FLAG 0 +#define GC_LOCK_DEPTH_SHIFT 0 +#endif + // This structure holds information about a single contiguous area of // memory reserved for the memory manager. typedef struct _mp_state_mem_area_t { @@ -268,6 +280,7 @@ typedef struct _mp_state_thread_t { #endif // Locking of the GC is done per thread. + // See GC_LOCK_DEPTH_SHIFT for an explanation of this field. uint16_t gc_lock_depth; //////////////////////////////////////////////////////////// From 4bcbe88e74de245fdee029b6a0746e3485d82a7e Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 10 Dec 2024 14:50:42 +1100 Subject: [PATCH 023/110] py: Add optional support for recursive mutexes, use for gc mutex. Enabled by default if using threading and no GIL This work was funded through GitHub Sponsors. Signed-off-by: Angus Gratton --- py/gc.c | 11 ++++++----- py/mpconfig.h | 5 +++++ py/mpstate.h | 2 +- py/mpthread.h | 6 ++++++ 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/py/gc.c b/py/gc.c index 3d48e062494a8..6a3f48ac3c0ce 100644 --- a/py/gc.c +++ b/py/gc.c @@ -113,9 +113,12 @@ #endif #if MICROPY_PY_THREAD && !MICROPY_PY_THREAD_GIL -#define GC_ENTER() mp_thread_mutex_lock(&MP_STATE_MEM(gc_mutex), 1) -#define GC_EXIT() mp_thread_mutex_unlock(&MP_STATE_MEM(gc_mutex)) +#define GC_MUTEX_INIT() mp_thread_recursive_mutex_init(&MP_STATE_MEM(gc_mutex)) +#define GC_ENTER() mp_thread_recursive_mutex_lock(&MP_STATE_MEM(gc_mutex), 1) +#define GC_EXIT() mp_thread_recursive_mutex_unlock(&MP_STATE_MEM(gc_mutex)) #else +// Either no threading, or assume callers to gc_collect() hold the GIL +#define GC_MUTEX_INIT() #define GC_ENTER() #define GC_EXIT() #endif @@ -210,9 +213,7 @@ void gc_init(void *start, void *end) { MP_STATE_MEM(gc_alloc_amount) = 0; #endif - #if MICROPY_PY_THREAD && !MICROPY_PY_THREAD_GIL - mp_thread_mutex_init(&MP_STATE_MEM(gc_mutex)); - #endif + GC_MUTEX_INIT(); } #if MICROPY_GC_SPLIT_HEAP diff --git a/py/mpconfig.h b/py/mpconfig.h index e84d258a1220a..64138a9ea7acc 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -1629,6 +1629,11 @@ typedef double mp_float_t; #define MICROPY_PY_THREAD_GIL_VM_DIVISOR (32) #endif +// Is a recursive mutex type in use? +#ifndef MICROPY_PY_THREAD_RECURSIVE_MUTEX +#define MICROPY_PY_THREAD_RECURSIVE_MUTEX (MICROPY_PY_THREAD && !MICROPY_PY_THREAD_GIL) +#endif + // Extended modules #ifndef MICROPY_PY_ASYNCIO diff --git a/py/mpstate.h b/py/mpstate.h index 51f290b55b5ed..138c5617300e8 100644 --- a/py/mpstate.h +++ b/py/mpstate.h @@ -145,7 +145,7 @@ typedef struct _mp_state_mem_t { #if MICROPY_PY_THREAD && !MICROPY_PY_THREAD_GIL // This is a global mutex used to make the GC thread-safe. - mp_thread_mutex_t gc_mutex; + mp_thread_recursive_mutex_t gc_mutex; #endif } mp_state_mem_t; diff --git a/py/mpthread.h b/py/mpthread.h index f335cc02911fc..795f230bb4a0c 100644 --- a/py/mpthread.h +++ b/py/mpthread.h @@ -48,6 +48,12 @@ void mp_thread_mutex_init(mp_thread_mutex_t *mutex); int mp_thread_mutex_lock(mp_thread_mutex_t *mutex, int wait); void mp_thread_mutex_unlock(mp_thread_mutex_t *mutex); +#if MICROPY_PY_THREAD_RECURSIVE_MUTEX +void mp_thread_recursive_mutex_init(mp_thread_recursive_mutex_t *mutex); +int mp_thread_recursive_mutex_lock(mp_thread_recursive_mutex_t *mutex, int wait); +void mp_thread_recursive_mutex_unlock(mp_thread_recursive_mutex_t *mutex); +#endif + #endif // MICROPY_PY_THREAD #if MICROPY_PY_THREAD && MICROPY_PY_THREAD_GIL From 3bfedd0f4a9e8b11ff97851615ec4761663b0e94 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 10 Dec 2024 14:51:10 +1100 Subject: [PATCH 024/110] rp2: Migrate to the new mp_thread_recursive_mutex_t. Necessary for GC support, also refactored pendsv usage. This work was funded through GitHub Sponsors. Signed-off-by: Angus Gratton --- ports/rp2/mpthreadport.c | 4 ++-- ports/rp2/mpthreadport.h | 20 +++++++++++++++++++- ports/rp2/pendsv.c | 14 +++++++------- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/ports/rp2/mpthreadport.c b/ports/rp2/mpthreadport.c index 0d3b343cab067..8fd5e5d790277 100644 --- a/ports/rp2/mpthreadport.c +++ b/ports/rp2/mpthreadport.c @@ -84,10 +84,10 @@ void mp_thread_deinit(void) { assert(get_core_num() == 0); // Must ensure that core1 is not currently holding the GC lock, otherwise // it will be terminated while holding the lock. - mp_thread_mutex_lock(&MP_STATE_MEM(gc_mutex), 1); + mp_thread_recursive_mutex_lock(&MP_STATE_MEM(gc_mutex), 1); multicore_reset_core1(); core1_entry = NULL; - mp_thread_mutex_unlock(&MP_STATE_MEM(gc_mutex)); + mp_thread_recursive_mutex_unlock(&MP_STATE_MEM(gc_mutex)); } void mp_thread_gc_others(void) { diff --git a/ports/rp2/mpthreadport.h b/ports/rp2/mpthreadport.h index 67a0da0e9373a..f2f2e17bb079d 100644 --- a/ports/rp2/mpthreadport.h +++ b/ports/rp2/mpthreadport.h @@ -26,9 +26,10 @@ #ifndef MICROPY_INCLUDED_RP2_MPTHREADPORT_H #define MICROPY_INCLUDED_RP2_MPTHREADPORT_H -#include "pico/mutex.h" +#include "mutex_extra.h" typedef struct mutex mp_thread_mutex_t; +typedef recursive_mutex_nowait_t mp_thread_recursive_mutex_t; extern void *core_state[2]; @@ -65,4 +66,21 @@ static inline void mp_thread_mutex_unlock(mp_thread_mutex_t *m) { mutex_exit(m); } +static inline void mp_thread_recursive_mutex_init(mp_thread_recursive_mutex_t *m) { + recursive_mutex_nowait_init(m); +} + +static inline int mp_thread_recursive_mutex_lock(mp_thread_recursive_mutex_t *m, int wait) { + if (wait) { + recursive_mutex_nowait_enter_blocking(m); + return 1; + } else { + return recursive_mutex_nowait_try_enter(m, NULL); + } +} + +static inline void mp_thread_recursive_mutex_unlock(mp_thread_recursive_mutex_t *m) { + recursive_mutex_nowait_exit(m); +} + #endif // MICROPY_INCLUDED_RP2_MPTHREADPORT_H diff --git a/ports/rp2/pendsv.c b/ports/rp2/pendsv.c index 905a5aa162ec5..4ba1e81604b0f 100644 --- a/ports/rp2/pendsv.c +++ b/ports/rp2/pendsv.c @@ -26,7 +26,7 @@ #include #include "py/mpconfig.h" -#include "mutex_extra.h" +#include "py/mpthread.h" #include "pendsv.h" #if PICO_RP2040 @@ -47,21 +47,21 @@ void PendSV_Handler(void); // Using the nowait variant here as softtimer updates PendSV from the loop of mp_wfe_or_timeout(), // where we don't want the CPU event bit to be set. -static recursive_mutex_nowait_t pendsv_mutex; +static mp_thread_recursive_mutex_t pendsv_mutex; void pendsv_init(void) { - recursive_mutex_nowait_init(&pendsv_mutex); + mp_thread_recursive_mutex_init(&pendsv_mutex); } void pendsv_suspend(void) { // Recursive Mutex here as either core may call pendsv_suspend() and expect // both mutual exclusion (other core can't enter pendsv_suspend() at the // same time), and that no PendSV handler will run. - recursive_mutex_nowait_enter_blocking(&pendsv_mutex); + mp_thread_recursive_mutex_lock(&pendsv_mutex, 1); } void pendsv_resume(void) { - recursive_mutex_nowait_exit(&pendsv_mutex); + mp_thread_recursive_mutex_unlock(&pendsv_mutex); // Run pendsv if needed. Find an entry with a dispatch and call pendsv dispatch // with it. If pendsv runs it will service all slots. @@ -97,7 +97,7 @@ void pendsv_schedule_dispatch(size_t slot, pendsv_dispatch_t f) { // PendSV interrupt handler to perform background processing. void PendSV_Handler(void) { - if (!recursive_mutex_nowait_try_enter(&pendsv_mutex, NULL)) { + if (!mp_thread_recursive_mutex_lock(&pendsv_mutex, 0)) { // Failure here means core 1 holds pendsv_mutex. ISR will // run again after core 1 calls pendsv_resume(). return; @@ -117,5 +117,5 @@ void PendSV_Handler(void) { } } - recursive_mutex_nowait_exit(&pendsv_mutex); + mp_thread_recursive_mutex_unlock(&pendsv_mutex); } From fd0e529a476650a070046a38b7ba7e253b65a77a Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Thu, 9 Jan 2025 14:38:31 +1100 Subject: [PATCH 025/110] unix: Add recursive mutex support. Allows refactoring the existing thread_mutex atomic section support to use the new recursive mutex type. This work was funded through GitHub Sponsors. Signed-off-by: Angus Gratton --- ports/unix/mpthreadport.c | 31 ++++++++++++++++++++++++------- ports/unix/mpthreadport.h | 1 + 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/ports/unix/mpthreadport.c b/ports/unix/mpthreadport.c index 16ac4da8bf56f..5172645bc147a 100644 --- a/ports/unix/mpthreadport.c +++ b/ports/unix/mpthreadport.c @@ -65,7 +65,7 @@ static pthread_key_t tls_key; // The mutex is used for any code in this port that needs to be thread safe. // Specifically for thread management, access to the linked list is one example. // But also, e.g. scheduler state. -static pthread_mutex_t thread_mutex; +static mp_thread_recursive_mutex_t thread_mutex; static mp_thread_t *thread; // this is used to synchronise the signal handler of the thread @@ -78,11 +78,11 @@ static sem_t thread_signal_done; #endif void mp_thread_unix_begin_atomic_section(void) { - pthread_mutex_lock(&thread_mutex); + mp_thread_recursive_mutex_lock(&thread_mutex, true); } void mp_thread_unix_end_atomic_section(void) { - pthread_mutex_unlock(&thread_mutex); + mp_thread_recursive_mutex_unlock(&thread_mutex); } // this signal handler is used to scan the regs and stack of a thread @@ -113,10 +113,7 @@ void mp_thread_init(void) { // Needs to be a recursive mutex to emulate the behavior of // BEGIN_ATOMIC_SECTION on bare metal. - pthread_mutexattr_t thread_mutex_attr; - pthread_mutexattr_init(&thread_mutex_attr); - pthread_mutexattr_settype(&thread_mutex_attr, PTHREAD_MUTEX_RECURSIVE); - pthread_mutex_init(&thread_mutex, &thread_mutex_attr); + mp_thread_recursive_mutex_init(&thread_mutex); // create first entry in linked list of all threads thread = malloc(sizeof(mp_thread_t)); @@ -321,6 +318,26 @@ void mp_thread_mutex_unlock(mp_thread_mutex_t *mutex) { // TODO check return value } +#if MICROPY_PY_THREAD_RECURSIVE_MUTEX + +void mp_thread_recursive_mutex_init(mp_thread_recursive_mutex_t *mutex) { + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(mutex, &attr); + pthread_mutexattr_destroy(&attr); +} + +int mp_thread_recursive_mutex_lock(mp_thread_recursive_mutex_t *mutex, int wait) { + return mp_thread_mutex_lock(mutex, wait); +} + +void mp_thread_recursive_mutex_unlock(mp_thread_recursive_mutex_t *mutex) { + mp_thread_mutex_unlock(mutex); +} + +#endif // MICROPY_PY_THREAD_RECURSIVE_MUTEX + #endif // MICROPY_PY_THREAD // this is used even when MICROPY_PY_THREAD is disabled diff --git a/ports/unix/mpthreadport.h b/ports/unix/mpthreadport.h index b365f200edf97..a38223e720b45 100644 --- a/ports/unix/mpthreadport.h +++ b/ports/unix/mpthreadport.h @@ -28,6 +28,7 @@ #include typedef pthread_mutex_t mp_thread_mutex_t; +typedef pthread_mutex_t mp_thread_recursive_mutex_t; void mp_thread_init(void); void mp_thread_deinit(void); From 990f50fbb829e41ce275d942c056bcd3f4b857df Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Wed, 29 Jan 2025 10:10:10 +1100 Subject: [PATCH 026/110] py/gc: Reorder static functions for clarity. - Renamed gc_sweep to gc_sweep_free_blocks. - Call gc_sweep_run_finalisers from top level. - Reordered the gc static functions to be in approximate runtime sequence (with forward declarations) rather than in declaration order. This work was funded through GitHub Sponsors. Signed-off-by: Angus Gratton --- py/gc.c | 175 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 93 insertions(+), 82 deletions(-) diff --git a/py/gc.c b/py/gc.c index 6a3f48ac3c0ce..eda63187b20a3 100644 --- a/py/gc.c +++ b/py/gc.c @@ -123,6 +123,18 @@ #define GC_EXIT() #endif +// Static functions for individual steps of the GC mark/sweep sequence +static void gc_collect_start_common(void); +static void *gc_get_ptr(void **ptrs, int i); +#if MICROPY_GC_SPLIT_HEAP +static void gc_mark_subtree(mp_state_mem_area_t *area, size_t block); +#else +static void gc_mark_subtree(size_t block); +#endif +static void gc_deal_with_stack_overflow(void); +static void gc_sweep_run_finalisers(void); +static void gc_sweep_free_blocks(void); + // TODO waste less memory; currently requires that all entries in alloc_table have a corresponding block in pool static void gc_setup_area(mp_state_mem_area_t *area, void *start, void *end) { // calculate parameters for GC (T=total, A=alloc table, F=finaliser table, P=pool; all in bytes): @@ -379,6 +391,64 @@ static inline mp_state_mem_area_t *gc_get_ptr_area(const void *ptr) { #endif #endif +void gc_collect_start(void) { + gc_collect_start_common(); + #if MICROPY_GC_ALLOC_THRESHOLD + MP_STATE_MEM(gc_alloc_amount) = 0; + #endif + + // Trace root pointers. This relies on the root pointers being organised + // correctly in the mp_state_ctx structure. We scan nlr_top, dict_locals, + // dict_globals, then the root pointer section of mp_state_vm. + void **ptrs = (void **)(void *)&mp_state_ctx; + size_t root_start = offsetof(mp_state_ctx_t, thread.dict_locals); + size_t root_end = offsetof(mp_state_ctx_t, vm.qstr_last_chunk); + gc_collect_root(ptrs + root_start / sizeof(void *), (root_end - root_start) / sizeof(void *)); + + #if MICROPY_ENABLE_PYSTACK + // Trace root pointers from the Python stack. + ptrs = (void **)(void *)MP_STATE_THREAD(pystack_start); + gc_collect_root(ptrs, (MP_STATE_THREAD(pystack_cur) - MP_STATE_THREAD(pystack_start)) / sizeof(void *)); + #endif +} + +static void gc_collect_start_common(void) { + GC_ENTER(); + assert((MP_STATE_THREAD(gc_lock_depth) & GC_COLLECT_FLAG) == 0); + MP_STATE_THREAD(gc_lock_depth) |= GC_COLLECT_FLAG; + MP_STATE_MEM(gc_stack_overflow) = 0; +} + +void gc_collect_root(void **ptrs, size_t len) { + #if !MICROPY_GC_SPLIT_HEAP + mp_state_mem_area_t *area = &MP_STATE_MEM(area); + #endif + for (size_t i = 0; i < len; i++) { + MICROPY_GC_HOOK_LOOP(i); + void *ptr = gc_get_ptr(ptrs, i); + #if MICROPY_GC_SPLIT_HEAP + mp_state_mem_area_t *area = gc_get_ptr_area(ptr); + if (!area) { + continue; + } + #else + if (!VERIFY_PTR(ptr)) { + continue; + } + #endif + size_t block = BLOCK_FROM_PTR(area, ptr); + if (ATB_GET_KIND(area, block) == AT_HEAD) { + // An unmarked head: mark it, and mark all its children + ATB_HEAD_TO_MARK(area, block); + #if MICROPY_GC_SPLIT_HEAP + gc_mark_subtree(area, block); + #else + gc_mark_subtree(block); + #endif + } + } +} + // Take the given block as the topmost block on the stack. Check all it's // children: mark the unmarked child blocks and put those newly marked // blocks on the stack. When all children have been checked, pop off the @@ -457,6 +527,25 @@ static void gc_mark_subtree(size_t block) } } +void gc_sweep_all(void) { + gc_collect_start_common(); + gc_collect_end(); +} + +void gc_collect_end(void) { + gc_deal_with_stack_overflow(); + gc_sweep_run_finalisers(); + gc_sweep_free_blocks(); + #if MICROPY_GC_SPLIT_HEAP + MP_STATE_MEM(gc_last_free_area) = &MP_STATE_MEM(area); + #endif + for (mp_state_mem_area_t *area = &MP_STATE_MEM(area); area != NULL; area = NEXT_AREA(area)) { + area->gc_last_free_atb_index = 0; + } + MP_STATE_THREAD(gc_lock_depth) &= ~GC_COLLECT_FLAG; + GC_EXIT(); +} + static void gc_deal_with_stack_overflow(void) { while (MP_STATE_MEM(gc_stack_overflow)) { MP_STATE_MEM(gc_stack_overflow) = 0; @@ -520,18 +609,16 @@ static void gc_sweep_run_finalisers(void) { #endif // MICROPY_ENABLE_FINALISER } -static void gc_sweep(void) { +// Free unmarked heads and their tails +static void gc_sweep_free_blocks(void) { #if MICROPY_PY_GC_COLLECT_RETVAL MP_STATE_MEM(gc_collected) = 0; #endif - // free unmarked heads and their tails int free_tail = 0; #if MICROPY_GC_SPLIT_HEAP_AUTO mp_state_mem_area_t *prev_area = NULL; #endif - gc_sweep_run_finalisers(); - for (mp_state_mem_area_t *area = &MP_STATE_MEM(area); area != NULL; area = NEXT_AREA(area)) { size_t last_used_block = 0; assert(area->gc_last_used_block <= area->gc_alloc_table_byte_len * BLOCKS_PER_ATB); @@ -541,7 +628,7 @@ static void gc_sweep(void) { switch (ATB_GET_KIND(area, block)) { case AT_HEAD: free_tail = 1; - DEBUG_printf("gc_sweep(%p)\n", (void *)PTR_FROM_BLOCK(area, block)); + DEBUG_printf("gc_sweep_free_blocks(%p)\n", (void *)PTR_FROM_BLOCK(area, block)); #if MICROPY_PY_GC_COLLECT_RETVAL MP_STATE_MEM(gc_collected)++; #endif @@ -572,7 +659,7 @@ static void gc_sweep(void) { #if MICROPY_GC_SPLIT_HEAP_AUTO // Free any empty area, aside from the first one if (last_used_block == 0 && prev_area != NULL) { - DEBUG_printf("gc_sweep free empty area %p\n", area); + DEBUG_printf("gc_sweep_free_blocks free empty area %p\n", area); NEXT_AREA(prev_area) = NEXT_AREA(area); MP_PLAT_FREE_HEAP(area); area = prev_area; @@ -582,34 +669,6 @@ static void gc_sweep(void) { } } -static void gc_collect_start_common(void) { - GC_ENTER(); - assert((MP_STATE_THREAD(gc_lock_depth) & GC_COLLECT_FLAG) == 0); - MP_STATE_THREAD(gc_lock_depth) |= GC_COLLECT_FLAG; - MP_STATE_MEM(gc_stack_overflow) = 0; -} - -void gc_collect_start(void) { - gc_collect_start_common(); - #if MICROPY_GC_ALLOC_THRESHOLD - MP_STATE_MEM(gc_alloc_amount) = 0; - #endif - - // Trace root pointers. This relies on the root pointers being organised - // correctly in the mp_state_ctx structure. We scan nlr_top, dict_locals, - // dict_globals, then the root pointer section of mp_state_vm. - void **ptrs = (void **)(void *)&mp_state_ctx; - size_t root_start = offsetof(mp_state_ctx_t, thread.dict_locals); - size_t root_end = offsetof(mp_state_ctx_t, vm.qstr_last_chunk); - gc_collect_root(ptrs + root_start / sizeof(void *), (root_end - root_start) / sizeof(void *)); - - #if MICROPY_ENABLE_PYSTACK - // Trace root pointers from the Python stack. - ptrs = (void **)(void *)MP_STATE_THREAD(pystack_start); - gc_collect_root(ptrs, (MP_STATE_THREAD(pystack_cur) - MP_STATE_THREAD(pystack_start)) / sizeof(void *)); - #endif -} - // Address sanitizer needs to know that the access to ptrs[i] must always be // considered OK, even if it's a load from an address that would normally be // prohibited (due to being undefined, in a red zone, etc). @@ -625,54 +684,6 @@ static void *gc_get_ptr(void **ptrs, int i) { return ptrs[i]; } -void gc_collect_root(void **ptrs, size_t len) { - #if !MICROPY_GC_SPLIT_HEAP - mp_state_mem_area_t *area = &MP_STATE_MEM(area); - #endif - for (size_t i = 0; i < len; i++) { - MICROPY_GC_HOOK_LOOP(i); - void *ptr = gc_get_ptr(ptrs, i); - #if MICROPY_GC_SPLIT_HEAP - mp_state_mem_area_t *area = gc_get_ptr_area(ptr); - if (!area) { - continue; - } - #else - if (!VERIFY_PTR(ptr)) { - continue; - } - #endif - size_t block = BLOCK_FROM_PTR(area, ptr); - if (ATB_GET_KIND(area, block) == AT_HEAD) { - // An unmarked head: mark it, and mark all its children - ATB_HEAD_TO_MARK(area, block); - #if MICROPY_GC_SPLIT_HEAP - gc_mark_subtree(area, block); - #else - gc_mark_subtree(block); - #endif - } - } -} - -void gc_collect_end(void) { - gc_deal_with_stack_overflow(); - gc_sweep(); - #if MICROPY_GC_SPLIT_HEAP - MP_STATE_MEM(gc_last_free_area) = &MP_STATE_MEM(area); - #endif - for (mp_state_mem_area_t *area = &MP_STATE_MEM(area); area != NULL; area = NEXT_AREA(area)) { - area->gc_last_free_atb_index = 0; - } - MP_STATE_THREAD(gc_lock_depth) &= ~GC_COLLECT_FLAG; - GC_EXIT(); -} - -void gc_sweep_all(void) { - gc_collect_start_common(); - gc_collect_end(); -} - void gc_info(gc_info_t *info) { GC_ENTER(); info->total = 0; From 112f65776588ef0cc3307f8bef208a480037d7d0 Mon Sep 17 00:00:00 2001 From: "Kwabena W. Agyeman" Date: Thu, 23 Jan 2025 23:17:00 -0800 Subject: [PATCH 027/110] stm32/eth: Make ETH DMA buffer attributes configurable. Signed-off-by: Kwabena W. Agyeman --- ports/stm32/eth.c | 2 +- ports/stm32/mpconfigboard_common.h | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ports/stm32/eth.c b/ports/stm32/eth.c index fd46bde23cb3f..9f655306816d9 100644 --- a/ports/stm32/eth.c +++ b/ports/stm32/eth.c @@ -121,7 +121,7 @@ typedef struct _eth_t { int16_t (*phy_get_link_status)(uint32_t phy_addr); } eth_t; -static eth_dma_t eth_dma __attribute__((aligned(16384))); +static eth_dma_t eth_dma MICROPY_HW_ETH_DMA_ATTRIBUTE; eth_t eth_instance; diff --git a/ports/stm32/mpconfigboard_common.h b/ports/stm32/mpconfigboard_common.h index e1c9c159ecf6f..4ce0a75b87e2b 100644 --- a/ports/stm32/mpconfigboard_common.h +++ b/ports/stm32/mpconfigboard_common.h @@ -669,3 +669,7 @@ #endif #define MICROPY_HW_USES_BOOTLOADER (MICROPY_HW_VTOR != 0x08000000) + +#ifndef MICROPY_HW_ETH_DMA_ATTRIBUTE +#define MICROPY_HW_ETH_DMA_ATTRIBUTE __attribute__((aligned(16384))); +#endif From 55ae597bb6044c392e9264f1af09ddcc19901f44 Mon Sep 17 00:00:00 2001 From: Carl Pottle Date: Sun, 29 Dec 2024 15:20:37 -0800 Subject: [PATCH 028/110] rp2/modmachine: Make lightsleep preserve SLEEP_EN0 and SLEEP_EN1. The problem was introduced in d1423ef7a23793de3777e84d985f9902241e788e, calling `machine.lightsleep()` overwrites RP2xxx registers `SLEEP_EN0` and `SLEEP_EN1` with their power on default values. Prior to that commit the register values were saved on entry to lightsleep and restored before returning. These changes restores the earlier behavior. Fixes issue #16502. Signed-off-by: Carl Pottle --- ports/rp2/modmachine.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ports/rp2/modmachine.c b/ports/rp2/modmachine.c index 3229aed277b35..954ea216497fe 100644 --- a/ports/rp2/modmachine.c +++ b/ports/rp2/modmachine.c @@ -196,6 +196,8 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) { #endif xosc_dormant(); } else { + uint32_t save_sleep_en0 = clocks_hw->sleep_en0; + uint32_t save_sleep_en1 = clocks_hw->sleep_en1; bool timer3_enabled = irq_is_enabled(3); const uint32_t alarm_num = 3; @@ -251,8 +253,8 @@ static void mp_machine_lightsleep(size_t n_args, const mp_obj_t *args) { if (!timer3_enabled) { irq_set_enabled(irq_num, false); } - clocks_hw->sleep_en0 |= ~(0u); - clocks_hw->sleep_en1 |= ~(0u); + clocks_hw->sleep_en0 = save_sleep_en0; + clocks_hw->sleep_en1 = save_sleep_en1; } // Enable ROSC. From 81ab49a60759376271ad1e1fbf05bae0ffe9e7b1 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 23 Jan 2025 16:37:33 +1100 Subject: [PATCH 029/110] tests/ports/rp2: Add test for SLEEP_ENx registers over lightsleep. Signed-off-by: Damien George --- tests/ports/rp2/rp2_lightsleep_regs.py | 58 ++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 tests/ports/rp2/rp2_lightsleep_regs.py diff --git a/tests/ports/rp2/rp2_lightsleep_regs.py b/tests/ports/rp2/rp2_lightsleep_regs.py new file mode 100644 index 0000000000000..4a833d0a9c339 --- /dev/null +++ b/tests/ports/rp2/rp2_lightsleep_regs.py @@ -0,0 +1,58 @@ +# Test that SLEEP_ENx registers are preserved over a call to machine.lightsleep(). + +import sys +from machine import mem32, lightsleep +import unittest + +is_rp2350 = "RP2350" in sys.implementation._machine + +if is_rp2350: + CLOCK_BASE = 0x40010000 + SLEEP_EN0 = CLOCK_BASE + 0xB4 + SLEEP_EN1 = CLOCK_BASE + 0xB8 + TO_DISABLE_EN0 = 1 << 30 # SHA256 + TO_DISABLE_EN1 = 1 << 4 # SRAM0 +else: + CLOCK_BASE = 0x40008000 + SLEEP_EN0 = CLOCK_BASE + 0xA8 + SLEEP_EN1 = CLOCK_BASE + 0xAC + TO_DISABLE_EN0 = 1 << 28 # SRAM0 + TO_DISABLE_EN1 = 1 << 0 # SRAM4 + + +class Test(unittest.TestCase): + def setUp(self): + self.orig_sleep_en0 = mem32[SLEEP_EN0] + self.orig_sleep_en1 = mem32[SLEEP_EN1] + + def tearDown(self): + mem32[SLEEP_EN0] = self.orig_sleep_en0 + mem32[SLEEP_EN1] = self.orig_sleep_en1 + + def test_sleep_en_regs(self): + print() + + # Disable some bits so the registers aren't just 0xffff. + mem32[SLEEP_EN0] &= ~TO_DISABLE_EN0 + mem32[SLEEP_EN1] &= ~TO_DISABLE_EN1 + + # Get the registers before the lightsleep. + sleep_en0_before = mem32[SLEEP_EN0] & 0xFFFFFFFF + sleep_en1_before = mem32[SLEEP_EN1] & 0xFFFFFFFF + print(hex(sleep_en0_before), hex(sleep_en1_before)) + + # Do a lightsleep. + lightsleep(100) + + # Get the registers after a lightsleep. + sleep_en0_after = mem32[SLEEP_EN0] & 0xFFFFFFFF + sleep_en1_after = mem32[SLEEP_EN1] & 0xFFFFFFFF + print(hex(sleep_en0_after), hex(sleep_en1_after)) + + # Check the registers have not changed. + self.assertEqual(sleep_en0_before, sleep_en0_after) + self.assertEqual(sleep_en1_before, sleep_en1_after) + + +if __name__ == "__main__": + unittest.main() From 3699cf5f38736b670a26786af4025584dcb5aef2 Mon Sep 17 00:00:00 2001 From: Mike Bell Date: Tue, 21 Jan 2025 21:35:08 +0000 Subject: [PATCH 030/110] rp2/rp2_flash: Workaround multicore lockout not being reset. With regression test. See upstream bug https://github.com/raspberrypi/pico-sdk/issues/2201 Tested-by: Angus Gratton Signed-off-by: Mike Bell --- ports/rp2/rp2_flash.c | 14 ++++++++++++-- tests/ports/rp2/rp2_thread_reset_part1.py | 18 ++++++++++++++++++ tests/ports/rp2/rp2_thread_reset_part1.py.exp | 1 + tests/ports/rp2/rp2_thread_reset_part2.py | 12 ++++++++++++ tests/ports/rp2/rp2_thread_reset_part2.py.exp | 3 +++ 5 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 tests/ports/rp2/rp2_thread_reset_part1.py create mode 100644 tests/ports/rp2/rp2_thread_reset_part1.py.exp create mode 100644 tests/ports/rp2/rp2_thread_reset_part2.py create mode 100644 tests/ports/rp2/rp2_thread_reset_part2.py.exp diff --git a/ports/rp2/rp2_flash.c b/ports/rp2/rp2_flash.c index c1acb54e75748..a487fb1633b21 100644 --- a/ports/rp2/rp2_flash.c +++ b/ports/rp2/rp2_flash.c @@ -70,10 +70,20 @@ bi_decl(bi_block_device( BINARY_INFO_BLOCK_DEV_FLAG_WRITE | BINARY_INFO_BLOCK_DEV_FLAG_PT_UNKNOWN)); +// This is a workaround to pico-sdk #2201: https://github.com/raspberrypi/pico-sdk/issues/2201 +// which means the multicore_lockout_victim_is_initialized returns true even after core1 is reset. +static bool use_multicore_lockout(void) { + return multicore_lockout_victim_is_initialized(1 - get_core_num()) + #if MICROPY_PY_THREAD + && core1_entry != NULL + #endif + ; +} + // Flash erase and write must run with interrupts disabled and the other core suspended, // because the XIP bit gets disabled. static uint32_t begin_critical_flash_section(void) { - if (multicore_lockout_victim_is_initialized(1 - get_core_num())) { + if (use_multicore_lockout()) { multicore_lockout_start_blocking(); } return save_and_disable_interrupts(); @@ -81,7 +91,7 @@ static uint32_t begin_critical_flash_section(void) { static void end_critical_flash_section(uint32_t state) { restore_interrupts(state); - if (multicore_lockout_victim_is_initialized(1 - get_core_num())) { + if (use_multicore_lockout()) { multicore_lockout_end_blocking(); } } diff --git a/tests/ports/rp2/rp2_thread_reset_part1.py b/tests/ports/rp2/rp2_thread_reset_part1.py new file mode 100644 index 0000000000000..d43868113f5eb --- /dev/null +++ b/tests/ports/rp2/rp2_thread_reset_part1.py @@ -0,0 +1,18 @@ +# This is a regression test for https://github.com/micropython/micropython/issues/16619 +# it runs in two parts by necessity: +# +# - This "part1" creates a non-terminating thread. +# - The test runner issues a soft reset, which will terminate that thread. +# - "part2" is the actual test, which is whether flash access works correctly +# after the thread was terminated by soft reset. + +import _thread + + +def infinite(): + while True: + pass + + +_thread.start_new_thread(infinite, ()) +print("Part 1 complete...") diff --git a/tests/ports/rp2/rp2_thread_reset_part1.py.exp b/tests/ports/rp2/rp2_thread_reset_part1.py.exp new file mode 100644 index 0000000000000..48ea7efad97ae --- /dev/null +++ b/tests/ports/rp2/rp2_thread_reset_part1.py.exp @@ -0,0 +1 @@ +Part 1 complete... diff --git a/tests/ports/rp2/rp2_thread_reset_part2.py b/tests/ports/rp2/rp2_thread_reset_part2.py new file mode 100644 index 0000000000000..15f0eaab8f857 --- /dev/null +++ b/tests/ports/rp2/rp2_thread_reset_part2.py @@ -0,0 +1,12 @@ +# This is part2 of a two-part regression test, see part1 +# for details of what's expected. +import os + +FILENAME = "/rp2_thread_reset_test.txt" + +print("Starting") +with open(FILENAME, "w") as f: + f.write("test") +print("Written") +os.unlink(FILENAME) +print("Removed") diff --git a/tests/ports/rp2/rp2_thread_reset_part2.py.exp b/tests/ports/rp2/rp2_thread_reset_part2.py.exp new file mode 100644 index 0000000000000..d7581c7d2601e --- /dev/null +++ b/tests/ports/rp2/rp2_thread_reset_part2.py.exp @@ -0,0 +1,3 @@ +Starting +Written +Removed From 4bed77cc233a84846a0c47347b520ce5b816758b Mon Sep 17 00:00:00 2001 From: Glenn Moloney Date: Fri, 19 May 2023 17:51:52 +1000 Subject: [PATCH 031/110] esp8266/network_wlan: Make WLAN.config('channel') use wifi_get_channel. Prior to this fix, `WLAN.config('channel')` would return an incorrect channel for AP_IF if STA has connected to an external AP running on a different channel. The esp8266 now has the same behaviour as for esp32 per commit 98d1c50159fe9427d72ec358ba0219eaebb1d991. Fixes issue #11463. Signed-off-by: Glenn Moloney --- ports/esp8266/network_wlan.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ports/esp8266/network_wlan.c b/ports/esp8266/network_wlan.c index 44154ff6d8e54..2e43ebfb2b7b1 100644 --- a/ports/esp8266/network_wlan.c +++ b/ports/esp8266/network_wlan.c @@ -641,8 +641,7 @@ static mp_obj_t esp_config(size_t n_args, const mp_obj_t *args, mp_map_t *kwargs val = MP_OBJ_NEW_SMALL_INT(cfg.ap.authmode); break; case MP_QSTR_channel: - req_if = SOFTAP_IF; - val = MP_OBJ_NEW_SMALL_INT(cfg.ap.channel); + val = MP_OBJ_NEW_SMALL_INT(wifi_get_channel()); break; case MP_QSTR_hostname: case MP_QSTR_dhcp_hostname: { From b1e6c2b655209022fb1dd1e7b3271af010fcccb5 Mon Sep 17 00:00:00 2001 From: Glenn Moloney Date: Mon, 26 Jun 2023 18:05:37 +1000 Subject: [PATCH 032/110] esp8266/network_wlan: Make WLAN.config(channel=x) use wifi_set_channel. Also permits channel option to be used for STA_IF interface. This provides compatibility with esp32 code, especially for espnow users. Signed-off-by: Glenn Moloney --- ports/esp8266/network_wlan.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ports/esp8266/network_wlan.c b/ports/esp8266/network_wlan.c index 2e43ebfb2b7b1..7131b88bac9ab 100644 --- a/ports/esp8266/network_wlan.c +++ b/ports/esp8266/network_wlan.c @@ -565,8 +565,8 @@ static mp_obj_t esp_config(size_t n_args, const mp_obj_t *args, mp_map_t *kwargs break; } case MP_QSTR_channel: { - req_if = SOFTAP_IF; cfg.ap.channel = mp_obj_get_int(kwargs->table[i].value); + error_check(wifi_set_channel(cfg.ap.channel), "can't set channel"); break; } case MP_QSTR_hostname: From 921f19fc9d697488f4f86bf8db73d43b04b781a6 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 4 Feb 2025 15:14:51 +1100 Subject: [PATCH 033/110] tests/multi_wlan: Remove esp8266 port workaround. Not needed due to parent commit. Signed-off-by: Angus Gratton --- tests/multi_wlan/01_ap_sta.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/multi_wlan/01_ap_sta.py b/tests/multi_wlan/01_ap_sta.py index 1a8e80cd33aa3..368addf1393e2 100644 --- a/tests/multi_wlan/01_ap_sta.py +++ b/tests/multi_wlan/01_ap_sta.py @@ -96,15 +96,7 @@ def instance1(): print("STA connected") - # Print the current channel, if the port support this - try: - print("channel", sta.config("channel")) - except OSError as e: - if "AP" in str(e): - # ESP8266 only supports reading channel on the AP interface, so fake this result - print("channel", CHANNEL) - else: - raise + print("channel", sta.config("channel")) print("STA waiting for disconnect...") From 71e8b27b26f46f11f642381d4f6699ee1fc073f6 Mon Sep 17 00:00:00 2001 From: eggfly Date: Mon, 23 Dec 2024 19:40:06 +0800 Subject: [PATCH 034/110] esp32/README: Fix board in octal-SPIRAM example make command. Signed-off-by: eggfly --- ports/esp32/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ports/esp32/README.md b/ports/esp32/README.md index 8597c85ec3103..6ed7eddb8b80f 100644 --- a/ports/esp32/README.md +++ b/ports/esp32/README.md @@ -152,7 +152,7 @@ $ make BOARD=ESP32_GENERIC BOARD_VARIANT=OTA or to enable octal-SPIRAM support for the `ESP32_GENERIC_S3` board: ```bash -$ make BOARD=ESP32_GENERIC BOARD_VARIANT=SPIRAM_OCT +$ make BOARD=ESP32_GENERIC_S3 BOARD_VARIANT=SPIRAM_OCT ``` From b603fa38b25c5136d36cdbfff91b69f96efacd69 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Mon, 3 Feb 2025 16:03:22 +1100 Subject: [PATCH 035/110] py/mkrules.mk: Reset USER_C_MODULES when building mpy-cross dependency. When a port automatically compiles `mpy-cross`, if `USER_C_MODULES` is provided by the user on the command line then it is also applied to the `mpy-cross` build. That can lead to build errors if the path is relative and not found when building `mpy-cross`. Fix that by explicitly resetting `USER_C_MODULES` when invoking the `mpy-cross` build. Signed-off-by: Andrew Leech --- py/mkrules.cmake | 2 +- py/mkrules.mk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/py/mkrules.cmake b/py/mkrules.cmake index bfc56abfe80b7..3ee4c4c31aa57 100644 --- a/py/mkrules.cmake +++ b/py/mkrules.cmake @@ -211,7 +211,7 @@ if(MICROPY_FROZEN_MANIFEST) endif() add_custom_command( OUTPUT ${MICROPY_MPYCROSS_DEPENDENCY} - COMMAND ${MICROPY_MAKE_EXECUTABLE} -C ${MICROPY_DIR}/mpy-cross + COMMAND ${MICROPY_MAKE_EXECUTABLE} -C ${MICROPY_DIR}/mpy-cross USER_C_MODULES= ) endif() diff --git a/py/mkrules.mk b/py/mkrules.mk index 373bda8996f19..e9504ce39f839 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -176,7 +176,7 @@ $(HEADER_BUILD): ifneq ($(MICROPY_MPYCROSS_DEPENDENCY),) # to automatically build mpy-cross, if needed $(MICROPY_MPYCROSS_DEPENDENCY): - $(MAKE) -C "$(abspath $(dir $@)..)" + $(MAKE) -C "$(abspath $(dir $@)..)" USER_C_MODULES= endif ifneq ($(FROZEN_DIR),) From dfd1d69a72956e650af5cb071e4d9be61b5d322a Mon Sep 17 00:00:00 2001 From: Alessandro Gatti Date: Fri, 24 Jan 2025 16:26:55 +0100 Subject: [PATCH 036/110] tests/run-natmodtests.py: Autodetect the test target architecture. This commit lets the natmod tests runner to automatically detect the architecture of the test target. This allows to avoid to explicitly pass the architecture name to the runner in test scripts. However, the ability to manually specify a target was not removed but it was made optional. This way the user is able to override the architecture name if needed (like if one wants to test an armv6 MPY on an armv7 board). Signed-off-by: Alessandro Gatti --- ports/qemu/README.md | 4 +- ports/qemu/boards/VIRT_RV32/mpconfigboard.mk | 2 - tests/run-natmodtests.py | 58 ++++++++++++++++++-- 3 files changed, 55 insertions(+), 9 deletions(-) diff --git a/ports/qemu/README.md b/ports/qemu/README.md index 70edf97f58e37..c7d0dc1f4ea70 100644 --- a/ports/qemu/README.md +++ b/ports/qemu/README.md @@ -105,9 +105,7 @@ can also be tested with this command (this is currently supported only for the $ make test_natmod The same remarks about manually running the tests apply for native modules, but -`run-natmodtests.py` should be run instead of `run-tests.py`. In this case you -also have to explicitly pass the architecture you are running native modules to -`run-natmodtests.py` ("--arch rv32imc" for the `VIRT_RV32` board). +`run-natmodtests.py` should be run instead of `run-tests.py`. Extra make options ------------------ diff --git a/ports/qemu/boards/VIRT_RV32/mpconfigboard.mk b/ports/qemu/boards/VIRT_RV32/mpconfigboard.mk index dd926480052e1..ce12720928e32 100644 --- a/ports/qemu/boards/VIRT_RV32/mpconfigboard.mk +++ b/ports/qemu/boards/VIRT_RV32/mpconfigboard.mk @@ -9,5 +9,3 @@ LDSCRIPT = mcu/rv32/virt.ld SRC_BOARD_O += shared/runtime/gchelper_native.o shared/runtime/gchelper_rv32i.o MPY_CROSS_FLAGS += -march=rv32imc - -RUN_NATMODTESTS_ARGS = --arch rv32imc diff --git a/tests/run-natmodtests.py b/tests/run-natmodtests.py index 1fe44bec1615a..b858989daa434 100755 --- a/tests/run-natmodtests.py +++ b/tests/run-natmodtests.py @@ -28,6 +28,23 @@ "re": "re/re_$(ARCH).mpy", } +# Supported architectures for native mpy modules +AVAILABLE_ARCHS = ( + "x86", + "x64", + "armv6", + "armv6m", + "armv7m", + "armv7em", + "armv7emsp", + "armv7emdp", + "xtensa", + "xtensawin", + "rv32imc", +) + +ARCH_MAPPINGS = {"armv7em": "armv7m"} + # Code to allow a target MicroPython to import an .mpy from RAM injected_import_hook_code = """\ import sys, io, vfs @@ -96,14 +113,33 @@ def run_script(self, script): return b"", er -def run_tests(target_truth, target, args, stats): +def detect_architecture(target): + with open("./feature_check/target_info.py", "rb") as f: + target_info_data = f.read() + result_out, error = target.run_script(target_info_data) + if error is not None: + return None, None, error + info = result_out.split(b" ") + if len(info) < 2: + return None, None, "unexpected target info: {}".format(info) + platform = info[0].strip().decode() + arch = info[1].strip().decode() + if arch not in AVAILABLE_ARCHS: + if arch == "None": + return None, None, "the target does not support dynamic modules" + else: + return None, None, "{} is not a supported architecture".format(arch) + return platform, arch, None + + +def run_tests(target_truth, target, args, stats, resolved_arch): for test_file in args.files: # Find supported test test_file_basename = os.path.basename(test_file) for k, v in TEST_MAPPINGS.items(): if test_file_basename.startswith(k): test_module = k - test_mpy = v.replace("$(ARCH)", args.arch) + test_mpy = v.replace("$(ARCH)", resolved_arch) break else: print("---- {} - no matching mpy".format(test_file)) @@ -174,7 +210,7 @@ def main(): "-d", "--device", default="/dev/ttyACM0", help="the device for pyboard.py" ) cmd_parser.add_argument( - "-a", "--arch", default="x64", help="native architecture of the target" + "-a", "--arch", choices=AVAILABLE_ARCHS, help="override native architecture of the target" ) cmd_parser.add_argument("files", nargs="*", help="input test files") args = cmd_parser.parse_args() @@ -186,8 +222,22 @@ def main(): else: target = TargetSubprocess([MICROPYTHON]) + if hasattr(args, "arch") and args.arch is not None: + target_arch = args.arch + target_platform = None + else: + target_platform, target_arch, error = detect_architecture(target) + if error: + print("Cannot run tests: {}".format(error)) + sys.exit(1) + target_arch = ARCH_MAPPINGS.get(target_arch, target_arch) + + if target_platform: + print("platform={} ".format(target_platform), end="") + print("arch={}".format(target_arch)) + stats = {"total": 0, "pass": 0, "fail": 0, "skip": 0} - run_tests(target_truth, target, args, stats) + run_tests(target_truth, target, args, stats, target_arch) target.close() target_truth.close() From ca3090a33f06da57221827ad506d80cb1f826b99 Mon Sep 17 00:00:00 2001 From: Alessandro Gatti Date: Fri, 24 Jan 2025 11:18:39 +0100 Subject: [PATCH 037/110] qemu/Makefile: Fix shell interpolation for automated natmod tests. This commit fixes the command used to run natmod tests, as it relied on a string interpolation feature of the POSIX shell that was not working as expected inside a makefile. The interpolation was not performed from inside the makefile and the raw command string was sent to the operating system for execution. Now the command is run by using a different type of string substitution, which explicitly performs the interpolation using a POSIX shell for-loop. Signed-off-by: Alessandro Gatti --- ports/qemu/Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ports/qemu/Makefile b/ports/qemu/Makefile index d2550cf883b7d..befef5d520061 100644 --- a/ports/qemu/Makefile +++ b/ports/qemu/Makefile @@ -189,11 +189,14 @@ test_full: $(BUILD)/firmware.elf cd $(TOP)/tests && ./run-tests.py $(RUN_TESTS_FULL_ARGS) --via-mpy cd $(TOP)/tests && ./run-tests.py $(RUN_TESTS_FULL_ARGS) --via-mpy --emit native +# "btree" currently does not build for rv32imc (Picolibc TLS incompatibility). .PHONY: test_natmod test_natmod: $(BUILD)/firmware.elf $(eval DIRNAME=ports/$(notdir $(CURDIR))) - # "btree" cannot build against Picolibc right now. - cd $(TOP)/tests && ./run-natmodtests.py -p -d execpty:"$(QEMU_SYSTEM) $(QEMU_ARGS) -serial pty -kernel ../$(DIRNAME)/$<" $(RUN_NATMODTESTS_ARGS) extmod/{deflate,framebuf,heapq,random_basic,re}*.py + cd $(TOP)/tests && \ + for natmod in deflate framebuf heapq random_basic re; do \ + ./run-natmodtests.py -p -d execpty:"$(QEMU_SYSTEM) $(QEMU_ARGS) -serial pty -kernel ../$(DIRNAME)/$<" extmod/$$natmod*.py; \ + done $(BUILD)/firmware.elf: $(LDSCRIPT) $(OBJ) $(Q)$(CC) $(LDFLAGS) -o $@ $(OBJ) $(LIBS) From f594c6f66ec49e02ba06165d1f1db202d82575ec Mon Sep 17 00:00:00 2001 From: Alessandro Gatti Date: Fri, 24 Jan 2025 18:00:05 +0100 Subject: [PATCH 038/110] tools/ci.sh: Add natmod tests for QEMU/Arm. This commit adds the natmod tests for the MPS2_AN385 board running inside QEMU to the CI pipeline. Now natmod tests capabilities are equal between the Arm and RV32 platforms for the QEMU port. Signed-off-by: Alessandro Gatti --- ports/qemu/Makefile | 2 ++ tools/ci.sh | 25 ++++++++++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/ports/qemu/Makefile b/ports/qemu/Makefile index befef5d520061..e9e1e0f957e2b 100644 --- a/ports/qemu/Makefile +++ b/ports/qemu/Makefile @@ -40,6 +40,8 @@ endif include $(TOP)/py/py.mk include $(TOP)/extmod/extmod.mk +GIT_SUBMODULES += lib/berkeley-db-1.xx + CFLAGS += -DMICROPY_HEAP_SIZE=$(MICROPY_HEAP_SIZE) ################################################################################ diff --git a/tools/ci.sh b/tools/ci.sh index 374395abd2ca5..2c647012f144d 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -266,6 +266,7 @@ function ci_powerpc_build { # ports/qemu function ci_qemu_setup_arm { + ci_mpy_format_setup ci_gcc_arm_setup sudo apt-get update sudo apt-get install qemu-system @@ -287,6 +288,10 @@ function ci_qemu_build_arm { make ${MAKEOPTS} -C ports/qemu clean make ${MAKEOPTS} -C ports/qemu test_full make ${MAKEOPTS} -C ports/qemu BOARD=SABRELITE test_full + + # Test building and running native .mpy with armv7m architecture. + ci_native_mpy_modules_build armv7m + make ${MAKEOPTS} -C ports/qemu test_natmod } function ci_qemu_build_rv32 { @@ -480,20 +485,18 @@ function ci_native_mpy_modules_build { else arch=$1 fi - make -C examples/natmod/features1 ARCH=$arch + for natmod in features1 features3 features4 deflate framebuf heapq random re + do + make -C examples/natmod/$natmod ARCH=$arch + done + # btree requires thread local storage support on rv32imc. if [ $arch != rv32imc ]; then - # This requires soft-float support on rv32imc. - make -C examples/natmod/features2 ARCH=$arch - # This requires thread local storage support on rv32imc. make -C examples/natmod/btree ARCH=$arch fi - make -C examples/natmod/features3 ARCH=$arch - make -C examples/natmod/features4 ARCH=$arch - make -C examples/natmod/deflate ARCH=$arch - make -C examples/natmod/framebuf ARCH=$arch - make -C examples/natmod/heapq ARCH=$arch - make -C examples/natmod/random ARCH=$arch - make -C examples/natmod/re ARCH=$arch + # features2 requires soft-float on armv7m and rv32imc. + if [ $arch != rv32imc ] && [ $arch != armv7m ]; then + make -C examples/natmod/features2 ARCH=$arch + fi } function ci_native_mpy_modules_32bit_build { From e37d498cc0891b67ea6b782fc1e9b1b7dab3c740 Mon Sep 17 00:00:00 2001 From: Alessandro Gatti Date: Tue, 4 Feb 2025 00:02:49 +0100 Subject: [PATCH 039/110] py/emitnative: Mark condition code tables as const. This commit marks as const the condition code tables used when figuring out which opcode sequence must be emitted depending on the requested comparison type. Signed-off-by: Alessandro Gatti --- py/emitnative.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/py/emitnative.c b/py/emitnative.c index 82ee729d3d50f..dd0644793231c 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -2541,7 +2541,7 @@ static void emit_native_binary_op(emit_t *emit, mp_binary_op_t op) { #if N_X64 asm_x64_xor_r64_r64(emit->as, REG_RET, REG_RET); asm_x64_cmp_r64_with_r64(emit->as, reg_rhs, REG_ARG_2); - static byte ops[6 + 6] = { + static const byte ops[6 + 6] = { // unsigned ASM_X64_CC_JB, ASM_X64_CC_JA, @@ -2561,7 +2561,7 @@ static void emit_native_binary_op(emit_t *emit, mp_binary_op_t op) { #elif N_X86 asm_x86_xor_r32_r32(emit->as, REG_RET, REG_RET); asm_x86_cmp_r32_with_r32(emit->as, reg_rhs, REG_ARG_2); - static byte ops[6 + 6] = { + static const byte ops[6 + 6] = { // unsigned ASM_X86_CC_JB, ASM_X86_CC_JA, @@ -2581,7 +2581,7 @@ static void emit_native_binary_op(emit_t *emit, mp_binary_op_t op) { #elif N_THUMB asm_thumb_cmp_rlo_rlo(emit->as, REG_ARG_2, reg_rhs); if (asm_thumb_allow_armv7m(emit->as)) { - static uint16_t ops[6 + 6] = { + static const uint16_t ops[6 + 6] = { // unsigned ASM_THUMB_OP_ITE_CC, ASM_THUMB_OP_ITE_HI, @@ -2601,7 +2601,7 @@ static void emit_native_binary_op(emit_t *emit, mp_binary_op_t op) { asm_thumb_mov_rlo_i8(emit->as, REG_RET, 1); asm_thumb_mov_rlo_i8(emit->as, REG_RET, 0); } else { - static uint16_t ops[6 + 6] = { + static const uint16_t ops[6 + 6] = { // unsigned ASM_THUMB_CC_CC, ASM_THUMB_CC_HI, @@ -2624,7 +2624,7 @@ static void emit_native_binary_op(emit_t *emit, mp_binary_op_t op) { } #elif N_ARM asm_arm_cmp_reg_reg(emit->as, REG_ARG_2, reg_rhs); - static uint ccs[6 + 6] = { + static const uint ccs[6 + 6] = { // unsigned ASM_ARM_CC_CC, ASM_ARM_CC_HI, @@ -2642,7 +2642,7 @@ static void emit_native_binary_op(emit_t *emit, mp_binary_op_t op) { }; asm_arm_setcc_reg(emit->as, REG_RET, ccs[op_idx]); #elif N_XTENSA || N_XTENSAWIN - static uint8_t ccs[6 + 6] = { + static const uint8_t ccs[6 + 6] = { // unsigned ASM_XTENSA_CC_LTU, 0x80 | ASM_XTENSA_CC_LTU, // for GTU we'll swap args From 44a7731669f1c8429bb61b790a7e8bf8a86b045d Mon Sep 17 00:00:00 2001 From: Alessandro Gatti Date: Tue, 4 Feb 2025 00:05:44 +0100 Subject: [PATCH 040/110] py/emitnative: Load and store words just once for Viper code. This commit fixes two Xtensa sequences in order to terminate early when loading and storing word values via an immediate index. This was meant to be part of 55ca3fd67512555707304c6b68b836eb89f09d1c but whilst it was part of the code being tested, it didn't end up in the commit. Signed-off-by: Alessandro Gatti --- py/emitnative.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/py/emitnative.c b/py/emitnative.c index dd0644793231c..1aab0a9eb7865 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -1609,6 +1609,7 @@ static void emit_native_load_subscr(emit_t *emit) { #elif N_XTENSA || N_XTENSAWIN if (index_value > 0 && index_value < 256) { asm_xtensa_l32i_optimised(emit->as, REG_RET, reg_base, index_value); + break; } #endif need_reg_single(emit, reg_index, 0); @@ -1887,6 +1888,7 @@ static void emit_native_store_subscr(emit_t *emit) { #elif N_XTENSA || N_XTENSAWIN if (index_value > 0 && index_value < 256) { asm_xtensa_s32i_optimised(emit->as, REG_RET, reg_base, index_value); + break; } #elif N_ARM ASM_MOV_REG_IMM(emit->as, reg_index, index_value); From 304467518fbe9fc679dbba087fba7931c51655fb Mon Sep 17 00:00:00 2001 From: robert-hh Date: Thu, 23 Jan 2025 13:34:47 +0100 Subject: [PATCH 041/110] samd/boards: Add generic SAMD21x18 board definitions. The definition uses the internal oscillator for clock and only internal flash for the file system. It works at SAMD21G18 and SAMD21E18 devices as well, only that fewer pins are accessible. Tested with a SAMD21E18, SAM21G18 and SAMD21J18 board. Signed-off-by: robert-hh --- ports/samd/boards/SAMD_GENERIC_D21X18/board.json | 16 ++++++++++++++++ ports/samd/boards/SAMD_GENERIC_D21X18/board.md | 4 ++++ .../boards/SAMD_GENERIC_D21X18/mpconfigboard.h | 4 ++++ .../boards/SAMD_GENERIC_D21X18/mpconfigboard.mk | 4 ++++ ports/samd/boards/SAMD_GENERIC_D21X18/pins.csv | 10 ++++++++++ 5 files changed, 38 insertions(+) create mode 100644 ports/samd/boards/SAMD_GENERIC_D21X18/board.json create mode 100644 ports/samd/boards/SAMD_GENERIC_D21X18/board.md create mode 100644 ports/samd/boards/SAMD_GENERIC_D21X18/mpconfigboard.h create mode 100644 ports/samd/boards/SAMD_GENERIC_D21X18/mpconfigboard.mk create mode 100644 ports/samd/boards/SAMD_GENERIC_D21X18/pins.csv diff --git a/ports/samd/boards/SAMD_GENERIC_D21X18/board.json b/ports/samd/boards/SAMD_GENERIC_D21X18/board.json new file mode 100644 index 0000000000000..a14730bd1bf49 --- /dev/null +++ b/ports/samd/boards/SAMD_GENERIC_D21X18/board.json @@ -0,0 +1,16 @@ +{ + "deploy": [ + "../deploy.md" + ], + "docs": "", + "features": [ + "USB" + ], + "images": [ + "generic_board.jpg" + ], + "mcu": "samd21", + "vendor": "Microchip", + "product": "Generic SAMD21J18", + "thumbnail": "" +} diff --git a/ports/samd/boards/SAMD_GENERIC_D21X18/board.md b/ports/samd/boards/SAMD_GENERIC_D21X18/board.md new file mode 100644 index 0000000000000..a10b08f2f1cd7 --- /dev/null +++ b/ports/samd/boards/SAMD_GENERIC_D21X18/board.md @@ -0,0 +1,4 @@ +The following firmware should work on most boards with a SAMD21E18, +SAMD21G18 and SAMD21J18 MCU. It uses only the features built into +the MCU. Additional devices at the board like external flash +are not supported. diff --git a/ports/samd/boards/SAMD_GENERIC_D21X18/mpconfigboard.h b/ports/samd/boards/SAMD_GENERIC_D21X18/mpconfigboard.h new file mode 100644 index 0000000000000..945975ab2384d --- /dev/null +++ b/ports/samd/boards/SAMD_GENERIC_D21X18/mpconfigboard.h @@ -0,0 +1,4 @@ +#define MICROPY_HW_BOARD_NAME "Generic SAMD21J18" +#define MICROPY_HW_MCU_NAME "SAMD21J18A" + +#define MICROPY_HW_DFLL_USB_SYNC (1) diff --git a/ports/samd/boards/SAMD_GENERIC_D21X18/mpconfigboard.mk b/ports/samd/boards/SAMD_GENERIC_D21X18/mpconfigboard.mk new file mode 100644 index 0000000000000..f95c6549381ba --- /dev/null +++ b/ports/samd/boards/SAMD_GENERIC_D21X18/mpconfigboard.mk @@ -0,0 +1,4 @@ +MCU_SERIES = SAMD21 +CMSIS_MCU = SAMD21J18A +LD_FILES = boards/samd21x18a.ld sections.ld +TEXT0 = 0x2000 diff --git a/ports/samd/boards/SAMD_GENERIC_D21X18/pins.csv b/ports/samd/boards/SAMD_GENERIC_D21X18/pins.csv new file mode 100644 index 0000000000000..d35bc9d8ea4a9 --- /dev/null +++ b/ports/samd/boards/SAMD_GENERIC_D21X18/pins.csv @@ -0,0 +1,10 @@ +# The lines contain pairs of Pin name and Pin number. +# Pin names must be valid Python identifiers. +# Pin numbers have the form Pxnn, with x being A, B, C or D. +# Lines starting with # or empty lines are ignored. + +USB_DM,PA24 +USB_DP,PA25 + +SWCLK,PA30 +SWDIO,PA31 From 9ced693ade1a1aca1e76d1eb6f11094e8aa3bfc9 Mon Sep 17 00:00:00 2001 From: robert-hh Date: Thu, 23 Jan 2025 13:42:18 +0100 Subject: [PATCH 042/110] samd/boards: Add generic SAMD51x19 board definitions. The definition uses the internal oscillator for clock and only internal flash for the file system. It works at SAMD51G19 and SAMD51J19 devices as well, only that fewer pins are accessible. Tested with a SAMD51G19 and SAMD51J9 board. Signed-off-by: robert-hh --- .../samd/boards/SAMD_GENERIC_D51X19/board.json | 16 ++++++++++++++++ ports/samd/boards/SAMD_GENERIC_D51X19/board.md | 4 ++++ .../boards/SAMD_GENERIC_D51X19/mpconfigboard.h | 2 ++ .../SAMD_GENERIC_D51X19/mpconfigboard.mk | 11 +++++++++++ ports/samd/boards/SAMD_GENERIC_D51X19/pins.csv | 18 ++++++++++++++++++ 5 files changed, 51 insertions(+) create mode 100644 ports/samd/boards/SAMD_GENERIC_D51X19/board.json create mode 100644 ports/samd/boards/SAMD_GENERIC_D51X19/board.md create mode 100644 ports/samd/boards/SAMD_GENERIC_D51X19/mpconfigboard.h create mode 100644 ports/samd/boards/SAMD_GENERIC_D51X19/mpconfigboard.mk create mode 100644 ports/samd/boards/SAMD_GENERIC_D51X19/pins.csv diff --git a/ports/samd/boards/SAMD_GENERIC_D51X19/board.json b/ports/samd/boards/SAMD_GENERIC_D51X19/board.json new file mode 100644 index 0000000000000..21cb114bf65fb --- /dev/null +++ b/ports/samd/boards/SAMD_GENERIC_D51X19/board.json @@ -0,0 +1,16 @@ +{ + "deploy": [ + "../deploy.md" + ], + "docs": "", + "features": [ + "USB" + ], + "images": [ + "generic_board.jpg" + ], + "mcu": "samd51", + "vendor": "Microchip", + "product": "Generic SAMD51P19", + "thumbnail": "" +} diff --git a/ports/samd/boards/SAMD_GENERIC_D51X19/board.md b/ports/samd/boards/SAMD_GENERIC_D51X19/board.md new file mode 100644 index 0000000000000..b4694cc2857d5 --- /dev/null +++ b/ports/samd/boards/SAMD_GENERIC_D51X19/board.md @@ -0,0 +1,4 @@ +The following firmware should work on most boards with a SAMD51G19, +SAMD51J19 and SAMD51P19 MCU. It uses only the features built into +the MCU. Additional devices at the board like external flash +are not supported. diff --git a/ports/samd/boards/SAMD_GENERIC_D51X19/mpconfigboard.h b/ports/samd/boards/SAMD_GENERIC_D51X19/mpconfigboard.h new file mode 100644 index 0000000000000..cce157f9e0ffc --- /dev/null +++ b/ports/samd/boards/SAMD_GENERIC_D51X19/mpconfigboard.h @@ -0,0 +1,2 @@ +#define MICROPY_HW_BOARD_NAME "Generic SAMD51P19" +#define MICROPY_HW_MCU_NAME "SAMD51P19A" diff --git a/ports/samd/boards/SAMD_GENERIC_D51X19/mpconfigboard.mk b/ports/samd/boards/SAMD_GENERIC_D51X19/mpconfigboard.mk new file mode 100644 index 0000000000000..1a20643214f1a --- /dev/null +++ b/ports/samd/boards/SAMD_GENERIC_D51X19/mpconfigboard.mk @@ -0,0 +1,11 @@ +MCU_SERIES = SAMD51 +CMSIS_MCU = SAMD51P19A +LD_FILES = boards/samd51x19a.ld sections.ld +TEXT0 = 0x4000 + +# The ?='s allow overriding in mpconfigboard.mk. +# MicroPython settings +# The size of a MCU flash filesystem will be +# 496k - MICROPY_HW_CODESIZE - MICROPY_HW_VFSROMSIZE +# The default for MICROPY_HW_VFSROMSIZE is 64K +MICROPY_HW_CODESIZE ?= 368K diff --git a/ports/samd/boards/SAMD_GENERIC_D51X19/pins.csv b/ports/samd/boards/SAMD_GENERIC_D51X19/pins.csv new file mode 100644 index 0000000000000..76e38a98467dc --- /dev/null +++ b/ports/samd/boards/SAMD_GENERIC_D51X19/pins.csv @@ -0,0 +1,18 @@ +# The lines contain pairs of Pin name and Pin number. +# Pin names must be valid Python identifiers. +# Pin numbers have the form Pxnn, with x being A, B, C or D. +# Lines starting with # or empty lines are ignored. + +USB_DM,PA24 +USB_DP,PA25 +USB_SOF,PA23 + +QSPI_CS,PB11 +QSPI_SCK,PB10 +QSPI_D0,PA08 +QSPI_D1,PA09 +QSPI_D2,PA10 +QSPI_D3,PA11 + +SWCLK,PA30 +SWDIO,PA31 From 6b2e359076ea394d8a20f2d17b1cd2c68e3b5c74 Mon Sep 17 00:00:00 2001 From: robert-hh Date: Thu, 23 Jan 2025 13:54:28 +0100 Subject: [PATCH 043/110] samd/boards: Add generic SAMD51x20 board definitions. The definition uses the internal oscillator for clock and only internal flash for the file system. It works at SAMD51J20 device as well, only that fewer pins are accessible. Tested with a SAMD51J20 board. Signed-off-by: robert-hh --- .../samd/boards/SAMD_GENERIC_D51X20/board.json | 16 ++++++++++++++++ ports/samd/boards/SAMD_GENERIC_D51X20/board.md | 4 ++++ .../boards/SAMD_GENERIC_D51X20/mpconfigboard.h | 2 ++ .../boards/SAMD_GENERIC_D51X20/mpconfigboard.mk | 13 +++++++++++++ ports/samd/boards/SAMD_GENERIC_D51X20/pins.csv | 17 +++++++++++++++++ 5 files changed, 52 insertions(+) create mode 100644 ports/samd/boards/SAMD_GENERIC_D51X20/board.json create mode 100644 ports/samd/boards/SAMD_GENERIC_D51X20/board.md create mode 100644 ports/samd/boards/SAMD_GENERIC_D51X20/mpconfigboard.h create mode 100644 ports/samd/boards/SAMD_GENERIC_D51X20/mpconfigboard.mk create mode 100644 ports/samd/boards/SAMD_GENERIC_D51X20/pins.csv diff --git a/ports/samd/boards/SAMD_GENERIC_D51X20/board.json b/ports/samd/boards/SAMD_GENERIC_D51X20/board.json new file mode 100644 index 0000000000000..ae4f83472ecd4 --- /dev/null +++ b/ports/samd/boards/SAMD_GENERIC_D51X20/board.json @@ -0,0 +1,16 @@ +{ + "deploy": [ + "../deploy.md" + ], + "docs": "", + "features": [ + "USB" + ], + "images": [ + "generic_board.jpg" + ], + "mcu": "samd51", + "vendor": "Microchip", + "product": "Generic SAMD51P20", + "thumbnail": "" +} diff --git a/ports/samd/boards/SAMD_GENERIC_D51X20/board.md b/ports/samd/boards/SAMD_GENERIC_D51X20/board.md new file mode 100644 index 0000000000000..4e75f901e3ca5 --- /dev/null +++ b/ports/samd/boards/SAMD_GENERIC_D51X20/board.md @@ -0,0 +1,4 @@ +The following firmware should work on most boards with a +SAMD51J20 and SAMD51P20 MCU. It uses only the features built into +the MCU. Additional devices at the board like external flash +are not supported. diff --git a/ports/samd/boards/SAMD_GENERIC_D51X20/mpconfigboard.h b/ports/samd/boards/SAMD_GENERIC_D51X20/mpconfigboard.h new file mode 100644 index 0000000000000..30e5fa9e6ebb3 --- /dev/null +++ b/ports/samd/boards/SAMD_GENERIC_D51X20/mpconfigboard.h @@ -0,0 +1,2 @@ +#define MICROPY_HW_BOARD_NAME "Generic SAMD51P20" +#define MICROPY_HW_MCU_NAME "SAMD51P20A" diff --git a/ports/samd/boards/SAMD_GENERIC_D51X20/mpconfigboard.mk b/ports/samd/boards/SAMD_GENERIC_D51X20/mpconfigboard.mk new file mode 100644 index 0000000000000..ddba3dcbbade9 --- /dev/null +++ b/ports/samd/boards/SAMD_GENERIC_D51X20/mpconfigboard.mk @@ -0,0 +1,13 @@ +MCU_SERIES = SAMD51 +CMSIS_MCU = SAMD51P20A +LD_FILES = boards/samd51x19a.ld sections.ld +TEXT0 = 0x4000 + + +# The ?='s allow overriding in mpconfigboard.mk. +# MicroPython settings +# The size of a MCU flash filesystem will be +# 1008k - MICROPY_HW_CODESIZE - MICROPY_HW_VFSROMSIZE +# The default for MICROPY_HW_VFSROMSIZE is 64K +MICROPY_HW_CODESIZE ?= 752K +MICROPY_HW_VFSROMSIZE ?= 128K diff --git a/ports/samd/boards/SAMD_GENERIC_D51X20/pins.csv b/ports/samd/boards/SAMD_GENERIC_D51X20/pins.csv new file mode 100644 index 0000000000000..24820c25bbdc9 --- /dev/null +++ b/ports/samd/boards/SAMD_GENERIC_D51X20/pins.csv @@ -0,0 +1,17 @@ +# The lines contain pairs of Pin name and Pin number. +# Pin names must be valid Python identifiers. +# Pin numbers have the form Pxnn, with x being A, B, C or D. +# Lines starting with # or empty lines are ignored. + +USB_DM,PA24 +USB_DP,PA25 + +QSPI_CS,PB11 +QSPI_SCK,PB10 +QSPI_D0,PA08 +QSPI_D1,PA09 +QSPI_D2,PA10 +QSPI_D3,PA11 + +SWCLK,PA30 +SWDIO,PA31 From ff9c6da88c82e8423e577e131d91c5708b788f26 Mon Sep 17 00:00:00 2001 From: robert-hh Date: Thu, 23 Jan 2025 14:16:13 +0100 Subject: [PATCH 044/110] samd/Makefile: Add support for board variants. Tested with a Adafruit SAMD QT board, which may optionally be equipped with SPIFLASH memory. Signed-off-by: robert-hh --- ports/samd/Makefile | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ports/samd/Makefile b/ports/samd/Makefile index 75a4d9b1de355..005664f178d3c 100644 --- a/ports/samd/Makefile +++ b/ports/samd/Makefile @@ -13,7 +13,19 @@ ifeq ($(wildcard $(BOARD_DIR)/.),) $(error Invalid BOARD specified: $(BOARD_DIR)) endif +ifneq ($(BOARD_VARIANT),) +ifeq ($(wildcard $(BOARD_DIR)/mpconfigvariant_$(BOARD_VARIANT).mk),) +$(error Invalid BOARD_VARIANT specified: $(BOARD_VARIANT)) +endif +endif + +# If the build directory is not given, make it reflect the board name (and +# optionally the board variant). +ifneq ($(BOARD_VARIANT),) +BUILD ?= build-$(BOARD)-$(BOARD_VARIANT) +else BUILD ?= build-$(BOARD) +endif CROSS_COMPILE ?= arm-none-eabi- UF2CONV ?= $(TOP)/tools/uf2conv.py @@ -21,7 +33,13 @@ UF2CONV ?= $(TOP)/tools/uf2conv.py MCU_SERIES_LOWER = $(shell echo $(MCU_SERIES) | tr '[:upper:]' '[:lower:]') include ../../py/mkenv.mk +# Include board specific .mk file, and optional board variant .mk file. include $(BOARD_DIR)/mpconfigboard.mk +ifeq ($(BOARD_VARIANT),) +-include $(BOARD_DIR)/mpconfigvariant.mk +else +include $(BOARD_DIR)/mpconfigvariant_$(BOARD_VARIANT).mk +endif include mcu/$(MCU_SERIES_LOWER)/mpconfigmcu.mk # Qstr definitions (must come before including py.mk) From 6cbe145ca8b46857cd2ba5289021022bb92cb02e Mon Sep 17 00:00:00 2001 From: robert-hh Date: Thu, 23 Jan 2025 14:19:07 +0100 Subject: [PATCH 045/110] samd/boards: Add support for the Adafruit QT Py board. Supporting a variant with an optional SPIFLASH device as well. Tested both variants with a QT Py board. Signed-off-by: robert-hh --- .../boards/ADAFRUIT_QTPY_SAMD21/board.json | 20 +++++++++++++++ .../ADAFRUIT_QTPY_SAMD21/mpconfigboard.h | 6 +++++ .../ADAFRUIT_QTPY_SAMD21/mpconfigboard.mk | 4 +++ .../mpconfigvariant_SPIFLASH.mk | 2 ++ .../samd/boards/ADAFRUIT_QTPY_SAMD21/pins.csv | 25 +++++++++++++++++++ 5 files changed, 57 insertions(+) create mode 100644 ports/samd/boards/ADAFRUIT_QTPY_SAMD21/board.json create mode 100644 ports/samd/boards/ADAFRUIT_QTPY_SAMD21/mpconfigboard.h create mode 100644 ports/samd/boards/ADAFRUIT_QTPY_SAMD21/mpconfigboard.mk create mode 100644 ports/samd/boards/ADAFRUIT_QTPY_SAMD21/mpconfigvariant_SPIFLASH.mk create mode 100644 ports/samd/boards/ADAFRUIT_QTPY_SAMD21/pins.csv diff --git a/ports/samd/boards/ADAFRUIT_QTPY_SAMD21/board.json b/ports/samd/boards/ADAFRUIT_QTPY_SAMD21/board.json new file mode 100644 index 0000000000000..f48416dd02ab2 --- /dev/null +++ b/ports/samd/boards/ADAFRUIT_QTPY_SAMD21/board.json @@ -0,0 +1,20 @@ +{ + "deploy": [ + "../deploy.md" + ], + "docs": "", + "features": [ + "USB-C" + ], + "images": [ + "qt_py_samd21.jpg" + ], + "mcu": "samd21", + "product": "QT Py - SAMD21", + "thumbnail": "", + "url": "https://www.adafruit.com/product/4600", + "variants": { + "SPIFLASH": "Support for an external Flash chip" + }, + "vendor": "Adafruit" +} diff --git a/ports/samd/boards/ADAFRUIT_QTPY_SAMD21/mpconfigboard.h b/ports/samd/boards/ADAFRUIT_QTPY_SAMD21/mpconfigboard.h new file mode 100644 index 0000000000000..0acf28afbfe15 --- /dev/null +++ b/ports/samd/boards/ADAFRUIT_QTPY_SAMD21/mpconfigboard.h @@ -0,0 +1,6 @@ +#define MICROPY_HW_BOARD_NAME "QT Py" +#define MICROPY_HW_MCU_NAME "SAMD21E18A" + +#define MICROPY_HW_DFLL_USB_SYNC (1) + +#define MICROPY_HW_SPIFLASH_ID (3) diff --git a/ports/samd/boards/ADAFRUIT_QTPY_SAMD21/mpconfigboard.mk b/ports/samd/boards/ADAFRUIT_QTPY_SAMD21/mpconfigboard.mk new file mode 100644 index 0000000000000..5b4d0b63e7e5a --- /dev/null +++ b/ports/samd/boards/ADAFRUIT_QTPY_SAMD21/mpconfigboard.mk @@ -0,0 +1,4 @@ +MCU_SERIES = SAMD21 +CMSIS_MCU = SAMD21E18A +LD_FILES = boards/samd21x18a.ld sections.ld +TEXT0 = 0x2000 diff --git a/ports/samd/boards/ADAFRUIT_QTPY_SAMD21/mpconfigvariant_SPIFLASH.mk b/ports/samd/boards/ADAFRUIT_QTPY_SAMD21/mpconfigvariant_SPIFLASH.mk new file mode 100644 index 0000000000000..69537d5bf3c51 --- /dev/null +++ b/ports/samd/boards/ADAFRUIT_QTPY_SAMD21/mpconfigvariant_SPIFLASH.mk @@ -0,0 +1,2 @@ +CFLAGS += -DMICROPY_HW_SPIFLASH=1 +MICROPY_HW_CODESIZE ?= 232K diff --git a/ports/samd/boards/ADAFRUIT_QTPY_SAMD21/pins.csv b/ports/samd/boards/ADAFRUIT_QTPY_SAMD21/pins.csv new file mode 100644 index 0000000000000..c3ada219f16b2 --- /dev/null +++ b/ports/samd/boards/ADAFRUIT_QTPY_SAMD21/pins.csv @@ -0,0 +1,25 @@ +A0,PA02 +A1,PA03 +A2,PA04 +A3,PA05 +SDA,PA16 +SCL,PA17 +TX,PA06 +RX,PA07 +SCK,PA11 +MISO,PA09 +MOSI,PA10 + +NEO_PWR,PA15 +NEOPIX,PA18 + +FLASH_MOSI,PA22 +FLASH_MISO,PA19 +FLASH_SCK,PA23 +FLASH_CS,PA08 + +USB_DM,PA24 +USB_DP,PA25 + +SWCLK,PA30 +SWDIO,PA31 From 36a0a839974c6e49723794421f46f6282e60e56f Mon Sep 17 00:00:00 2001 From: robert-hh Date: Thu, 23 Jan 2025 14:21:52 +0100 Subject: [PATCH 046/110] samd/boards: Add support for the Adafruit NeoKey Trinkey board. Tested with that board. Signed-off-by: robert-hh --- .../boards/ADAFRUIT_NEOKEY_TRINKEY/board.json | 18 ++++++++++++++++++ .../ADAFRUIT_NEOKEY_TRINKEY/mpconfigboard.h | 14 ++++++++++++++ .../ADAFRUIT_NEOKEY_TRINKEY/mpconfigboard.mk | 4 ++++ .../boards/ADAFRUIT_NEOKEY_TRINKEY/pins.csv | 14 ++++++++++++++ 4 files changed, 50 insertions(+) create mode 100644 ports/samd/boards/ADAFRUIT_NEOKEY_TRINKEY/board.json create mode 100644 ports/samd/boards/ADAFRUIT_NEOKEY_TRINKEY/mpconfigboard.h create mode 100644 ports/samd/boards/ADAFRUIT_NEOKEY_TRINKEY/mpconfigboard.mk create mode 100644 ports/samd/boards/ADAFRUIT_NEOKEY_TRINKEY/pins.csv diff --git a/ports/samd/boards/ADAFRUIT_NEOKEY_TRINKEY/board.json b/ports/samd/boards/ADAFRUIT_NEOKEY_TRINKEY/board.json new file mode 100644 index 0000000000000..9b0c307d679c3 --- /dev/null +++ b/ports/samd/boards/ADAFRUIT_NEOKEY_TRINKEY/board.json @@ -0,0 +1,18 @@ +{ + "deploy": [ + "../deploy.md" + ], + "docs": "", + "features": [ + "RGB LED", + "USB" + ], + "images": [ + "neokey_trinkey.jpg" + ], + "mcu": "samd21", + "product": "NeoKey Trinkey", + "thumbnail": "", + "url": "https://www.adafruit.com/product/5020", + "vendor": "Adafruit" +} diff --git a/ports/samd/boards/ADAFRUIT_NEOKEY_TRINKEY/mpconfigboard.h b/ports/samd/boards/ADAFRUIT_NEOKEY_TRINKEY/mpconfigboard.h new file mode 100644 index 0000000000000..eb4704ff8cb97 --- /dev/null +++ b/ports/samd/boards/ADAFRUIT_NEOKEY_TRINKEY/mpconfigboard.h @@ -0,0 +1,14 @@ +#define MICROPY_HW_BOARD_NAME "NeoKey Trinkey" +#define MICROPY_HW_MCU_NAME "SAMD21E18A" + +#define MICROPY_HW_DFLL_USB_SYNC (1) + +// The NEOKEY board has just two accessible GPIO pins. +// So many classes and modules are useless. +#define MICROPY_PY_MACHINE_SOFTI2C (0) +#define MICROPY_PY_MACHINE_SOFTSPI (0) +#define MICROPY_PY_MACHINE_I2C (0) +#define MICROPY_PY_MACHINE_SPI (0) +#define MICROPY_PY_MACHINE_UART (0) +#define MICROPY_PY_MACHINE_ADC (0) +#define MICROPY_PY_MACHINE_DAC (0) diff --git a/ports/samd/boards/ADAFRUIT_NEOKEY_TRINKEY/mpconfigboard.mk b/ports/samd/boards/ADAFRUIT_NEOKEY_TRINKEY/mpconfigboard.mk new file mode 100644 index 0000000000000..5b4d0b63e7e5a --- /dev/null +++ b/ports/samd/boards/ADAFRUIT_NEOKEY_TRINKEY/mpconfigboard.mk @@ -0,0 +1,4 @@ +MCU_SERIES = SAMD21 +CMSIS_MCU = SAMD21E18A +LD_FILES = boards/samd21x18a.ld sections.ld +TEXT0 = 0x2000 diff --git a/ports/samd/boards/ADAFRUIT_NEOKEY_TRINKEY/pins.csv b/ports/samd/boards/ADAFRUIT_NEOKEY_TRINKEY/pins.csv new file mode 100644 index 0000000000000..c979f5b9bc147 --- /dev/null +++ b/ports/samd/boards/ADAFRUIT_NEOKEY_TRINKEY/pins.csv @@ -0,0 +1,14 @@ +# The lines contain pairs of Pin name and Pin number. +# Pin names must be valid Python identifiers. +# Pin numbers have the form Pxnn, with x being A, B, C or D. +# Lines starting with # or empty lines are ignored. + +NEOPIXEL,PA15 +SWITCH,PA18 +TOUCH,PA07 + +USB_DM,PA24 +USB_DP,PA25 + +SWCLK,PA30 +SWDIO,PA31 From a84143d2230daa3843f00dd1f09474441810f08d Mon Sep 17 00:00:00 2001 From: robert-hh Date: Thu, 23 Jan 2025 17:38:40 +0100 Subject: [PATCH 047/110] docs/samd/pinout: Add pinout for Adafruit NeoKey Trinkey and QT Py. Only pins accessible at the board are shown. Signed-off-by: robert-hh --- docs/samd/pinout.rst | 60 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/docs/samd/pinout.rst b/docs/samd/pinout.rst index 5da4ae78d45b5..f6473e5495f5a 100644 --- a/docs/samd/pinout.rst +++ b/docs/samd/pinout.rst @@ -567,6 +567,66 @@ The default devices at the board are: - SPI 0 at pins PA06/PA09/PA08, labelled D4, D2 and D0 - DAC output on pin PA02, labelled D1 +Adafruit QT PY pin assignment table +----------------------------------- + +=== ==== ============ ==== ==== ====== ====== ====== ====== +Pin GPIO Pin name IRQ ADC Serial Serial TCC/TC TCC/TC +=== ==== ============ ==== ==== ====== ====== ====== ====== + 2 PA02 A0 2 0 - - - - + 3 PA03 A1 3 1 - - - - + 4 PA04 A2 4 4 - 0/0 0/0 - + 5 PA05 A3 5 5 - 0/1 0/1 - + 7 PA07 RX 7 7 - 0/3 1/1 - + 6 PA06 TX 6 6 - 0/2 1/0 - + 8 PA08 FLASH_CS - 16 0/0 2/0 0/0 1/2 + 19 PA19 FLASH_MISO 3 - 1/3 3/3 3/1 0/3 + 22 PA22 FLASH_MOSI 6 - 3/0 5/0 4/0 0/4 + 23 PA23 FLASH_SCK 7 - 3/1 5/1 4/1 0/5 + 9 PA09 MISO 9 17 0/1 2/1 0/1 1/3 + 10 PA10 MOSI 10 18 0/2 2/2 1/0 0/2 + 18 PA18 NEOPIX 2 - 1/2 3/2 3/0 0/2 + 15 PA15 NEO_PWR 15 - 2/3 4/3 3/1 0/5 + 11 PA11 SCK 11 19 0/3 2/3 1/1 0/3 + 17 PA17 SCL 1 - 1/1 3/1 2/1 0/7 + 16 PA16 SDA 0 - 1/0 3/0 2/0 0/6 + 30 PA30 SWCLK 10 - - 1/2 1/0 - + 31 PA31 SWDIO 11 - - 1/3 1/1 - + 24 PA24 USB_DM 12 - 3/2 5/2 5/0 1/2 + 25 PA25 USB_DP 13 - 3/3 5/3 5/1 1/3 +=== ==== ============ ==== ==== ====== ====== ====== ====== + +For the definition of the table columns see the explanation at the table for +Adafruit ItsyBitsy M0 Express :ref:`samd21_pinout_table`. + +The default devices at the board are: + +- UART 0 at pins PA07/PA06, labelled RX/TX +- I2C 1 at pins PA16/PA17, labelled SDA/SCL +- SPI 0 at pins PA09/PA10/PA11, labelled MISO, MOSI and SCK +- DAC output on pin PA02, labelled A0 + +Adafruit NeoKey Trinkey pin assignment table +-------------------------------------------- + +=== ==== ============ ==== ==== ====== ====== ====== ====== +Pin GPIO Pin name IRQ ADC Serial Serial TCC/TC TCC/TC +=== ==== ============ ==== ==== ====== ====== ====== ====== + 15 PA15 NEOPIXEL 15 - 2/3 4/3 3/1 0/5 + 30 PA30 SWCLK 10 - - 1/2 1/0 - + 31 PA31 SWDIO 11 - - 1/3 1/1 - + 18 PA18 SWITCH 2 - 1/2 3/2 3/0 0/2 + 7 PA07 TOUCH 7 7 - 0/3 1/1 - + 24 PA24 USB_DM 12 - 3/2 5/2 5/0 1/2 + 25 PA25 USB_DP 13 - 3/3 5/3 5/1 1/3 +=== ==== ============ ==== ==== ====== ====== ====== ====== + +For the definition of the table columns see the explanation at the table for +Adafruit ItsyBitsy M0 Express :ref:`samd21_pinout_table`. + +The board does not provide access to UART, I2C, SPI or DAC. + + SAMD21 Xplained PRO pin assignment table ---------------------------------------- From e44a2c6921919d0d19356a70a23bf21cd301856c Mon Sep 17 00:00:00 2001 From: robert-hh Date: Thu, 23 Jan 2025 20:14:27 +0100 Subject: [PATCH 048/110] docs/samd/pinout: Add pinout for the Generic SAMD board types. The table shows the devices available at the pin and the respective package letter. Signed-off-by: robert-hh --- docs/samd/pinout.rst | 182 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 182 insertions(+) diff --git a/docs/samd/pinout.rst b/docs/samd/pinout.rst index f6473e5495f5a..776ab74bcdb22 100644 --- a/docs/samd/pinout.rst +++ b/docs/samd/pinout.rst @@ -935,6 +935,188 @@ The default devices at the board are: - SPI 4 at pins PB12/PB11/PB13, labelled MOSI, MISO and SCK - DAC output on pins PA02 and PA05, labelled A0 and A4 +Generic SAMD21x18 pin assignment table +-------------------------------------- + +=== ==== ============ ==== ==== ====== ====== ====== ====== +Pin GPIO Name/Package IRQ ADC Serial Serial TCC/TC TCC/TC +=== ==== ============ ==== ==== ====== ====== ====== ====== + 0 PA00 EGJ 0 - - 1/0 2/0 - + 1 PA01 EGJ 1 - - 1/1 2/1 - + 2 PA02 EGJ 2 0 - - - - + 3 PA03 EGJ 3 1 - - - - + 4 PA04 EGJ 4 4 - 0/0 0/0 - + 5 PA05 EGJ 5 5 - 0/1 0/1 - + 6 PA06 EGJ 6 6 - 0/2 1/0 - + 7 PA07 EGJ 7 7 - 0/3 1/1 - + 8 PA08 EGJ - 16 0/0 2/0 0/0 1/2 + 9 PA09 EGJ 9 17 0/1 2/1 0/1 1/3 + 10 PA10 EGJ 10 18 0/2 2/2 1/0 0/2 + 11 PA11 EGJ 11 19 0/3 2/3 1/1 0/3 + 12 PA12 GJ 12 - 2/0 4/0 2/0 0/6 + 13 PA13 GJ 13 - 2/1 4/1 2/0 0/7 + 14 PA14 EGJ 14 - 2/2 4/2 3/0 0/4 + 15 PA15 EGJ 15 - 2/3 4/3 3/1 0/5 + 16 PA16 EGJ 0 - 1/0 3/0 2/0 0/6 + 17 PA17 EGJ 1 - 1/1 3/1 2/1 0/7 + 18 PA18 EGJ 2 - 1/2 3/2 3/0 0/2 + 19 PA19 EGJ 3 - 1/3 3/3 3/1 0/3 + 20 PA20 GJ 4 - 5/2 3/2 7/0 0/4 + 21 PA21 GJ 5 - 5/3 3/3 7/1 0/7 + 22 PA22 EGJ 6 - 3/0 5/0 4/0 0/4 + 23 PA23 EGJ 7 - 3/1 5/1 4/1 0/5 + 24 PA24 USB_DM 12 - 3/2 5/2 5/0 1/2 + 25 PA25 USB_DP 13 - 3/3 5/3 5/1 1/3 + 27 PA27 EGJ 15 - - - - - + 28 PA28 EGJ 8 - - - - - + 30 PA30 SWCLK 10 - - 1/2 1/0 - + 31 PA31 SWDIO 11 - - 1/3 1/1 - + 32 PB00 J 0 8 - 5/2 7/0 - + 33 PB01 J 1 9 - 5/3 7/1 - + 34 PB02 GJ 2 10 - 5/0 6/0 - + 35 PB03 GJ 3 11 - 5/1 6/1 - + 36 PB04 J 4 12 - - - - + 37 PB05 J 5 13 - - - - + 38 PB06 J 6 14 - - - - + 39 PB07 J 7 15 - - - - + 40 PB08 GJ 8 2 - 4/0 4/0 - + 41 PB09 GJ 9 3 - 4/1 4/1 - + 42 PB10 GJ 10 - - 4/2 5/0 0/4 + 43 PB11 GJ 11 - - 4/3 5/1 0/5 + 44 PB12 J 12 - 4/0 - 4/0 0/6 + 45 PB13 J 13 - 4/1 - 4/1 0/7 + 46 PB14 J 14 - 4/2 - 5/0 - + 47 PB15 J 15 - 4/3 - 5/1 - + 48 PB16 J 0 - 5/0 - 6/0 0/4 + 49 PB17 J 1 - 5/1 - 6/1 0/5 + 54 PB22 GJ 6 - - 5/2 7/0 - + 55 PB23 GJ 7 - - 5/3 7/1 - + 62 PB30 J 14 - - 5/0 0/0 1/2 + 63 PB31 J 15 - - 5/1 0/1 1/3 +=== ==== ============ ==== ==== ====== ====== ====== ====== + +For the definition of the table columns see the explanation at the table for +Adafruit ItsyBitsy M0 Express :ref:`samd21_pinout_table`. + +The Package column indicates the package letter providing this pin. An entry +EGJ tells for instance, that the pin is available for SAMD21E18, SAMD21G18 and +SAMD21J18. + + +Generic SAMD51x19 and SAM51x20 pin assignment table +--------------------------------------------------- + +For the definition of the table columns see the explanation at the table for +Adafruit ItsyBitsy M4 Express :ref:`samd51_pinout_table`. + +=== ==== ============ ==== ==== ==== ====== ====== ===== ===== ===== +Pin GPIO Name/Package IRQ ADC ADC Serial Serial TC PWM PWM +=== ==== ============ ==== ==== ==== ====== ====== ===== ===== ===== + 8 PA08 QSPI_D0 - 8 2 0/0 2/1 0/0 0/0 1/4 + 9 PA09 QSPI_D1 9 9 3 0/1 2/0 0/1 0/1 1/5 + 10 PA10 QSPI_D2 10 10 - 0/2 2/2 1/0 0/2 1/6 + 11 PA11 QSPI_D3 11 11 - 0/3 2/3 1/1 0/3 1/7 + 42 PB10 QSPI_SCK 10 - - - 4/2 5/0 0/4 1/0 + 23 PA23 USB_SOF 7 - - 3/1 5/0 4/1 1/7 0/3 + 24 PA24 USB_DM 8 - - 3/2 5/2 5/0 2/2 - + 25 PA25 USB_DP 9 - - 3/3 5/3 5/1 - - + 0 PA00 GJP 0 - - - 1/0 2/0 - - + 1 PA01 GJP 1 - - - 1/1 2/1 - - + 2 PA02 GJP 2 0 - - - - - - + 3 PA03 GJP 3 10 - - - - - - + 4 PA04 GJP 4 4 - - 0/0 0/0 - - + 5 PA05 GJP 5 5 - - 0/1 0/1 - - + 6 PA06 GJP 6 6 - - 0/2 1/0 - - + 7 PA07 GJP 7 7 - - 0/3 1/1 - - + 12 PA12 GJP 12 - - 2/0 4/1 2/0 0/6 1/2 + 13 PA13 GJP 13 - - 2/1 4/0 2/1 0/7 1/3 + 14 PA14 GJP 14 - - 2/2 4/2 3/0 2/0 1/2 + 15 PA15 GJP 15 - - 2/3 4/3 3/1 2/1 1/3 + 16 PA16 GJP 0 - - 1/0 3/1 2/0 1/0 0/4 + 17 PA17 GJP 1 - - 1/1 3/0 2/1 1/1 0/5 + 18 PA18 GJP 2 - - 1/2 3/2 3/0 1/2 0/6 + 19 PA19 GJP 3 - - 1/3 3/3 3/1 1/3 0/7 + 20 PA20 GJP 4 - - 5/2 3/2 7/0 1/4 0/0 + 21 PA21 GJP 5 - - 5/3 3/3 7/1 1/5 0/1 + 22 PA22 GJP 6 - - 3/0 5/1 4/0 1/6 0/2 + 27 PA27 GJP 11 - - - - - - - + 30 PA30 SWCLK 14 - - 7/2 1/2 6/0 2/0 - + 31 PA31 SWDIO 15 - - 7/3 1/3 6/1 2/1 - + 32 PB00 JP 0 12 - - 5/2 7/0 - - + 33 PB01 JP 1 13 - - 5/3 7/1 - - + 34 PB02 GJP 2 14 - - 5/0 6/0 2/2 - + 35 PB03 GJP 3 15 - - 5/1 6/1 - - + 36 PB04 JP 4 - 6 - - - - - + 37 PB05 JP 5 - 7 - - - - - + 38 PB06 JP 6 - 8 - - - - - + 39 PB07 JP 7 - 9 - - - - - + 40 PB08 GJP 8 2 0 - 4/0 4/0 - - + 41 PB09 GJP 9 3 1 - 4/1 4/1 - - + 44 PB12 JP 12 - - 4/0 - 4/0 3/0 0/0 + 45 PB13 JP 13 - - 4/1 - 4/1 3/1 0/1 + 46 PB14 JP 14 - - 4/2 - 5/0 4/0 0/2 + 47 PB15 JP 15 - - 4/3 - 5/1 4/1 0/3 + 48 PB16 JP 0 - - 5/0 - 6/0 3/0 0/4 + 49 PB17 JP 1 - - 5/1 - 6/1 3/1 0/5 + 50 PB18 P 2 - - 5/2 7/2 - 1/0 - + 51 PB19 P 3 - - 5/3 7/3 - 1/1 - + 52 PB20 P 4 - - 3/0 7/1 - 1/2 - + 53 PB21 P 5 - - 3/1 7/0 - 1/3 - + 54 PB22 GJP 6 - - 1/2 5/2 7/0 - - + 55 PB23 GJP 7 - - 1/3 5/3 7/1 - - + 56 PB24 P 8 - - 0/0 2/1 - - - + 57 PB25 P 9 - - 0/1 2/0 - - - + 58 PB26 P 12 - - 2/0 4/1 - 1/2 - + 59 PB27 P 13 - - 2/1 4/0 - 1/3 - + 60 PB28 P 14 - - 2/2 4/2 - 1/4 - + 61 PB29 P 15 - - 2/3 4/3 - 1/5 - + 62 PB30 JP 14 - - 7/0 5/1 0/0 4/0 0/6 + 63 PB31 JP 15 - - 7/1 5/0 0/1 4/1 0/7 + 64 PC00 P 0 - 10 - - - - - + 65 PC01 P 1 - 11 - - - - - + 66 PC02 P 2 - 4 - - - - - + 67 PC03 P 3 - 5 - - - - - + 68 PC04 P 4 - - 6/0 - - 0/0 - + 69 PC05 P 5 - - 6/1 - - - - + 70 PC06 P 6 - - 6/2 - - - - + 71 PC07 P 9 - - 6/3 - - - - + 74 PC10 P 10 - - 6/2 7/2 - 0/0 1/4 + 75 PC11 P 11 - - 6/3 7/3 - 0/1 1/5 + 76 PC12 P 12 - - 7/0 6/1 - 0/2 1/6 + 77 PC13 P 13 - - 7/1 6/0 - 0/3 1/7 + 78 PC14 P 14 - - 7/2 6/2 - 0/4 1/0 + 79 PC15 P 15 - - 7/3 6/3 - 0/5 1/1 + 80 PC16 P 0 - - 6/0 0/1 - 0/0 - + 81 PC17 P 1 - - 6/1 0/0 - 0/1 - + 82 PC18 P 2 - - 6/2 0/2 - 0/2 - + 83 PC19 P 3 - - 6/3 0/3 - 0/3 - + 84 PC20 P 4 - - - - - 0/4 - + 85 PC21 P 5 - - - - - 0/5 - + 86 PC22 P 6 - - 1/0 3/1 - 0/5 - + 87 PC23 P 7 - - 1/1 3/0 - 0/7 - + 88 PC24 P 8 - - 0/2 2/2 - - - + 89 PC25 P 9 - - 0/3 2/3 - - - + 90 PC26 P 10 - - - - - - - + 91 PC27 P 11 - - 1/0 - - - - + 92 PC28 P 12 - - 1/1 - - - - + 94 PC30 P 14 - 12 - - - - - + 95 PC31 P 15 - 13 - - - - - + 96 PD00 P 0 - 14 - - - - - + 97 PD01 P 1 - 15 - - - - - +104 PD08 P 3 - - 7/0 6/1 - 0/1 - +105 PD09 P 4 - - 7/1 6/0 - 0/2 - +106 PD10 P 5 - - 7/2 6/2 - 0/3 - +107 PD11 P 6 - - 7/3 6/3 - 0/4 - +108 PD12 P 7 - - - - - 0/5 - +116 PD20 P 10 - - 1/2 3/2 - 1/0 - +117 PD21 P 11 - - 1/3 3/3 - 1/1 - +=== ==== ============ ==== ==== ==== ====== ====== ===== ===== ===== + + +The Package column indicates the package letter providing this pin. An entry +GJP tells for instance, that the pin is available for SAMD51G19, SAMD51J19/-J20 and +SAMD51P19/-P20. + Scripts for creating the pin assignment tables ---------------------------------------------- From e176fea95cda017cf351e779d1aa323da52f1151 Mon Sep 17 00:00:00 2001 From: iabdalkader Date: Sat, 25 Jan 2025 08:50:45 +0100 Subject: [PATCH 049/110] mimxrt/irq: Add CSI IRQ. Signed-off-by: iabdalkader --- ports/mimxrt/irq.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ports/mimxrt/irq.h b/ports/mimxrt/irq.h index b83eb49ffabfa..8202a98e8748c 100644 --- a/ports/mimxrt/irq.h +++ b/ports/mimxrt/irq.h @@ -69,6 +69,8 @@ static inline void restore_irq_pri(uint32_t basepri) { #define IRQ_PRI_SYSTICK NVIC_EncodePriority(NVIC_PRIORITYGROUP_4, 0, 0) +#define IRQ_PRI_CSI NVIC_EncodePriority(NVIC_PRIORITYGROUP_4, 3, 0) + #define IRQ_PRI_OTG_HS NVIC_EncodePriority(NVIC_PRIORITYGROUP_4, 6, 0) #define IRQ_PRI_EXTINT NVIC_EncodePriority(NVIC_PRIORITYGROUP_4, 14, 0) From 67ebc537c3d0095d3b6a0ef8a7160ba06ea95dff Mon Sep 17 00:00:00 2001 From: iabdalkader Date: Sat, 25 Jan 2025 08:59:47 +0100 Subject: [PATCH 050/110] mimxrt/machine_rtc: Fix build with new SDKs. In more recent SDKs, this feature is actually disabled for the MIMXRT1062. Signed-off-by: iabdalkader --- ports/mimxrt/machine_rtc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ports/mimxrt/machine_rtc.c b/ports/mimxrt/machine_rtc.c index 5feeb0da096ae..e6c519991983b 100644 --- a/ports/mimxrt/machine_rtc.c +++ b/ports/mimxrt/machine_rtc.c @@ -156,8 +156,10 @@ void machine_rtc_start(void) { SNVS->HPCOMR |= SNVS_HPCOMR_NPSWA_EN_MASK; // Do a basic init. SNVS_LP_Init(SNVS); + #if FSL_FEATURE_SNVS_HAS_MULTIPLE_TAMPER // Disable all external Tamper SNVS_LP_DisableAllExternalTamper(SNVS); + #endif SNVS_LP_SRTC_StartTimer(SNVS); // If the date is not set, set it to a more recent start date, From d76733d05856d389f216d60c71d1a7043d78c510 Mon Sep 17 00:00:00 2001 From: iabdalkader Date: Sat, 25 Jan 2025 13:42:24 +0100 Subject: [PATCH 051/110] mimxrt/mpconfigport: Remove hard-coded CMSIS header. Signed-off-by: iabdalkader --- ports/mimxrt/mpconfigport.h | 1 - 1 file changed, 1 deletion(-) diff --git a/ports/mimxrt/mpconfigport.h b/ports/mimxrt/mpconfigport.h index 146c86015afd2..923eff4dc7064 100644 --- a/ports/mimxrt/mpconfigport.h +++ b/ports/mimxrt/mpconfigport.h @@ -29,7 +29,6 @@ // Board specific definitions #include "mpconfigboard.h" #include "fsl_common.h" -#include "lib/nxp_driver/sdk/CMSIS/Include/core_cm7.h" uint32_t trng_random_u32(void); From 9d0a5ac7e9ac8b90d1bbea402fd16a410aa4008f Mon Sep 17 00:00:00 2001 From: StrayCat Date: Thu, 2 Jan 2025 23:46:44 +1300 Subject: [PATCH 052/110] esp32/boards: Enable I2S on ESP32C3 boards. Signed-off-by: StrayCat --- ports/esp32/boards/ESP32_GENERIC_C3/mpconfigboard.h | 1 - ports/esp32/boards/LOLIN_C3_MINI/mpconfigboard.h | 1 - 2 files changed, 2 deletions(-) diff --git a/ports/esp32/boards/ESP32_GENERIC_C3/mpconfigboard.h b/ports/esp32/boards/ESP32_GENERIC_C3/mpconfigboard.h index 0dbfae03a367c..a6cebdf6ee64f 100644 --- a/ports/esp32/boards/ESP32_GENERIC_C3/mpconfigboard.h +++ b/ports/esp32/boards/ESP32_GENERIC_C3/mpconfigboard.h @@ -4,7 +4,6 @@ #define MICROPY_HW_MCU_NAME "ESP32C3" #define MICROPY_HW_ENABLE_SDCARD (0) -#define MICROPY_PY_MACHINE_I2S (0) // Enable UART REPL for modules that have an external USB-UART and don't use native USB. #define MICROPY_HW_ENABLE_UART_REPL (1) diff --git a/ports/esp32/boards/LOLIN_C3_MINI/mpconfigboard.h b/ports/esp32/boards/LOLIN_C3_MINI/mpconfigboard.h index 9b304b69f6fba..46d720edc758f 100644 --- a/ports/esp32/boards/LOLIN_C3_MINI/mpconfigboard.h +++ b/ports/esp32/boards/LOLIN_C3_MINI/mpconfigboard.h @@ -3,7 +3,6 @@ #define MICROPY_PY_NETWORK_HOSTNAME_DEFAULT "mpy-c3-mini" #define MICROPY_HW_ENABLE_SDCARD (0) -#define MICROPY_PY_MACHINE_I2S (0) #define MICROPY_HW_I2C0_SCL (10) #define MICROPY_HW_I2C0_SDA (8) From 0662c551216db1a3ab40dafbeffcc3ee929d7690 Mon Sep 17 00:00:00 2001 From: Markus Gyger Date: Tue, 31 Dec 2024 15:34:17 +0700 Subject: [PATCH 053/110] rp2/rp2_pio: Add side_pindir support for PIO. Side-setting can also be used to change pin directions instead of pin values. This adds a parameter `side_pindir` to decorator `asm_pio()` to configure it. Also replaces a few close-by 0s with corresponding PIO.* constants. Addresses issue #10027. Signed-off-by: Markus Gyger --- docs/library/rp2.rst | 8 +++++--- ports/rp2/modules/rp2.py | 9 +++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/library/rp2.rst b/docs/library/rp2.rst index f0189327dfb77..a215e98b51e55 100644 --- a/docs/library/rp2.rst +++ b/docs/library/rp2.rst @@ -23,7 +23,7 @@ The ``rp2`` module includes functions for assembling PIO programs. For running PIO programs, see :class:`rp2.StateMachine`. -.. function:: asm_pio(*, out_init=None, set_init=None, sideset_init=None, in_shiftdir=0, out_shiftdir=0, autopush=False, autopull=False, push_thresh=32, pull_thresh=32, fifo_join=PIO.JOIN_NONE) +.. function:: asm_pio(*, out_init=None, set_init=None, sideset_init=None, side_pindir=False, in_shiftdir=PIO.SHIFT_LEFT, out_shiftdir=PIO.SHIFT_LEFT, autopush=False, autopull=False, push_thresh=32, pull_thresh=32, fifo_join=PIO.JOIN_NONE) Assemble a PIO program. @@ -35,8 +35,10 @@ For running PIO programs, see :class:`rp2.StateMachine`. - *out_init* configures the pins used for ``out()`` instructions. - *set_init* configures the pins used for ``set()`` instructions. There can be at most 5. - - *sideset_init* configures the pins used side-setting. There can be at - most 5. + - *sideset_init* configures the pins used for ``.side()`` modifiers. There + can be at most 5. + - *side_pindir* when set to ``True`` configures ``.side()`` modifiers to be + used for pin directions, instead of pin values (the default, when ``False``). The following parameters are used by default, but can be overridden in `StateMachine.init()`: diff --git a/ports/rp2/modules/rp2.py b/ports/rp2/modules/rp2.py index 9d13bf1b52cd5..e9be7dac8d58e 100644 --- a/ports/rp2/modules/rp2.py +++ b/ports/rp2/modules/rp2.py @@ -26,20 +26,21 @@ def __init__( out_init=None, set_init=None, sideset_init=None, - in_shiftdir=0, - out_shiftdir=0, + side_pindir=False, + in_shiftdir=PIO.SHIFT_LEFT, + out_shiftdir=PIO.SHIFT_LEFT, autopush=False, autopull=False, push_thresh=32, pull_thresh=32, - fifo_join=0, + fifo_join=PIO.JOIN_NONE, ): # array is a built-in module so importing it here won't require # scanning the filesystem. from array import array self.labels = {} - execctrl = 0 + execctrl = side_pindir << 29 shiftctrl = ( fifo_join << 30 | (pull_thresh & 0x1F) << 25 From b2ce9b6fb03ed27943733235eac2f3f5e0d5b97d Mon Sep 17 00:00:00 2001 From: Dryw Wade Date: Wed, 4 Dec 2024 16:32:42 -0700 Subject: [PATCH 054/110] rp2/boards: Add SparkFun IoT Node LoRaWAN board. Signed-off-by: Dryw Wade --- .../board.json | 25 ++++++ .../mpconfigboard.cmake | 7 ++ .../mpconfigboard.h | 33 +++++++ .../SPARKFUN_IOTNODE_LORAWAN_RP2350/pins.csv | 34 +++++++ .../sparkfun_iotnode_lorawan_rp2350.h | 88 +++++++++++++++++++ 5 files changed, 187 insertions(+) create mode 100644 ports/rp2/boards/SPARKFUN_IOTNODE_LORAWAN_RP2350/board.json create mode 100644 ports/rp2/boards/SPARKFUN_IOTNODE_LORAWAN_RP2350/mpconfigboard.cmake create mode 100644 ports/rp2/boards/SPARKFUN_IOTNODE_LORAWAN_RP2350/mpconfigboard.h create mode 100644 ports/rp2/boards/SPARKFUN_IOTNODE_LORAWAN_RP2350/pins.csv create mode 100644 ports/rp2/boards/SPARKFUN_IOTNODE_LORAWAN_RP2350/sparkfun_iotnode_lorawan_rp2350.h diff --git a/ports/rp2/boards/SPARKFUN_IOTNODE_LORAWAN_RP2350/board.json b/ports/rp2/boards/SPARKFUN_IOTNODE_LORAWAN_RP2350/board.json new file mode 100644 index 0000000000000..4ddcd999fe5f8 --- /dev/null +++ b/ports/rp2/boards/SPARKFUN_IOTNODE_LORAWAN_RP2350/board.json @@ -0,0 +1,25 @@ +{ + "deploy": [ + "../deploy.md" + ], + "docs": "", + "features": [ + "Battery Charging", + "Dual-core", + "External Flash", + "External RAM", + "JST-SH", + "LoRa", + "RGB LED", + "USB-C", + "microSD" + ], + "images": [ + "26060-IoT-Node-LoRaWAN-Feature-new.jpg" + ], + "mcu": "rp2350", + "product": "IoT Node LoRaWAN RP2350", + "thumbnail": "", + "url": "https://www.sparkfun.com/products/26060", + "vendor": "Sparkfun" +} diff --git a/ports/rp2/boards/SPARKFUN_IOTNODE_LORAWAN_RP2350/mpconfigboard.cmake b/ports/rp2/boards/SPARKFUN_IOTNODE_LORAWAN_RP2350/mpconfigboard.cmake new file mode 100644 index 0000000000000..11fe1ee28fce2 --- /dev/null +++ b/ports/rp2/boards/SPARKFUN_IOTNODE_LORAWAN_RP2350/mpconfigboard.cmake @@ -0,0 +1,7 @@ +# cmake file for SparkFun IoT Node LoRaWAN RP2350 + +# TODO: DELETE THIS LINE ONCE PICO SDK 2.1.1 IS RELEASED +set(PICO_BOARD_HEADER_DIRS ${MICROPY_PORT_DIR}/boards/${MICROPY_BOARD}) + +set(PICO_BOARD "sparkfun_iotnode_lorawan_rp2350") +set(PICO_PLATFORM "rp2350") diff --git a/ports/rp2/boards/SPARKFUN_IOTNODE_LORAWAN_RP2350/mpconfigboard.h b/ports/rp2/boards/SPARKFUN_IOTNODE_LORAWAN_RP2350/mpconfigboard.h new file mode 100644 index 0000000000000..106ec95aa255a --- /dev/null +++ b/ports/rp2/boards/SPARKFUN_IOTNODE_LORAWAN_RP2350/mpconfigboard.h @@ -0,0 +1,33 @@ +// Board and hardware specific configuration +#define MICROPY_HW_BOARD_NAME "SparkFun IoT Node LoRaWAN" +#define MICROPY_HW_FLASH_STORAGE_BYTES (14 * 1024 * 1024) + +#define MICROPY_HW_USB_VID (0x1B4F) +#define MICROPY_HW_USB_PID (0x0044) + +#define MICROPY_HW_I2C0_SDA (20) +#define MICROPY_HW_I2C0_SCL (21) + +#define MICROPY_HW_I2C1_SDA (6) +#define MICROPY_HW_I2C1_SCL (7) + +#define MICROPY_HW_SPI0_SCK (2) +#define MICROPY_HW_SPI0_MOSI (3) +#define MICROPY_HW_SPI0_MISO (4) + +#define MICROPY_HW_SPI1_SCK (14) +#define MICROPY_HW_SPI1_MOSI (15) +#define MICROPY_HW_SPI1_MISO (12) + +#define MICROPY_HW_UART0_TX (18) +#define MICROPY_HW_UART0_RX (19) +#define MICROPY_HW_UART0_CTS (2) +#define MICROPY_HW_UART0_RTS (3) + +#define MICROPY_HW_UART1_TX (4) +#define MICROPY_HW_UART1_RX (5) +#define MICROPY_HW_UART1_CTS (6) +#define MICROPY_HW_UART1_RTS (7) + +#define MICROPY_HW_PSRAM_CS_PIN (0) +#define MICROPY_HW_ENABLE_PSRAM (1) diff --git a/ports/rp2/boards/SPARKFUN_IOTNODE_LORAWAN_RP2350/pins.csv b/ports/rp2/boards/SPARKFUN_IOTNODE_LORAWAN_RP2350/pins.csv new file mode 100644 index 0000000000000..2526b36d10ea2 --- /dev/null +++ b/ports/rp2/boards/SPARKFUN_IOTNODE_LORAWAN_RP2350/pins.csv @@ -0,0 +1,34 @@ +GP0,GPIO0 +GP1,GPIO1 +GP2,GPIO2 +GP3,GPIO3 +GP4,GPIO4 +GP5,GPIO5 +GP6,GPIO6 +GP7,GPIO7 +GP8,GPIO8 +GP9,GPIO9 +GP10,GPIO10 +GP11,GPIO11 +GP12,GPIO12 +GP13,GPIO13 +GP14,GPIO14 +GP15,GPIO15 +GP16,GPIO16 +GP17,GPIO17 +GP18,GPIO18 +GP19,GPIO19 +GP20,GPIO20 +GP21,GPIO21 +GP22,GPIO22 +GP23,GPIO23 +GP24,GPIO24 +GP25,GPIO25 +GP26,GPIO26 +GP27,GPIO27 +GP28,GPIO28 +GP29,GPIO29 +LED,GPIO25 +LED_RGB,GPIO25 +RGB_LED,GPIO25 +NEOPIXEL,GPIO25 diff --git a/ports/rp2/boards/SPARKFUN_IOTNODE_LORAWAN_RP2350/sparkfun_iotnode_lorawan_rp2350.h b/ports/rp2/boards/SPARKFUN_IOTNODE_LORAWAN_RP2350/sparkfun_iotnode_lorawan_rp2350.h new file mode 100644 index 0000000000000..5bea33cc4c219 --- /dev/null +++ b/ports/rp2/boards/SPARKFUN_IOTNODE_LORAWAN_RP2350/sparkfun_iotnode_lorawan_rp2350.h @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2024 Raspberry Pi (Trading) Ltd. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +// ----------------------------------------------------- +// NOTE: THIS HEADER IS ALSO INCLUDED BY ASSEMBLER SO +// SHOULD ONLY CONSIST OF PREPROCESSOR DIRECTIVES +// ----------------------------------------------------- +// Board definition for the SparkFun IoT Node LoRaWAN +// +// This header may be included by other board headers as "boards/sparkfun_iotnode_lorawan_rp2350.h" + +// pico_cmake_set PICO_PLATFORM=rp2350 + +#ifndef _BOARDS_SPARKFUN_IOTNODE_LORAWAN_RP2350_H +#define _BOARDS_SPARKFUN_IOTNODE_LORAWAN_RP2350_H + +// For board detection +#define SPARKFUN_IOTNODE_LORAWAN_RP2350 + +// --- RP2350 VARIANT --- +#define PICO_RP2350A 1 + +// --- UART --- +#ifndef PICO_DEFAULT_UART +#define PICO_DEFAULT_UART 0 +#endif +#ifndef PICO_DEFAULT_UART_TX_PIN +#define PICO_DEFAULT_UART_TX_PIN 18 +#endif +#ifndef PICO_DEFAULT_UART_RX_PIN +#define PICO_DEFAULT_UART_RX_PIN 19 +#endif + +// --- LED --- +#ifndef PICO_DEFAULT_WS2812_PIN +#define PICO_DEFAULT_WS2812_PIN 25 +#endif + +// --- I2C --- +#ifndef PICO_DEFAULT_I2C +#define PICO_DEFAULT_I2C 0 +#endif +#ifndef PICO_DEFAULT_I2C_SDA_PIN +#define PICO_DEFAULT_I2C_SDA_PIN 20 +#endif +#ifndef PICO_DEFAULT_I2C_SCL_PIN +#define PICO_DEFAULT_I2C_SCL_PIN 21 +#endif + +// --- SPI --- +#ifndef PICO_DEFAULT_SPI +#define PICO_DEFAULT_SPI 1 +#endif +#ifndef PICO_DEFAULT_SPI_SCK_PIN +#define PICO_DEFAULT_SPI_SCK_PIN 14 +#endif +#ifndef PICO_DEFAULT_SPI_TX_PIN +#define PICO_DEFAULT_SPI_TX_PIN 15 +#endif +#ifndef PICO_DEFAULT_SPI_RX_PIN +#define PICO_DEFAULT_SPI_RX_PIN 12 +#endif +#ifndef PICO_DEFAULT_SPI_CSN_PIN +#define PICO_DEFAULT_SPI_CSN_PIN 13 +#endif + +// --- FLASH --- + +#define PICO_BOOT_STAGE2_CHOOSE_W25Q080 1 + +#ifndef PICO_FLASH_SPI_CLKDIV +#define PICO_FLASH_SPI_CLKDIV 2 +#endif + +// pico_cmake_set_default PICO_FLASH_SIZE_BYTES = (16 * 1024 * 1024) +#ifndef PICO_FLASH_SIZE_BYTES +#define PICO_FLASH_SIZE_BYTES (16 * 1024 * 1024) +#endif + +// pico_cmake_set_default PICO_RP2350_A2_SUPPORTED = 1 +#ifndef PICO_RP2350_A2_SUPPORTED +#define PICO_RP2350_A2_SUPPORTED 1 +#endif + +#endif From e574f6882082058b868b2f81955d8204fa1c852d Mon Sep 17 00:00:00 2001 From: robert-hh Date: Thu, 3 Oct 2024 09:07:48 +0200 Subject: [PATCH 055/110] mimxrt: Add support for a UF2 bootloader. Allowing to use e.g. the Adafruit bootloaders with MicroPython. The .uf2 file is created in addition to the .bin and .hex files allowing to use the latter ones without the bootloader for debugging and testing. Changes: - Set the location of the ISR Vector and .text segment to 0x6000C000 and 0x6000C400. - Reserve an area at the start of ITCM for a copy of the interrupt vector table and copy the table on reset to this place. - Extend `machine.bootloader()` by setting the magic number to enable the bootloader on reset. - Create a .uf2 file which skips the segments below 0x6000C000. The bootloader has to be installed as a preparation step using the board specific methods, but then the firmware's .uf2 file version can be installed using the bootloader. The bootloader can be invoked with: - double reset - calling machine.bootloader() - Using the touch1200 method Double reset is hard to achieve on MIMXRT boards, since there is no clean reset pin. Some MIMXRT boards provide it by switching the power. Some boards are excluded from the .uf2 build: - MIMXRT1050_EVK: The uf2 bootloader is built for the QSPI version of the board. MicroPython supports the Hyperflash version. - MIMXRT1176_EVK: No support for this board yet, but it should be possible. Signed-off-by: robert-hh --- ports/mimxrt/Makefile | 21 +++++++++++++++++++ .../boards/ADAFRUIT_METRO_M7/mpconfigboard.mk | 2 ++ .../boards/MIMXRT1010_EVK/mpconfigboard.mk | 2 ++ ports/mimxrt/boards/MIMXRT1011.ld | 4 ++-- ports/mimxrt/boards/MIMXRT1015.ld | 4 ++-- .../boards/MIMXRT1015_EVK/mpconfigboard.mk | 2 ++ .../boards/MIMXRT1020_EVK/mpconfigboard.mk | 2 ++ ports/mimxrt/boards/MIMXRT1021.ld | 4 ++-- ports/mimxrt/boards/MIMXRT1052.ld | 4 ++-- .../boards/MIMXRT1060_EVK/mpconfigboard.mk | 2 ++ ports/mimxrt/boards/MIMXRT1062.ld | 4 ++-- ports/mimxrt/boards/MIMXRT1064.ld | 4 ++-- .../boards/MIMXRT1064_EVK/mpconfigboard.mk | 2 ++ .../boards/OLIMEX_RT1010/mpconfigboard.mk | 2 ++ .../boards/SEEED_ARCH_MIX/mpconfigboard.mk | 2 ++ ports/mimxrt/boards/TEENSY40/mpconfigboard.mk | 2 ++ ports/mimxrt/boards/TEENSY41/mpconfigboard.mk | 2 ++ ports/mimxrt/boards/common.ld | 11 +++++----- ports/mimxrt/hal/resethandler_MIMXRT10xx.S | 13 ++++++++++++ ports/mimxrt/modmachine.c | 14 +++++++++++-- ports/mimxrt/mpconfigport.h | 2 ++ tools/autobuild/build-boards.sh | 2 +- 22 files changed, 87 insertions(+), 20 deletions(-) diff --git a/ports/mimxrt/Makefile b/ports/mimxrt/Makefile index 3a9550cc974d8..ad7d853b677a5 100644 --- a/ports/mimxrt/Makefile +++ b/ports/mimxrt/Makefile @@ -21,6 +21,7 @@ BUILD ?= build-$(BOARD) PORT ?= /dev/ttyACM0 CROSS_COMPILE ?= arm-none-eabi- GIT_SUBMODULES += lib/tinyusb lib/nxp_driver +UF2CONV ?= $(TOP)/tools/uf2conv.py # MicroPython feature configurations MICROPY_VFS_LFS2 ?= 1 @@ -162,6 +163,13 @@ SRC_HAL_IMX_C += \ $(MCU_DIR)/drivers/fsl_romapi.c endif +# If not empty, then it is 10xx. +ifneq ($(findstring MIMXRT10, $(MCU_SERIES)),) +APPLICATION_ADDR := 0x6000C000 +else +APPLICATION_ADDR := 0x3000C000 +endif + ifeq ($(MCU_SERIES), MIMXRT1176) INC += -I$(TOP)/$(MCU_DIR)/drivers/cm7 @@ -253,6 +261,11 @@ else $(error Error: Unknown board flash type $(MICROPY_HW_FLASH_TYPE)) endif +# Set a flag if the UF2 bootloader is used +ifeq ($(USE_UF2_BOOTLOADER),1) + CFLAGS += -DMICROPY_MACHINE_UF2_BOOTLOADER=1 +endif + # Add sources for respective board flash type # Add hal/flexspi_nor_flash.c or hal/flashspi_hyper_flash.c respectively SRC_HAL_C += hal/flexspi_$(subst qspi_,,$(FLEXSPI_FLASH_TYPE)).c @@ -470,7 +483,11 @@ $(BUILD)/lib/tinyusb/src/device/usbd.o: CFLAGS += -Wno-missing-braces # Build targets # ============================================================================= +ifeq ($(USE_UF2_BOOTLOADER),1) +all: $(BUILD)/firmware.hex $(BUILD)/firmware.bin $(BUILD)/firmware.uf2 +else all: $(BUILD)/firmware.hex $(BUILD)/firmware.bin +endif # Process linker scripts with C preprocessor to exchange LDDEFINES and # aggregate output of preprocessor in a single linker script `link.ld` @@ -487,6 +504,10 @@ $(BUILD)/firmware.bin: $(BUILD)/firmware.elf $(BUILD)/firmware.hex: $(BUILD)/firmware.elf $(Q)$(OBJCOPY) -O ihex -R .eeprom $< $@ +$(BUILD)/firmware.uf2: $(BUILD)/firmware.elf + $(Q)$(OBJCOPY) -O binary -R .stack -R .ivt -R .flash_config $^ $@-binpart + $(Q)$(PYTHON) $(UF2CONV) -b $(APPLICATION_ADDR) -f MIMXRT10XX -c -o $@ $@-binpart + # Making OBJ use an order-only dependency on the generated pins.h file # has the side effect of making the pins.h file before we actually compile # any of the objects. The normal dependency generation will deal with the diff --git a/ports/mimxrt/boards/ADAFRUIT_METRO_M7/mpconfigboard.mk b/ports/mimxrt/boards/ADAFRUIT_METRO_M7/mpconfigboard.mk index a8f7add6d26ae..74d3a6f0f10e3 100644 --- a/ports/mimxrt/boards/ADAFRUIT_METRO_M7/mpconfigboard.mk +++ b/ports/mimxrt/boards/ADAFRUIT_METRO_M7/mpconfigboard.mk @@ -8,3 +8,5 @@ MICROPY_HW_FLASH_SIZE ?= 0x800000 # 8MB MICROPY_PY_NETWORK_NINAW10 ?= 1 MICROPY_PY_SSL ?= 1 MICROPY_SSL_MBEDTLS ?= 1 + +USE_UF2_BOOTLOADER = 1 diff --git a/ports/mimxrt/boards/MIMXRT1010_EVK/mpconfigboard.mk b/ports/mimxrt/boards/MIMXRT1010_EVK/mpconfigboard.mk index dd525859063d9..1aef451e2a6a5 100644 --- a/ports/mimxrt/boards/MIMXRT1010_EVK/mpconfigboard.mk +++ b/ports/mimxrt/boards/MIMXRT1010_EVK/mpconfigboard.mk @@ -5,6 +5,8 @@ MICROPY_FLOAT_IMPL = single MICROPY_HW_FLASH_TYPE = qspi_nor_flash MICROPY_HW_FLASH_SIZE = 0x1000000 # 16MB +USE_UF2_BOOTLOADER = 1 + JLINK_PATH ?= /media/RT1010-EVK/ JLINK_COMMANDER_SCRIPT = $(BUILD)/script.jlink diff --git a/ports/mimxrt/boards/MIMXRT1011.ld b/ports/mimxrt/boards/MIMXRT1011.ld index 908eefffd643f..ab363bd56a025 100644 --- a/ports/mimxrt/boards/MIMXRT1011.ld +++ b/ports/mimxrt/boards/MIMXRT1011.ld @@ -14,9 +14,9 @@ flash_config_start = flash_start + 0x00000400; flash_config_size = 0x00000C00; ivt_start = flash_start + 0x00001000; ivt_size = 0x00001000; -interrupts_start = flash_start + 0x00002000; +interrupts_start = flash_start + 0x0000C000; interrupts_size = 0x00000400; -text_start = flash_start + 0x00002400; +text_start = flash_start + 0x0000C400; vfs_start = flash_start + 0x00100000; text_size = ((vfs_start) - (text_start)); vfs_size = ((flash_end) - (vfs_start)); diff --git a/ports/mimxrt/boards/MIMXRT1015.ld b/ports/mimxrt/boards/MIMXRT1015.ld index 90336a2437127..0237d348c2498 100644 --- a/ports/mimxrt/boards/MIMXRT1015.ld +++ b/ports/mimxrt/boards/MIMXRT1015.ld @@ -14,9 +14,9 @@ flash_config_start = flash_start; flash_config_size = 0x00001000; ivt_start = flash_start + 0x00001000; ivt_size = 0x00001000; -interrupts_start = flash_start + 0x00002000; +interrupts_start = flash_start + 0x0000C000; interrupts_size = 0x00000400; -text_start = flash_start + 0x00002400; +text_start = flash_start + 0x0000C400; vfs_start = flash_start + 0x00100000; text_size = ((vfs_start) - (text_start)); vfs_size = ((flash_end) - (vfs_start)); diff --git a/ports/mimxrt/boards/MIMXRT1015_EVK/mpconfigboard.mk b/ports/mimxrt/boards/MIMXRT1015_EVK/mpconfigboard.mk index 34e5cdee51134..0f719ac0a9753 100644 --- a/ports/mimxrt/boards/MIMXRT1015_EVK/mpconfigboard.mk +++ b/ports/mimxrt/boards/MIMXRT1015_EVK/mpconfigboard.mk @@ -6,3 +6,5 @@ MICROPY_HW_FLASH_TYPE = qspi_nor_flash MICROPY_HW_FLASH_SIZE = 0x1000000 # 16MB MICROPY_BOOT_BUFFER_SIZE = (32 * 1024) + +USE_UF2_BOOTLOADER = 1 diff --git a/ports/mimxrt/boards/MIMXRT1020_EVK/mpconfigboard.mk b/ports/mimxrt/boards/MIMXRT1020_EVK/mpconfigboard.mk index c98843a1a3c3d..7715e669f8d09 100644 --- a/ports/mimxrt/boards/MIMXRT1020_EVK/mpconfigboard.mk +++ b/ports/mimxrt/boards/MIMXRT1020_EVK/mpconfigboard.mk @@ -12,6 +12,8 @@ MICROPY_PY_LWIP = 1 MICROPY_PY_SSL = 1 MICROPY_SSL_MBEDTLS = 1 +USE_UF2_BOOTLOADER = 1 + FROZEN_MANIFEST ?= $(BOARD_DIR)/manifest.py JLINK_PATH ?= /media/RT1020-EVK/ diff --git a/ports/mimxrt/boards/MIMXRT1021.ld b/ports/mimxrt/boards/MIMXRT1021.ld index bef0c13df5505..78add04c0c26c 100644 --- a/ports/mimxrt/boards/MIMXRT1021.ld +++ b/ports/mimxrt/boards/MIMXRT1021.ld @@ -14,9 +14,9 @@ flash_config_start = flash_start; flash_config_size = 0x00001000; ivt_start = flash_start + 0x00001000; ivt_size = 0x00001000; -interrupts_start = flash_start + 0x00002000; +interrupts_start = flash_start + 0x0000C000; interrupts_size = 0x00000400; -text_start = flash_start + 0x00002400; +text_start = flash_start + 0x0000C400; vfs_start = flash_start + 0x00100000; text_size = ((vfs_start) - (text_start)); vfs_size = ((flash_end) - (vfs_start)); diff --git a/ports/mimxrt/boards/MIMXRT1052.ld b/ports/mimxrt/boards/MIMXRT1052.ld index ca656711a5a77..ea034d713e2f6 100644 --- a/ports/mimxrt/boards/MIMXRT1052.ld +++ b/ports/mimxrt/boards/MIMXRT1052.ld @@ -16,9 +16,9 @@ flash_config_start = flash_start; flash_config_size = 0x00001000; ivt_start = flash_start + 0x00001000; ivt_size = 0x00001000; -interrupts_start = flash_start + 0x00002000; +interrupts_start = flash_start + 0x0000C000; interrupts_size = 0x00000400; -text_start = flash_start + 0x00002400; +text_start = flash_start + 0x0000C400; vfs_start = flash_start + 0x00200000; text_size = ((vfs_start) - (text_start)); vfs_size = ((flash_end) - (vfs_start)); diff --git a/ports/mimxrt/boards/MIMXRT1060_EVK/mpconfigboard.mk b/ports/mimxrt/boards/MIMXRT1060_EVK/mpconfigboard.mk index 3af7cd231a2ab..f57aaff3b8b17 100644 --- a/ports/mimxrt/boards/MIMXRT1060_EVK/mpconfigboard.mk +++ b/ports/mimxrt/boards/MIMXRT1060_EVK/mpconfigboard.mk @@ -12,6 +12,8 @@ MICROPY_PY_LWIP = 1 MICROPY_PY_SSL = 1 MICROPY_SSL_MBEDTLS = 1 +USE_UF2_BOOTLOADER = 1 + FROZEN_MANIFEST ?= $(BOARD_DIR)/manifest.py JLINK_PATH ?= /media/RT1060-EVK/ diff --git a/ports/mimxrt/boards/MIMXRT1062.ld b/ports/mimxrt/boards/MIMXRT1062.ld index 5b91550d97d60..3d7e6d0634196 100644 --- a/ports/mimxrt/boards/MIMXRT1062.ld +++ b/ports/mimxrt/boards/MIMXRT1062.ld @@ -16,9 +16,9 @@ flash_config_start = flash_start; flash_config_size = 0x00001000; ivt_start = flash_start + 0x00001000; ivt_size = 0x00001000; -interrupts_start = flash_start + 0x00002000; +interrupts_start = flash_start + 0x0000C000; interrupts_size = 0x00000400; -text_start = flash_start + 0x00002400; +text_start = flash_start + 0x0000C400; vfs_start = flash_start + 0x00100000; text_size = ((vfs_start) - (text_start)); vfs_size = ((flash_end) - (vfs_start)); diff --git a/ports/mimxrt/boards/MIMXRT1064.ld b/ports/mimxrt/boards/MIMXRT1064.ld index 708dac4d515c4..7c35cb60c750b 100644 --- a/ports/mimxrt/boards/MIMXRT1064.ld +++ b/ports/mimxrt/boards/MIMXRT1064.ld @@ -10,9 +10,9 @@ flash_config_start = flash_start; flash_config_size = 0x00001000; ivt_start = flash_start + 0x00001000; ivt_size = 0x00001000; -interrupts_start = flash_start + 0x00002000; +interrupts_start = flash_start + 0x0000C000; interrupts_size = 0x00000400; -text_start = flash_start + 0x00002400; +text_start = flash_start + 0x0000C400; vfs_start = flash_start + 0x00100000; text_size = ((vfs_start) - (text_start)); vfs_size = ((flash_end) - (vfs_start)); diff --git a/ports/mimxrt/boards/MIMXRT1064_EVK/mpconfigboard.mk b/ports/mimxrt/boards/MIMXRT1064_EVK/mpconfigboard.mk index 95cfb4585acc1..f175d26377abc 100644 --- a/ports/mimxrt/boards/MIMXRT1064_EVK/mpconfigboard.mk +++ b/ports/mimxrt/boards/MIMXRT1064_EVK/mpconfigboard.mk @@ -12,6 +12,8 @@ MICROPY_PY_LWIP = 1 MICROPY_PY_SSL = 1 MICROPY_SSL_MBEDTLS = 1 +USE_UF2_BOOTLOADER = 1 + FROZEN_MANIFEST ?= $(BOARD_DIR)/manifest.py JLINK_PATH ?= /media/RT1064-EVK/ diff --git a/ports/mimxrt/boards/OLIMEX_RT1010/mpconfigboard.mk b/ports/mimxrt/boards/OLIMEX_RT1010/mpconfigboard.mk index 58429d298118e..1de497d74e3f0 100644 --- a/ports/mimxrt/boards/OLIMEX_RT1010/mpconfigboard.mk +++ b/ports/mimxrt/boards/OLIMEX_RT1010/mpconfigboard.mk @@ -6,6 +6,8 @@ MICROPY_HW_FLASH_TYPE = qspi_nor_flash MICROPY_HW_FLASH_SIZE = 0x200000 # 2MB MICROPY_HW_FLASH_RESERVED ?= 0x1000 # 4KB +USE_UF2_BOOTLOADER = 1 + CFLAGS += -DMICROPY_HW_FLASH_DQS=kFlexSPIReadSampleClk_LoopbackInternally SRC_C += \ diff --git a/ports/mimxrt/boards/SEEED_ARCH_MIX/mpconfigboard.mk b/ports/mimxrt/boards/SEEED_ARCH_MIX/mpconfigboard.mk index 63e68e1e57994..c691e2dcb55c4 100644 --- a/ports/mimxrt/boards/SEEED_ARCH_MIX/mpconfigboard.mk +++ b/ports/mimxrt/boards/SEEED_ARCH_MIX/mpconfigboard.mk @@ -12,6 +12,8 @@ MICROPY_PY_LWIP = 1 MICROPY_PY_SSL = 1 MICROPY_SSL_MBEDTLS = 1 +USE_UF2_BOOTLOADER = 1 + FROZEN_MANIFEST ?= $(BOARD_DIR)/manifest.py CFLAGS += -DSPI_RETRY_TIMES=1000000 diff --git a/ports/mimxrt/boards/TEENSY40/mpconfigboard.mk b/ports/mimxrt/boards/TEENSY40/mpconfigboard.mk index 07f174944b79c..87c122da9d49d 100644 --- a/ports/mimxrt/boards/TEENSY40/mpconfigboard.mk +++ b/ports/mimxrt/boards/TEENSY40/mpconfigboard.mk @@ -6,5 +6,7 @@ MICROPY_HW_FLASH_TYPE = qspi_nor_flash MICROPY_HW_FLASH_SIZE = 0x200000 # 2MB MICROPY_HW_FLASH_RESERVED ?= 0x1000 # 4KB +USE_UF2_BOOTLOADER = 1 + deploy: $(BUILD)/firmware.hex teensy_loader_cli --mcu=imxrt1062 -v -w $< diff --git a/ports/mimxrt/boards/TEENSY41/mpconfigboard.mk b/ports/mimxrt/boards/TEENSY41/mpconfigboard.mk index b297448a307e7..e28310d091ad2 100755 --- a/ports/mimxrt/boards/TEENSY41/mpconfigboard.mk +++ b/ports/mimxrt/boards/TEENSY41/mpconfigboard.mk @@ -10,6 +10,8 @@ MICROPY_PY_LWIP = 1 MICROPY_PY_SSL = 1 MICROPY_SSL_MBEDTLS = 1 +USE_UF2_BOOTLOADER = 1 + FROZEN_MANIFEST ?= $(BOARD_DIR)/manifest.py deploy: $(BUILD)/firmware.hex diff --git a/ports/mimxrt/boards/common.ld b/ports/mimxrt/boards/common.ld index 587fc82a1953d..477ba38bc89a3 100644 --- a/ports/mimxrt/boards/common.ld +++ b/ports/mimxrt/boards/common.ld @@ -42,7 +42,8 @@ MEMORY m_vfs (RX) : ORIGIN = vfs_start, LENGTH = vfs_size /* Teensy uses the last bit of flash for recovery. */ m_reserved (RX) : ORIGIN = (vfs_start + vfs_size), LENGTH = reserved_size - m_itcm (RX) : ORIGIN = itcm_start, LENGTH = itcm_size + m_isr (RX) : ORIGIN = itcm_start, LENGTH = 0x400 + m_itcm (RX) : ORIGIN = itcm_start + 0x400, LENGTH = itcm_size - 0x400 m_dtcm (RW) : ORIGIN = dtcm_start, LENGTH = dtcm_size m_ocrm (RW) : ORIGIN = ocrm_start, LENGTH = ocrm_size @@ -80,7 +81,8 @@ SECTIONS . = ALIGN(4); } > m_ivt - /* The startup code goes first into internal RAM */ + /* ISR Vector table in flash. Copied to ITCM by ResetHandler(). */ + .interrupts : { __Vectors = .; @@ -90,10 +92,9 @@ SECTIONS . = ALIGN(4); } > m_interrupts - __VECTOR_RAM = __Vectors; - __RAM_VECTOR_TABLE_SIZE_BYTES = 0x0; + __Vectors_RAM = ORIGIN(m_isr); - /* The program code and other data goes into internal RAM */ + /* Some program code and other data goes into internal RAM */ .text : { . = ALIGN(4); diff --git a/ports/mimxrt/hal/resethandler_MIMXRT10xx.S b/ports/mimxrt/hal/resethandler_MIMXRT10xx.S index fe933c6d6002b..2fb8772850b6a 100644 --- a/ports/mimxrt/hal/resethandler_MIMXRT10xx.S +++ b/ports/mimxrt/hal/resethandler_MIMXRT10xx.S @@ -64,6 +64,19 @@ Reset_Handler: * __ram_function_start__/__ram_function_end__ : ramfunction region * copied to. Both must be aligned to 4 bytes boundary. */ +/* Copy the ISR Vector table to the start of ITCM to be available when the + .uf2 bootloader is used */ + + ldr r1, = __Vectors + ldr r2, = __Vectors_RAM + movs r3, 1024 + +.LC_ISR: + subs r3, #4 + ldr r0, [r1, r3] + str r0, [r2, r3] + bgt .LC_ISR + ldr r1, =__etext ldr r2, =__data_start__ ldr r3, =__data_end__ diff --git a/ports/mimxrt/modmachine.c b/ports/mimxrt/modmachine.c index 1212914d3b452..204699bcc4263 100644 --- a/ports/mimxrt/modmachine.c +++ b/ports/mimxrt/modmachine.c @@ -76,6 +76,11 @@ typedef enum { MP_SOFT_RESET } reset_reason_t; +// Copied from inc/uf2.h in https://github.com/Microsoft/uf2-samd21 +#define DBL_TAP_REG SNVS->LPGPR[3] +#define DBL_TAP_MAGIC 0xf01669ef // Randomly selected, adjusted to have first and last bit set +#define DBL_TAP_MAGIC_QUICK_BOOT 0xf02669ef + static mp_obj_t mp_machine_unique_id(void) { unsigned char id[8]; mp_hal_get_unique_id(id); @@ -158,12 +163,17 @@ NORETURN void mp_machine_bootloader(size_t n_args, const mp_obj_t *args) { #if defined(MICROPY_BOARD_ENTER_BOOTLOADER) // If a board has a custom bootloader, call it first. MICROPY_BOARD_ENTER_BOOTLOADER(n_args, args); - #elif FSL_ROM_HAS_RUNBOOTLOADER_API + #elif FSL_ROM_HAS_RUNBOOTLOADER_API && !MICROPY_MACHINE_UF2_BOOTLOADER // If not, enter ROM bootloader in serial downloader / USB mode. + // Skip that in case of the UF2 bootloader being available. uint32_t arg = 0xEB110000; ROM_RunBootloader(&arg); #else - // No custom bootloader, or run bootloader API, then just reset. + // No custom bootloader, or run bootloader API, the set + // the flag for the UF2 bootloader + // Pretend to be the first of the two reset presses needed to enter the + // bootloader. That way one reset will end in the bootloader. + DBL_TAP_REG = DBL_TAP_MAGIC; WDOG_TriggerSystemSoftwareReset(WDOG1); #endif while (1) { diff --git a/ports/mimxrt/mpconfigport.h b/ports/mimxrt/mpconfigport.h index 923eff4dc7064..b45f8408e6415 100644 --- a/ports/mimxrt/mpconfigport.h +++ b/ports/mimxrt/mpconfigport.h @@ -115,6 +115,7 @@ uint32_t trng_random_u32(void); #define MICROPY_PY_MACHINE_UART_SENDBREAK (1) #define MICROPY_PY_MACHINE_UART_IRQ (1) #define MICROPY_PY_ONEWIRE (1) +#define MICROPY_PY_MACHINE_BOOTLOADER (1) // fatfs configuration used in ffconf.h #define MICROPY_FATFS_ENABLE_LFN (2) @@ -151,6 +152,7 @@ uint32_t trng_random_u32(void); #define MICROPY_HW_ENABLE_USBDEV (1) #define MICROPY_HW_USB_CDC (1) +#define MICROPY_HW_USB_CDC_1200BPS_TOUCH (1) // Hooks to add builtins diff --git a/tools/autobuild/build-boards.sh b/tools/autobuild/build-boards.sh index 801e7062e5798..bd6828cb40f50 100755 --- a/tools/autobuild/build-boards.sh +++ b/tools/autobuild/build-boards.sh @@ -99,7 +99,7 @@ function build_esp8266_boards { } function build_mimxrt_boards { - build_boards modmimxrt.c $1 $2 bin hex + build_boards modmimxrt.c $1 $2 bin hex uf2 } function build_nrf_boards { From b251aec0fcfef75b43b69c329b9a34c3703a6b90 Mon Sep 17 00:00:00 2001 From: robert-hh Date: Sat, 5 Oct 2024 12:57:47 +0200 Subject: [PATCH 056/110] mimxrt/hal: Update the LUT and re-enable PAGEPROGRAM_QUAD. Changes: - Change the LUT table ordering to be similar to the order of the UF2-Bootloader and fsl_romapi.h. - Rewrite the LUT entry for PAGEPROGRAM_QUAD and update the LUT. That enabled QUAD program again. Signed-off-by: robert-hh --- ports/mimxrt/flash.c | 6 +-- ports/mimxrt/hal/flexspi_flash_config.h | 24 ++++++---- ports/mimxrt/hal/flexspi_hyper_flash.c | 3 ++ ports/mimxrt/hal/flexspi_nor_flash.c | 40 ++++++++-------- ports/mimxrt/hal/qspi_nor_flash_config.c | 60 +++++++++++------------- 5 files changed, 67 insertions(+), 66 deletions(-) diff --git a/ports/mimxrt/flash.c b/ports/mimxrt/flash.c index 3a18f8f51b930..7df799884e9e7 100644 --- a/ports/mimxrt/flash.c +++ b/ports/mimxrt/flash.c @@ -28,10 +28,8 @@ void flash_init(void) { // Upload the custom flash configuration - // This should be performed by the boot ROM but for some reason it is not. - FLEXSPI_UpdateLUT(BOARD_FLEX_SPI, 0, - qspiflash_config.memConfig.lookupTable, - ARRAY_SIZE(qspiflash_config.memConfig.lookupTable)); + // And fix the entry for PAGEPROGRAM_QUAD + flexspi_nor_update_lut(); // Configure FLEXSPI IP FIFO access. BOARD_FLEX_SPI->MCR0 &= ~(FLEXSPI_MCR0_ARDFEN_MASK); diff --git a/ports/mimxrt/hal/flexspi_flash_config.h b/ports/mimxrt/hal/flexspi_flash_config.h index 5320231a1495c..7538537aeae5b 100644 --- a/ports/mimxrt/hal/flexspi_flash_config.h +++ b/ports/mimxrt/hal/flexspi_flash_config.h @@ -75,6 +75,12 @@ (FLEXSPI_LUT_OPERAND0(op0) | FLEXSPI_LUT_NUM_PADS0(pad0) | FLEXSPI_LUT_OPCODE0(cmd0) | FLEXSPI_LUT_OPERAND1(op1) | \ FLEXSPI_LUT_NUM_PADS1(pad1) | FLEXSPI_LUT_OPCODE1(cmd1)) +#define EMPTY_LUT \ + FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), \ + FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), \ + FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), \ + FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), \ + // !@brief Definitions for FlexSPI Serial Clock Frequency typedef enum _FlexSpiSerialClockFreq { @@ -209,20 +215,20 @@ typedef struct _FlexSPIConfig } flexspi_mem_config_t; /* */ -#define NOR_CMD_LUT_SEQ_IDX_READ_NORMAL 0 +#define NOR_CMD_LUT_SEQ_IDX_READ 0 #define NOR_CMD_LUT_SEQ_IDX_READSTATUSREG 1 -#define NOR_CMD_LUT_SEQ_IDX_READ_FAST_QUAD 2 +#define NOR_CMD_LUT_SEQ_IDX_READSTATUS_XPI 2 #define NOR_CMD_LUT_SEQ_IDX_WRITEENABLE 3 -#define NOR_CMD_LUT_SEQ_IDX_READSTATUS_XPI 4 +#define NOR_CMD_LUT_SEQ_IDX_WRITESTATUSREG 4 #define NOR_CMD_LUT_SEQ_IDX_ERASESECTOR 5 -#define NOR_CMD_LUT_SEQ_IDX_WRITESTATUSREG 6 -#define NOR_CMD_LUT_SEQ_IDX_PAGEPROGRAM_QUAD 7 -#define NOR_CMD_LUT_SEQ_IDX_READID 8 +#define NOR_CMD_LUT_SEQ_IDX_READQUAD 6 +#define NOR_CMD_LUT_SEQ_IDX_READID 7 +#define NOR_CMD_LUT_SEQ_IDX_ERASEBLOCK 8 #define NOR_CMD_LUT_SEQ_IDX_PAGEPROGRAM 9 -#define NOR_CMD_LUT_SEQ_IDX_ENTERQPI 10 +#define NOR_CMD_LUT_SEQ_IDX_PAGEPROGRAM_QUAD 10 #define NOR_CMD_LUT_SEQ_IDX_CHIPERASE 11 -#define NOR_CMD_LUT_SEQ_IDX_EXITQPI 12 -#define NOR_CMD_LUT_SEQ_IDX_ERASEBLOCK 13 +// Index 12 is left empty +#define NOR_CMD_LUT_SEQ_IDX_READ_SFDP 13 #define HYPERFLASH_CMD_LUT_SEQ_IDX_READDATA 0 #define HYPERFLASH_CMD_LUT_SEQ_IDX_WRITEDATA 1 diff --git a/ports/mimxrt/hal/flexspi_hyper_flash.c b/ports/mimxrt/hal/flexspi_hyper_flash.c index 5e5d87166d850..c0cd63ca0be8e 100644 --- a/ports/mimxrt/hal/flexspi_hyper_flash.c +++ b/ports/mimxrt/hal/flexspi_hyper_flash.c @@ -9,6 +9,9 @@ #include "fsl_clock.h" #include "flexspi_hyper_flash.h" +void flexspi_nor_update_lut(void) { +} + // Copy of a few (pseudo-)functions from fsl_clock.h, which were nor reliably // inlined when they should be. That caused DEBUG mode to fail. // It does not increase the code size, since they were supposed to be inline. diff --git a/ports/mimxrt/hal/flexspi_nor_flash.c b/ports/mimxrt/hal/flexspi_nor_flash.c index 956fb657db6dd..a890533a69661 100644 --- a/ports/mimxrt/hal/flexspi_nor_flash.c +++ b/ports/mimxrt/hal/flexspi_nor_flash.c @@ -36,6 +36,23 @@ #include #include "fsl_common.h" #include "flexspi_nor_flash.h" +#include "flexspi_flash_config.h" + +uint32_t LUT_pageprogram_quad[4] = { + // 10 Page Program - quad mode + FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x32, RADDR_SDR, FLEXSPI_1PAD, 24), + FLEXSPI_LUT_SEQ(WRITE_SDR, FLEXSPI_4PAD, 0x04, STOP, FLEXSPI_1PAD, 0), + FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler + FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler +}; + +void flexspi_nor_update_lut(void) { + uint32_t lookuptable_copy[64]; + memcpy(lookuptable_copy, (const uint32_t *)&qspiflash_config.memConfig.lookupTable, 64 * sizeof(uint32_t)); + // write PAGEPROGRAM_QUAD code to entry 10 + memcpy(&lookuptable_copy[10 * 4], LUT_pageprogram_quad, 4 * sizeof(uint32_t)); + FLEXSPI_UpdateLUT(BOARD_FLEX_SPI, 0, lookuptable_copy, 64); +} void flexspi_nor_reset(FLEXSPI_Type *base) __attribute__((section(".ram_functions"))); void flexspi_nor_reset(FLEXSPI_Type *base) { @@ -106,9 +123,9 @@ status_t flexspi_nor_enable_quad_mode(FLEXSPI_Type *base) __attribute__((section status_t flexspi_nor_enable_quad_mode(FLEXSPI_Type *base) { flexspi_transfer_t flashXfer; status_t status; - uint32_t writeValue = 0x40; + uint32_t writeValue = qspiflash_config.memConfig.deviceModeArg; - /* Write neable */ + /* Write enable */ status = flexspi_nor_write_enable(base, 0); if (status != kStatus_Success) { @@ -228,22 +245,3 @@ status_t flexspi_nor_flash_page_program(FLEXSPI_Type *base, uint32_t dstAddr, co return status; } - -status_t flexspi_nor_get_vendor_id(FLEXSPI_Type *base, uint8_t *vendorId) __attribute__((section(".ram_functions"))); -status_t flexspi_nor_get_vendor_id(FLEXSPI_Type *base, uint8_t *vendorId) { - uint32_t temp; - flexspi_transfer_t flashXfer; - flashXfer.deviceAddress = 0; - flashXfer.port = kFLEXSPI_PortA1; - flashXfer.cmdType = kFLEXSPI_Read; - flashXfer.SeqNumber = 1; - flashXfer.seqIndex = NOR_CMD_LUT_SEQ_IDX_READID; - flashXfer.data = &temp; - flashXfer.dataSize = 2; - - status_t status = FLEXSPI_TransferBlocking(base, &flashXfer); - - *vendorId = temp; - - return status; -} diff --git a/ports/mimxrt/hal/qspi_nor_flash_config.c b/ports/mimxrt/hal/qspi_nor_flash_config.c index a8040737e51b0..9edf44fc64c84 100644 --- a/ports/mimxrt/hal/qspi_nor_flash_config.c +++ b/ports/mimxrt/hal/qspi_nor_flash_config.c @@ -49,76 +49,75 @@ const flexspi_nor_config_t qspiflash_config = { .seqNum = 1u, }, .deviceModeArg = 0x40, - // Enable DDR mode, Wordaddassable, Safe configuration, Differential clock .deviceType = kFlexSpiDeviceType_SerialNOR, .sflashPadType = kSerialFlash_4Pads, .serialClkFreq = MICROPY_HW_FLASH_CLK, .sflashA1Size = MICROPY_HW_FLASH_SIZE, .lookupTable = { - // 0 Read LUTs 0 -> 0 - FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0xEB, RADDR_SDR, FLEXSPI_4PAD, 0x18), + // 0 Read LUTs 0 + FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0xEB, RADDR_SDR, FLEXSPI_4PAD, 24), FLEXSPI_LUT_SEQ(DUMMY_SDR, FLEXSPI_4PAD, 0x06, READ_SDR, FLEXSPI_4PAD, 0x04), FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler - // 1 Read status register -> 1 + // 1 Read status register FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x05, READ_SDR, FLEXSPI_1PAD, 0x01), FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler - // 2 Fast read quad mode - SDR - FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x6B, RADDR_SDR, FLEXSPI_1PAD, 0x18), - FLEXSPI_LUT_SEQ(DUMMY_SDR, FLEXSPI_4PAD, 0x08, READ_SDR, FLEXSPI_4PAD, 0x04), + // 2 Read extend parameters + FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x81, READ_SDR, FLEXSPI_1PAD, 0x04), + FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler - // 3 Write Enable -> 3 + // 3 Write Enable FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x06, STOP, FLEXSPI_1PAD, 0), FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler - // 4 Read extend parameters - FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x81, READ_SDR, FLEXSPI_1PAD, 0x04), + // 4 Write Status Reg + FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x01, WRITE_SDR, FLEXSPI_1PAD, 0x01), FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler - // 5 Erase Sector -> 5 + // 5 Erase Sector FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x20, RADDR_SDR, FLEXSPI_1PAD, 24), FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler - // 6 Write Status Reg - FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x01, WRITE_SDR, FLEXSPI_1PAD, 0x04), - FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler + // 6 Fast read quad mode - SDR + FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x6B, RADDR_SDR, FLEXSPI_1PAD, 24), + FLEXSPI_LUT_SEQ(DUMMY_SDR, FLEXSPI_4PAD, 0x08, READ_SDR, FLEXSPI_4PAD, 0x04), FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler - // 7 Page Program - quad mode (-> 9) - FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x32, RADDR_SDR, FLEXSPI_1PAD, 0x18), - FLEXSPI_LUT_SEQ(WRITE_SDR, FLEXSPI_4PAD, 0x04, STOP, FLEXSPI_1PAD, 0), + // 7 Read ID + FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x90, DUMMY_SDR, FLEXSPI_1PAD, 24), + FLEXSPI_LUT_SEQ(READ_SDR, FLEXSPI_1PAD, 0x00, 0, 0, 0), FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler - // 8 Read ID - FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x90, DUMMY_SDR, FLEXSPI_1PAD, 24), - FLEXSPI_LUT_SEQ(READ_SDR, FLEXSPI_1PAD, 0x00, 0, 0, 0), + // 8 Erase Block (32k) + FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x52, RADDR_SDR, FLEXSPI_1PAD, 24), + FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler - // 9 Page Program - single mode -> 9 + // 9 Page Program - single mode FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x02, RADDR_SDR, FLEXSPI_1PAD, 24), FLEXSPI_LUT_SEQ(WRITE_SDR, FLEXSPI_1PAD, 0, 0, 0, 0), FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler - // 10 Enter QPI mode - FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x35, STOP, FLEXSPI_1PAD, 0), - FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler + // 10 Page Program - quad mode + FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x32, RADDR_SDR, FLEXSPI_1PAD, 24), + FLEXSPI_LUT_SEQ(WRITE_SDR, FLEXSPI_4PAD, 0x04, STOP, FLEXSPI_1PAD, 0), FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler @@ -128,15 +127,12 @@ const flexspi_nor_config_t qspiflash_config = { FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler - // 12 Exit QPI mode - FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_4PAD, 0xF5, STOP, FLEXSPI_1PAD, 0), - FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler - FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler - FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler + // 12 Empty LUT + EMPTY_LUT - // 13 Erase Block (32k) -> 13 - FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x52, RADDR_SDR, FLEXSPI_1PAD, 24), - FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler + // 13 READ SDFP + FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x5A, RADDR_SDR, FLEXSPI_1PAD, 24), + FLEXSPI_LUT_SEQ(DUMMY_SDR, FLEXSPI_1PAD, 8, READ_SDR, FLEXSPI_1PAD, 0x04), FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler }, From 2a80d5c68b5e03570784d18985d5388995ba0e40 Mon Sep 17 00:00:00 2001 From: robert-hh Date: Sun, 26 Jan 2025 21:39:58 +0100 Subject: [PATCH 057/110] mimxrt/flash: Swap the order of disabling IRQ and disabling the cache. This change stopped problems with USB IRQ happening during flash writes. Signed-off-by: robert-hh --- ports/mimxrt/flash.c | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/ports/mimxrt/flash.c b/ports/mimxrt/flash.c index 7df799884e9e7..cdcdc7483d833 100644 --- a/ports/mimxrt/flash.c +++ b/ports/mimxrt/flash.c @@ -43,14 +43,13 @@ void flash_init(void) { __attribute__((section(".ram_functions"))) status_t flash_erase_block(uint32_t erase_addr) { status_t status = kStatus_Fail; - SCB_CleanInvalidateDCache(); - SCB_DisableDCache(); __disable_irq(); + SCB_DisableDCache(); status = flexspi_nor_flash_erase_block(BOARD_FLEX_SPI, erase_addr); - __enable_irq(); SCB_EnableDCache(); + __enable_irq(); return status; } @@ -60,14 +59,13 @@ __attribute__((section(".ram_functions"))) status_t flash_erase_block(uint32_t e __attribute__((section(".ram_functions"))) status_t flash_erase_sector(uint32_t erase_addr) { status_t status = kStatus_Fail; - SCB_CleanInvalidateDCache(); - SCB_DisableDCache(); __disable_irq(); + SCB_DisableDCache(); status = flexspi_nor_flash_erase_sector(BOARD_FLEX_SPI, erase_addr); - __enable_irq(); SCB_EnableDCache(); + __enable_irq(); return status; } @@ -83,10 +81,6 @@ __attribute__((section(".ram_functions"))) status_t flash_write_block(uint32_t d if (length == 0) { status = kStatus_Success; // Nothing to do } else { - - SCB_CleanInvalidateDCache(); - SCB_DisableDCache(); - // write data in chunks not crossing a page boundary do { next_addr = dest_addr - (dest_addr % PAGE_SIZE_BYTES) + PAGE_SIZE_BYTES; // next page boundary @@ -96,7 +90,11 @@ __attribute__((section(".ram_functions"))) status_t flash_write_block(uint32_t d } __disable_irq(); + SCB_DisableDCache(); + status = flexspi_nor_flash_page_program(BOARD_FLEX_SPI, dest_addr, (uint32_t *)src, write_length); + + SCB_EnableDCache(); __enable_irq(); // Update remaining data length @@ -106,9 +104,6 @@ __attribute__((section(".ram_functions"))) status_t flash_write_block(uint32_t d src += write_length; dest_addr += write_length; } while ((length > 0) && (status == kStatus_Success)); - - SCB_EnableDCache(); - } return status; } From 9eb9451d8377c2adc5187cfb51ef68de843b680f Mon Sep 17 00:00:00 2001 From: robert-hh Date: Mon, 7 Oct 2024 08:45:54 +0200 Subject: [PATCH 058/110] mimxrt/boards: Update the deploy instructions for the UF2 bootloader. Signed-off-by: robert-hh --- .../ADAFRUIT_METRO_M7/deploy_metro_m7.md | 148 ++---------------- .../boards/OLIMEX_RT1010/deploy_olimex.md | 42 ++--- ports/mimxrt/boards/SEEED_ARCH_MIX/deploy.md | 58 ++++++- ports/mimxrt/boards/deploy_mimxrt.md | 44 +++++- 4 files changed, 123 insertions(+), 169 deletions(-) diff --git a/ports/mimxrt/boards/ADAFRUIT_METRO_M7/deploy_metro_m7.md b/ports/mimxrt/boards/ADAFRUIT_METRO_M7/deploy_metro_m7.md index bc92016b71751..c5bfd1a28a79f 100644 --- a/ports/mimxrt/boards/ADAFRUIT_METRO_M7/deploy_metro_m7.md +++ b/ports/mimxrt/boards/ADAFRUIT_METRO_M7/deploy_metro_m7.md @@ -1,141 +1,11 @@ ## 1. Deploy the MicroPython firmware to the Metro M7 board. -### 1.1 Deploy the firmware using the serial bootloader. - -For initial deployment of the firmware a few preparation steps are required, which -have to be done once. - -1. Get the files ufconv.py and uf2families.json from the micropython/tools directory, -e.g. at https://github.com/micropython/micropython/tree/master/tools. - -2. Get the NXP program sdphost for your operating system, e.g. from -https://github.com/adafruit/tinyuf2/tree/master/ports/mimxrt10xx/sdphost. -You can also get them from the NXP web sites. - -3. Get the UF2 boot-loader package https://github.com/adafruit/tinyuf2/releases/download/0.9.0/tinyuf2-imxrt1010_evk-0.9.0.zip -and extract the file tinyuf2-imxrt1010_evk-0.9.0.bin. - -Now you have all files at hand that you will need for updating. - -1. Get the firmware you want to upload from the MicroPython download page. - -2. Set the two BOOTSEL DIP switches to the 1/0 position, which is the opposite position of the normal use mode. - -3. Push the reset button. - -4. Run the commands: - -``` -sudo ./sdphost -u 0x1fc9,0x0145 -- write-file 0x20206400 tinyuf2-imxrt1010_evk-0.9.0.bin -sudo ./sdphost -u 0x1fc9,0x0145 -- jump-address 0x20207000 -``` -Wait until a drive icon appears on the computer (or mount it explicitly), and then run: -``` -python3 uf2conv.py --base 0x60000400 -f 0x4fb2d5bd -``` -You can put all of that in a script. Just add a short wait before the 3rd command to let the drive connect. - -5. Once the upload is finished, set the BOOTSEL DIP switches back to the 0/1 position and push reset. - -Using sudo is Linux specific. You may not need it at all, if the access rights are set properly, -and you will not need it for Windows. - -### 1.2 Deploy the firmware using a JTAG adapter. - -With a JTAG adapter the firmware can be easily installed. Appropriate tools are Segger JFlash Lite and -the Segger Edu Mini adapter. Just use the firmware.hex file for loading, which will be loaded at the -proper address. - - -## 2. Deploy the WiFi firmware. - -The NINA firmware in the NINA module has to be updated for use with MicroPython. That can be done -using MicroPython and two small Python scripts. - -The firmware binaries are available at -https://github.com/micropython/micropython-lib/tree/master/micropython/espflash -or https://github.com/robert-hh/Shared-Stuff. For the Metro M7 board, the -NINA_FW_v1.5.0_Airlift.bin file is needed. - -For firmware upload, the following connections to the WiFi module are required: - -- Pin Reset (as above) -- Pin GPIO0 -- UART RX -- UART TX - -The GPIO pins and UART device id varies between boards. At the Adafruit Metro M7 board, -the UART is UART(1), and the Pin names for reset and GPIO0 are ESP_RESET and ESP_GPIO0. -The firmware can be uploaded, using the espflash.py module, a short script -using espflash.py and mpremote. espflash.py is available at -https://github.com/micropython/micropython-lib/tree/master/micropython/espflash. -This place also holds the example script. - -``` -import espflash -from machine import Pin -from machine import UART -import sys -sys.path.append("/flash") - -reset = Pin("ESP_RESET", Pin.OUT) -gpio0 = Pin("ESP_GPIO0", Pin.OUT) -uart = UART(0, 115200, timeout=350) - -md5sum = b"b0b9ab23da820a469e597c41364acb3a" -path = "/remote/NINA_FW_v1.5.0_Airlift.bin" - -esp = espflash.ESPFlash(reset, gpio0, uart) -# Enter bootloader download mode, at 115200 -esp.bootloader() -# Can now change to higher/lower baud rate -esp.set_baudrate(921600) -# Must call this first before any flash functions. -esp.flash_attach() -# Read flash size -size = esp.flash_read_size() -# Configure flash parameters. -esp.flash_config(size) -# Write firmware image from internal storage. -esp.flash_write_file(path) -# Compares file and flash MD5 checksum. -esp.flash_verify_file(path, md5sum) -# Resets the ESP32 chip. -esp.reboot() -``` - -The script shows the set-up for the Metro M7 board. -The md5sum is the one of the WiFi firmware. It may change and -can be recalculated using e.g. the Linux `md5sum` command. It is used to -verify the firmware upload. To upload the firmware, place the firmware -and the above script (let's call it ninaflash.py) into the same directory -on your PC, and run the command: -``` -mpremote connect mount . run ninaflash.py -``` -After a while, the upload will start. A typical start sequence looks like: -``` -Local directory . is mounted at /remote -Failed to read response to command 8. -Failed to read response to command 8. -Changing baudrate => 921600 -Flash attached -Flash size 2.0 MBytes -Flash write size: 1310720 total_blocks: 320 block size: 4096 -Writing sequence number 0/320... -Writing sequence number 1/320... -Writing sequence number 2/320... -Writing sequence number 3/320... -Writing sequence number 4/320... -.... -.... -Writing sequence number 317/320... -Writing sequence number 318/320... -Writing sequence number 319/320... -Flash write finished -Flash verify: File MD5 b'b0b9ab23da820a469e597c41364acb3a' -Flash verify: Flash MD5 b'b0b9ab23da820a469e597c41364acb3a' -Firmware verified. -``` -The initial messages `Failed to read response to command 8.` -can be ignored. +The Metro M7 board comes pre-installed with a UF2 bootloader. It can +be started by pushing reset twice. Then the bootloader drive will +appear. If that does not happen or the bootloader was lost, you can +reinstall the bootloader using the instructions by Adafruit +here: https://learn.adafruit.com/adafruit-metro-m7-microsd/installing-the-bootloader + +Once the bootloader is installed and started, you can install MicroPython +by copying the .uf2 version of the firmware file to the bootloader +drive. When the firmware is installed, the drive will disappear. diff --git a/ports/mimxrt/boards/OLIMEX_RT1010/deploy_olimex.md b/ports/mimxrt/boards/OLIMEX_RT1010/deploy_olimex.md index 92f3df57c9b84..7abba784b940b 100644 --- a/ports/mimxrt/boards/OLIMEX_RT1010/deploy_olimex.md +++ b/ports/mimxrt/boards/OLIMEX_RT1010/deploy_olimex.md @@ -1,39 +1,43 @@ -For initial deployment of the firmware a few preparation steps are required, which -have to be done once. +The Olimex RT1011 board is delivered without firmware. The best option to +install MicroPython is installing a UF2 bootstrap loader first, which then can be +used to install and update MicroPython. The bootloader has to be installed +only once. -1. Get the files ufconv.py and uf2families.json from the micropython/tools directory, -e.g. at https://github.com/micropython/micropython/tree/master/tools. +For initial deployment of the bootloader a few preparation steps are required, which +have to be done once. -2. Get the NXP program sdphost for your operating system, e.g. from +1. Get the NXP program sdphost for your operating system, e.g. from https://github.com/adafruit/tinyuf2/tree/master/ports/mimxrt10xx/sdphost. You can also get them from the NXP web sites. -3. Get the UF2 boot-loader package https://github.com/adafruit/tinyuf2/releases/download/0.9.0/tinyuf2-imxrt1010_evk-0.9.0.zip -and extract the file tinyuf2-imxrt1010_evk-0.9.0.bin. +2. Get the UF2 boot-loader package https://github.com/adafruit/tinyuf2/releases/tag/0.21.0/tinyuf2-imxrt1010_evk-0.20.1.zip and extract the files `tinyuf2-imxrt1010_evk-0.21.0.bin` +. You may as well go for a newer release. Now you have all files at hand that you will need for updating. -1. Get the firmware you want to upload from the MicroPython download page. +1. Get the firmware file you want to upload with the .uf2 extension from the MicroPython download page. 2. Push and hold the "Boot" button, then press "Reset", and release both buttons. 3. Run the commands: ``` -sudo ./sdphost -u 0x1fc9,0x0145 -- write-file 0x20206400 tinyuf2-imxrt1010_evk-0.9.0.bin +sudo ./sdphost -u 0x1fc9,0x0145 -- write-file 0x20206400 tinyuf2-imxrt1010_evk-0.21.0.bin sudo ./sdphost -u 0x1fc9,0x0145 -- jump-address 0x20207000 ``` -Wait until a drive icon appears on the computer (or mount it explicitly), and then run: -``` -python3 uf2conv.py --base 0x60000400 -f 0x4fb2d5bd -``` -You can put all of that in a script. Just add a short wait before the 3rd command to let the drive connect. - -4. Once the upload is finished, push Reset again. +Wait until a drive icon appears on the computer (or mount it explicitly). Then the UF2 bootloader +is permanently installed. Using sudo is Linux specific. You may not need it at all, if the access rights are set properly, and you will not need it for Windows. -Once the generic boot-loader is available, this procedure is only required for the first -firmware load or in case the flash is corrupted and the existing firmware is not functioning -any more. +4. Once the upload of the bootloader is finished, push Reset twice. + +The bootloader should start and show a drive icon. Copy the .uf2 version of MicroPython +to this drive to install or update MicroPython. + +Once the UF2 bootloader is installed, only step 4 is required to deploy MicroPython. If +MicroPython is already installed, the bootloader can as well be invoked by calling +`machine.bootloader()`. + +If at any time the flash content is corrupted you can always start over from the beginning. diff --git a/ports/mimxrt/boards/SEEED_ARCH_MIX/deploy.md b/ports/mimxrt/boards/SEEED_ARCH_MIX/deploy.md index e91c48d46cafd..9178c717e36e6 100644 --- a/ports/mimxrt/boards/SEEED_ARCH_MIX/deploy.md +++ b/ports/mimxrt/boards/SEEED_ARCH_MIX/deploy.md @@ -1,6 +1,52 @@ -Firmware upload to the Seed ARCH MIX board can be done using the J-Link interface -For that, follow the instructions given by Seed in their Wiki at -https://wiki.seeedstudio.com/Arch_Mix/#flashing-arduino-bootloader-to-arch-mix. -You will need a J-Link debug probe and software. What has been tested was the -Segger JLlink edu or Segger JLink edu mini. As matching loader tool you can use -Segger JFlashLite. The target address for loading is 0x60000000. +The Seeed Arch Mix board is delivered without firmware. The best option to +install MicroPython is installing a UF2 bootstrap loader first, which then can be +used to install and update MicroPython. The bootloader has to be installed +only once. + +For initial deployment of the bootloader a few preparation steps are required, which +have to be done once. + +1. Get the NXP program sdphost for your operating system, e.g. from +https://github.com/adafruit/tinyuf2/tree/master/ports/mimxrt10xx/sdphost. +You can also get them from the NXP web sites. + +2. Get the UF2 boot-loader package https://github.com/adafruit/tinyuf2/releases/tag/0.20.1/tinyuf2-imxrt1050_evkb-0.21.0.zip and extract the files `tinyuf2-imxrt1050_evkb-0.21.0.bin`. +You may as well go for a newer release. + +Now you have all files at hand that you will need for updating. + +1. Get the firmware file you want to upload with the .uf2 extension from the MicroPython download page. + +2. At the Seeed Arch Mix board, connect the RX pin (J3-19) with 3.3V, and change the DIP switches +3 an 4 at SW1 from 1-0 to 0-1. + +3. Push Reset. + +4. Run the commands: + +``` +sudo ./sdphost -u 0x1fc9,0x0130 -- write-file 0x1000 tinyuf2-imxrt1050_evkb-0.21.0.bin +sudo ./sdphost -u 0x1fc9,0x0130 -- jump-address 0x2000 +``` +Wait until a drive icon appears on the computer (or mount it explicitly). When using the above +mentioned bootloader, it has the label `RT1050BOOT`. Then the UF2 bootloader +is permanently installed. + +Using sudo is Linux specific. You may not need it at all, if the access rights are set properly, +and you will not need it for Windows. + +5. At the Seeed Arch Mix board, disconnect the RX pin (J3-19) with 3.3V, and change the DIP switches +3 an 4 at SW1 back to 1-0. + +6. Once the upload of the bootloader is finished or when it is already installed, push Reset twice. + +The bootloader should start and show a drive icon. Do not push too fast. The i.MX RT MCU +have no dedicated Reset Pin and are reset through power cycling, which may be slow. +Copy the .uf2 version of MicroPython to this drive to install or update MicroPython. +If after steps 1-4 the bootloader drive is already shown, you do not have to reset again. + +Once the UF2 bootloader is installed, only step 6 is required to deploy MicroPython. If +MicroPython is already installed, the bootloader can as well be invoked by calling +`machine.bootloader()` or switching the USB baud rate at the PC to 1200 baud and back. + +If at any time the flash content is corrupted you can always start over from the beginning. diff --git a/ports/mimxrt/boards/deploy_mimxrt.md b/ports/mimxrt/boards/deploy_mimxrt.md index 35752a8362cef..7ad21fe64ca17 100644 --- a/ports/mimxrt/boards/deploy_mimxrt.md +++ b/ports/mimxrt/boards/deploy_mimxrt.md @@ -1,11 +1,45 @@ -Firmware can be loaded to the MIMXRT development boards in various ways. The most convenient -one is using the built-in support MCU. When a PC is connected to the debug USB port, a drive -icon will appear. Firmware can be uploaded to the board by copying it to this drive. The copy -and flashing will take a few moments. At the end of the upload, the drive icon will disappear -and reappear again. Then the reset button has to be pushed, which starts the MicroPython firmware. +## Firmware installation options + +There are two ways to load the MicroPython firmware to the device: + +1. Load the MicroPython firmware directly to the device. The MicroPython +firmware files for that method have the extension .bin or .hex and are available +at the MicroPython download site. +2. Install a resident UF2 bootstrap loader to the device first and use that later for loading +MicroPython. The MicroPython firmware files for that method have the extension .uf2 +and are available at the MicroPython download site. The UF2 bootstrap loader can be obtained +from the site https://github.com/adafruit/tinyuf2. Open the recent release page and +get the version of the bootloader for your board. If there is no specific bootloader +for a specific board, get versions for the respective imxrt10xx-evk board. The file +with the .bin or .hex extension is the one to be installed first. + +## Direct loading of MicroPython or installation of the UF2 bootloader + +The MicroPython firmware or the UF2 bootstrap loader can be loaded to the MIMXRT development +boards in various ways. The most convenient one is using the built-in support MCU. When a PC +is connected to the debug USB port, a drive icon will appear. Firmware can be uploaded to +the board by copying it to this drive. The copy and flashing will take a few moments. +At the end of the upload, the drive icon will disappear and reappear again. Then the reset +button has to be pushed, which starts the MicroPython firmware. Depending on the power jumper settings, both the debug USB and OTG USB port have to be powered during firmware upload. You may as well load the firmware using the JLink port or openSDA interface with the appropriate tools. For more options, consult the user guide of the board. + +## Installing the MicroPython firmware using the UF2 bootloader + +When using the UF2 bootloader, the OTG USB port will be used. +Once the UF2 bootloader is installed, it has to be started to upload MicroPython.The +methods to start the bootloader are: + +- Push reset twice. +- Call machine.bootloader() e.g. from REPL. +- Switch the USB port shortly to 1200 baud and back. That requires MicroPython to be +installed. + +If there is no valid Firmware on the device, the bootloader will start automatically. +Once it's started, a drive ICON will appear. The MicroPython firmware file with .uf2 +extension must then be copied to that drive. When the file is copied and MicroPython +is installed, the drive disappears and MicroPython starts. From d40849d07d2523fa16070bfcae34b51b7b14942b Mon Sep 17 00:00:00 2001 From: robert-hh Date: Mon, 3 Feb 2025 21:12:35 +0100 Subject: [PATCH 059/110] mimxrt/boards: Add flash configuration constants to mpconfigboard.mk. And use these to initialize the LUT table properly for the various flash types. The different flash types differ by 3 parameters. Thus it is easier to just keep one copy of the qspiflash_config structure with the LUT table and update it during flash initialisation as needed. Signed-off-by: robert-hh --- ports/mimxrt/Makefile | 6 ++++ .../boards/ADAFRUIT_METRO_M7/mpconfigboard.mk | 3 ++ .../boards/MIMXRT1010_EVK/mpconfigboard.mk | 3 ++ .../boards/MIMXRT1015_EVK/mpconfigboard.mk | 3 ++ .../boards/MIMXRT1020_EVK/mpconfigboard.mk | 3 ++ .../boards/MIMXRT1050_EVK/mpconfigboard.mk | 3 ++ .../boards/MIMXRT1060_EVK/mpconfigboard.mk | 3 ++ .../boards/MIMXRT1064_EVK/mpconfigboard.mk | 3 ++ .../boards/MIMXRT1170_EVK/mpconfigboard.mk | 3 ++ .../boards/OLIMEX_RT1010/mpconfigboard.mk | 3 ++ .../boards/SEEED_ARCH_MIX/mpconfigboard.mk | 3 ++ ports/mimxrt/boards/TEENSY40/mpconfigboard.mk | 3 ++ ports/mimxrt/boards/TEENSY41/mpconfigboard.mk | 3 ++ ports/mimxrt/hal/flexspi_flash_config.h | 2 +- ports/mimxrt/hal/flexspi_nor_flash.c | 16 +++++++-- ports/mimxrt/hal/flexspi_nor_flash.h | 1 + ports/mimxrt/hal/qspi_nor_flash_config.c | 34 ++++++++----------- 17 files changed, 72 insertions(+), 23 deletions(-) diff --git a/ports/mimxrt/Makefile b/ports/mimxrt/Makefile index ad7d853b677a5..224b38384f718 100644 --- a/ports/mimxrt/Makefile +++ b/ports/mimxrt/Makefile @@ -425,6 +425,12 @@ endif ifdef MICROPY_HW_FLASH_CLK CFLAGS += -DMICROPY_HW_FLASH_CLK=$(MICROPY_HW_FLASH_CLK) endif +ifdef MICROPY_HW_FLASH_QE_CMD + CFLAGS += -DMICROPY_HW_FLASH_QE_CMD=$(MICROPY_HW_FLASH_QE_CMD) +endif +ifdef MICROPY_HW_FLASH_QE_ARG + CFLAGS += -DMICROPY_HW_FLASH_QE_ARG=$(MICROPY_HW_FLASH_QE_ARG) +endif CFLAGS += $(CFLAGS_EXTRA) diff --git a/ports/mimxrt/boards/ADAFRUIT_METRO_M7/mpconfigboard.mk b/ports/mimxrt/boards/ADAFRUIT_METRO_M7/mpconfigboard.mk index 74d3a6f0f10e3..b9c1716a7f60c 100644 --- a/ports/mimxrt/boards/ADAFRUIT_METRO_M7/mpconfigboard.mk +++ b/ports/mimxrt/boards/ADAFRUIT_METRO_M7/mpconfigboard.mk @@ -4,6 +4,9 @@ MCU_VARIANT = MIMXRT1011DAE5A MICROPY_FLOAT_IMPL = single MICROPY_HW_FLASH_TYPE ?= qspi_nor_flash MICROPY_HW_FLASH_SIZE ?= 0x800000 # 8MB +MICROPY_HW_FLASH_CLK = kFlexSpiSerialClk_133MHz +MICROPY_HW_FLASH_QE_CMD = 0x31 +MICROPY_HW_FLASH_QE_ARG = 0x02 MICROPY_PY_NETWORK_NINAW10 ?= 1 MICROPY_PY_SSL ?= 1 diff --git a/ports/mimxrt/boards/MIMXRT1010_EVK/mpconfigboard.mk b/ports/mimxrt/boards/MIMXRT1010_EVK/mpconfigboard.mk index 1aef451e2a6a5..592b89c68ecf7 100644 --- a/ports/mimxrt/boards/MIMXRT1010_EVK/mpconfigboard.mk +++ b/ports/mimxrt/boards/MIMXRT1010_EVK/mpconfigboard.mk @@ -4,6 +4,9 @@ MCU_VARIANT = MIMXRT1011DAE5A MICROPY_FLOAT_IMPL = single MICROPY_HW_FLASH_TYPE = qspi_nor_flash MICROPY_HW_FLASH_SIZE = 0x1000000 # 16MB +MICROPY_HW_FLASH_CLK = kFlexSpiSerialClk_100MHz +MICROPY_HW_FLASH_QE_CMD = 0x31 +MICROPY_HW_FLASH_QE_ARG = 0x02 USE_UF2_BOOTLOADER = 1 diff --git a/ports/mimxrt/boards/MIMXRT1015_EVK/mpconfigboard.mk b/ports/mimxrt/boards/MIMXRT1015_EVK/mpconfigboard.mk index 0f719ac0a9753..c5d02294bc3df 100644 --- a/ports/mimxrt/boards/MIMXRT1015_EVK/mpconfigboard.mk +++ b/ports/mimxrt/boards/MIMXRT1015_EVK/mpconfigboard.mk @@ -4,6 +4,9 @@ MCU_VARIANT = MIMXRT1015DAF5A MICROPY_FLOAT_IMPL = single MICROPY_HW_FLASH_TYPE = qspi_nor_flash MICROPY_HW_FLASH_SIZE = 0x1000000 # 16MB +MICROPY_HW_FLASH_CLK = kFlexSpiSerialClk_100MHz +MICROPY_HW_FLASH_QE_CMD = 0x31 +MICROPY_HW_FLASH_QE_ARG = 0x02 MICROPY_BOOT_BUFFER_SIZE = (32 * 1024) diff --git a/ports/mimxrt/boards/MIMXRT1020_EVK/mpconfigboard.mk b/ports/mimxrt/boards/MIMXRT1020_EVK/mpconfigboard.mk index 7715e669f8d09..5ef078210a2a2 100644 --- a/ports/mimxrt/boards/MIMXRT1020_EVK/mpconfigboard.mk +++ b/ports/mimxrt/boards/MIMXRT1020_EVK/mpconfigboard.mk @@ -4,6 +4,9 @@ MCU_VARIANT = MIMXRT1021DAG5A MICROPY_FLOAT_IMPL = double MICROPY_HW_FLASH_TYPE = qspi_nor_flash MICROPY_HW_FLASH_SIZE = 0x800000 # 8MB +MICROPY_HW_FLASH_CLK = kFlexSpiSerialClk_133MHz +MICROPY_HW_FLASH_QE_CMD = 0x01 +MICROPY_HW_FLASH_QE_ARG = 0x40 MICROPY_HW_SDRAM_AVAIL = 1 MICROPY_HW_SDRAM_SIZE = 0x2000000 # 32MB diff --git a/ports/mimxrt/boards/MIMXRT1050_EVK/mpconfigboard.mk b/ports/mimxrt/boards/MIMXRT1050_EVK/mpconfigboard.mk index ef7bbc8f56cb9..8b048c85eaa98 100644 --- a/ports/mimxrt/boards/MIMXRT1050_EVK/mpconfigboard.mk +++ b/ports/mimxrt/boards/MIMXRT1050_EVK/mpconfigboard.mk @@ -7,6 +7,9 @@ MICROPY_HW_FLASH_SIZE = 0x4000000 # 64MB MICROPY_HW_SDRAM_AVAIL = 1 MICROPY_HW_SDRAM_SIZE = 0x2000000 # 32MB +MICROPY_HW_FLASH_CLK = kFlexSpiSerialClk_133MHz +MICROPY_HW_FLASH_QE_CMD = 0x01 +MICROPY_HW_FLASH_QE_ARG = 0x40 MICROPY_PY_LWIP = 1 MICROPY_PY_SSL = 1 diff --git a/ports/mimxrt/boards/MIMXRT1060_EVK/mpconfigboard.mk b/ports/mimxrt/boards/MIMXRT1060_EVK/mpconfigboard.mk index f57aaff3b8b17..54539952fb4f9 100644 --- a/ports/mimxrt/boards/MIMXRT1060_EVK/mpconfigboard.mk +++ b/ports/mimxrt/boards/MIMXRT1060_EVK/mpconfigboard.mk @@ -4,6 +4,9 @@ MCU_VARIANT = MIMXRT1062DVJ6A MICROPY_FLOAT_IMPL = double MICROPY_HW_FLASH_TYPE = qspi_nor_flash MICROPY_HW_FLASH_SIZE = 0x800000 # 8MB +MICROPY_HW_FLASH_CLK = kFlexSpiSerialClk_133MHz +MICROPY_HW_FLASH_QE_CMD = 0x01 +MICROPY_HW_FLASH_QE_ARG = 0x40 MICROPY_HW_SDRAM_AVAIL = 1 MICROPY_HW_SDRAM_SIZE = 0x2000000 # 32MB diff --git a/ports/mimxrt/boards/MIMXRT1064_EVK/mpconfigboard.mk b/ports/mimxrt/boards/MIMXRT1064_EVK/mpconfigboard.mk index f175d26377abc..b393d7ed9f2c6 100644 --- a/ports/mimxrt/boards/MIMXRT1064_EVK/mpconfigboard.mk +++ b/ports/mimxrt/boards/MIMXRT1064_EVK/mpconfigboard.mk @@ -4,6 +4,9 @@ MCU_VARIANT = MIMXRT1064DVL6A MICROPY_FLOAT_IMPL = double MICROPY_HW_FLASH_TYPE = internal MICROPY_HW_FLASH_SIZE = 0x400000 # 4MB +MICROPY_HW_FLASH_CLK = kFlexSpiSerialClk_100MHz +MICROPY_HW_FLASH_QE_CMD = 0x31 +MICROPY_HW_FLASH_QE_ARG = 0x02 MICROPY_HW_SDRAM_AVAIL = 1 MICROPY_HW_SDRAM_SIZE = 0x2000000 # 32MB diff --git a/ports/mimxrt/boards/MIMXRT1170_EVK/mpconfigboard.mk b/ports/mimxrt/boards/MIMXRT1170_EVK/mpconfigboard.mk index 6e5ee56aa1081..ef4ab683a215c 100644 --- a/ports/mimxrt/boards/MIMXRT1170_EVK/mpconfigboard.mk +++ b/ports/mimxrt/boards/MIMXRT1170_EVK/mpconfigboard.mk @@ -6,6 +6,9 @@ MICROPY_FLOAT_IMPL = double MICROPY_HW_FLASH_TYPE ?= qspi_nor_flash MICROPY_HW_FLASH_SIZE ?= 0x1000000 # 16MB MICROPY_HW_FLASH_RESERVED ?= 0x100000 # 1MB CM4 Code address space +MICROPY_HW_FLASH_CLK = kFlexSpiSerialClk_133MHz +MICROPY_HW_FLASH_QE_CMD = 0x31 +MICROPY_HW_FLASH_QE_ARG = 0x02 MICROPY_HW_SDRAM_AVAIL = 1 MICROPY_HW_SDRAM_SIZE = 0x4000000 # 64MB diff --git a/ports/mimxrt/boards/OLIMEX_RT1010/mpconfigboard.mk b/ports/mimxrt/boards/OLIMEX_RT1010/mpconfigboard.mk index 1de497d74e3f0..7e9987de0cfe3 100644 --- a/ports/mimxrt/boards/OLIMEX_RT1010/mpconfigboard.mk +++ b/ports/mimxrt/boards/OLIMEX_RT1010/mpconfigboard.mk @@ -5,6 +5,9 @@ MICROPY_FLOAT_IMPL = single MICROPY_HW_FLASH_TYPE = qspi_nor_flash MICROPY_HW_FLASH_SIZE = 0x200000 # 2MB MICROPY_HW_FLASH_RESERVED ?= 0x1000 # 4KB +MICROPY_HW_FLASH_CLK = kFlexSpiSerialClk_100MHz +MICROPY_HW_FLASH_QE_CMD = 0x01 +MICROPY_HW_FLASH_QE_ARG = 0x40 USE_UF2_BOOTLOADER = 1 diff --git a/ports/mimxrt/boards/SEEED_ARCH_MIX/mpconfigboard.mk b/ports/mimxrt/boards/SEEED_ARCH_MIX/mpconfigboard.mk index c691e2dcb55c4..7ea107b00df86 100644 --- a/ports/mimxrt/boards/SEEED_ARCH_MIX/mpconfigboard.mk +++ b/ports/mimxrt/boards/SEEED_ARCH_MIX/mpconfigboard.mk @@ -4,6 +4,9 @@ MCU_VARIANT = MIMXRT1052DVL6B MICROPY_FLOAT_IMPL = double MICROPY_HW_FLASH_TYPE = qspi_nor_flash MICROPY_HW_FLASH_SIZE = 0x800000 # 8MB +MICROPY_HW_FLASH_CLK = kFlexSpiSerialClk_133MHz +MICROPY_HW_FLASH_QE_CMD = 0x01 +MICROPY_HW_FLASH_QE_ARG = 0x40 MICROPY_HW_SDRAM_AVAIL = 1 MICROPY_HW_SDRAM_SIZE = 0x2000000 # 32MB diff --git a/ports/mimxrt/boards/TEENSY40/mpconfigboard.mk b/ports/mimxrt/boards/TEENSY40/mpconfigboard.mk index 87c122da9d49d..9393252d1a863 100644 --- a/ports/mimxrt/boards/TEENSY40/mpconfigboard.mk +++ b/ports/mimxrt/boards/TEENSY40/mpconfigboard.mk @@ -5,6 +5,9 @@ MICROPY_FLOAT_IMPL = double MICROPY_HW_FLASH_TYPE = qspi_nor_flash MICROPY_HW_FLASH_SIZE = 0x200000 # 2MB MICROPY_HW_FLASH_RESERVED ?= 0x1000 # 4KB +MICROPY_HW_FLASH_CLK = kFlexSpiSerialClk_133MHz +MICROPY_HW_FLASH_QE_CMD = 0x31 +MICROPY_HW_FLASH_QE_ARG = 0x02 USE_UF2_BOOTLOADER = 1 diff --git a/ports/mimxrt/boards/TEENSY41/mpconfigboard.mk b/ports/mimxrt/boards/TEENSY41/mpconfigboard.mk index e28310d091ad2..6bb9e607f6522 100755 --- a/ports/mimxrt/boards/TEENSY41/mpconfigboard.mk +++ b/ports/mimxrt/boards/TEENSY41/mpconfigboard.mk @@ -5,6 +5,9 @@ MICROPY_FLOAT_IMPL = double MICROPY_HW_FLASH_TYPE = qspi_nor_flash MICROPY_HW_FLASH_SIZE = 0x800000 # 8MB MICROPY_HW_FLASH_RESERVED ?= 0x1000 # 4KB +MICROPY_HW_FLASH_CLK = kFlexSpiSerialClk_133MHz +MICROPY_HW_FLASH_QE_CMD = 0x31 +MICROPY_HW_FLASH_QE_ARG = 0x02 MICROPY_PY_LWIP = 1 MICROPY_PY_SSL = 1 diff --git a/ports/mimxrt/hal/flexspi_flash_config.h b/ports/mimxrt/hal/flexspi_flash_config.h index 7538537aeae5b..d9aa5ed5eab69 100644 --- a/ports/mimxrt/hal/flexspi_flash_config.h +++ b/ports/mimxrt/hal/flexspi_flash_config.h @@ -75,7 +75,7 @@ (FLEXSPI_LUT_OPERAND0(op0) | FLEXSPI_LUT_NUM_PADS0(pad0) | FLEXSPI_LUT_OPCODE0(cmd0) | FLEXSPI_LUT_OPERAND1(op1) | \ FLEXSPI_LUT_NUM_PADS1(pad1) | FLEXSPI_LUT_OPCODE1(cmd1)) -#define EMPTY_LUT \ +#define EMPTY_LUT_SEQ \ FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), \ FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), \ FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), \ diff --git a/ports/mimxrt/hal/flexspi_nor_flash.c b/ports/mimxrt/hal/flexspi_nor_flash.c index a890533a69661..6d5b830383f68 100644 --- a/ports/mimxrt/hal/flexspi_nor_flash.c +++ b/ports/mimxrt/hal/flexspi_nor_flash.c @@ -46,11 +46,23 @@ uint32_t LUT_pageprogram_quad[4] = { FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler }; +uint32_t LUT_write_status[4] = { + // 4 Write status word for Quad mode + FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, MICROPY_HW_FLASH_QE_CMD, WRITE_SDR, FLEXSPI_1PAD, 0x01), + FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler + FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler + FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler +}; + void flexspi_nor_update_lut(void) { uint32_t lookuptable_copy[64]; memcpy(lookuptable_copy, (const uint32_t *)&qspiflash_config.memConfig.lookupTable, 64 * sizeof(uint32_t)); + // write WRITESTATUSREG code to entry 10 + memcpy(&lookuptable_copy[NOR_CMD_LUT_SEQ_IDX_WRITESTATUSREG * 4], + LUT_write_status, 4 * sizeof(uint32_t)); // write PAGEPROGRAM_QUAD code to entry 10 - memcpy(&lookuptable_copy[10 * 4], LUT_pageprogram_quad, 4 * sizeof(uint32_t)); + memcpy(&lookuptable_copy[NOR_CMD_LUT_SEQ_IDX_PAGEPROGRAM_QUAD * 4], + LUT_pageprogram_quad, 4 * sizeof(uint32_t)); FLEXSPI_UpdateLUT(BOARD_FLEX_SPI, 0, lookuptable_copy, 64); } @@ -123,7 +135,7 @@ status_t flexspi_nor_enable_quad_mode(FLEXSPI_Type *base) __attribute__((section status_t flexspi_nor_enable_quad_mode(FLEXSPI_Type *base) { flexspi_transfer_t flashXfer; status_t status; - uint32_t writeValue = qspiflash_config.memConfig.deviceModeArg; + uint32_t writeValue = MICROPY_HW_FLASH_QE_ARG; /* Write enable */ status = flexspi_nor_write_enable(base, 0); diff --git a/ports/mimxrt/hal/flexspi_nor_flash.h b/ports/mimxrt/hal/flexspi_nor_flash.h index 6526142af22f1..2e61effc2b7d8 100644 --- a/ports/mimxrt/hal/flexspi_nor_flash.h +++ b/ports/mimxrt/hal/flexspi_nor_flash.h @@ -47,6 +47,7 @@ extern flexspi_nor_config_t qspiflash_config; status_t flexspi_nor_get_vendor_id(FLEXSPI_Type *base, uint8_t *vendorId); status_t flexspi_nor_init(void); void flexspi_nor_update_lut(void); +status_t flexspi_nor_enable_quad_mode(FLEXSPI_Type *base); status_t flexspi_nor_flash_erase_sector(FLEXSPI_Type *base, uint32_t address); status_t flexspi_nor_flash_erase_block(FLEXSPI_Type *base, uint32_t address); status_t flexspi_nor_flash_page_program(FLEXSPI_Type *base, uint32_t address, const uint32_t *src, uint32_t size); diff --git a/ports/mimxrt/hal/qspi_nor_flash_config.c b/ports/mimxrt/hal/qspi_nor_flash_config.c index 9edf44fc64c84..36ccc30fc1922 100644 --- a/ports/mimxrt/hal/qspi_nor_flash_config.c +++ b/ports/mimxrt/hal/qspi_nor_flash_config.c @@ -48,7 +48,7 @@ const flexspi_nor_config_t qspiflash_config = { .seqId = 4u, .seqNum = 1u, }, - .deviceModeArg = 0x40, + .deviceModeArg = MICROPY_HW_FLASH_QE_ARG, .deviceType = kFlexSpiDeviceType_SerialNOR, .sflashPadType = kSerialFlash_4Pads, .serialClkFreq = MICROPY_HW_FLASH_CLK, @@ -68,10 +68,7 @@ const flexspi_nor_config_t qspiflash_config = { FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler // 2 Read extend parameters - FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x81, READ_SDR, FLEXSPI_1PAD, 0x04), - FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler - FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler - FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler + EMPTY_LUT_SEQ // 3 Write Enable FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x06, STOP, FLEXSPI_1PAD, 0), @@ -80,7 +77,7 @@ const flexspi_nor_config_t qspiflash_config = { FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler // 4 Write Status Reg - FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x01, WRITE_SDR, FLEXSPI_1PAD, 0x01), + FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, MICROPY_HW_FLASH_QE_CMD, WRITE_SDR, FLEXSPI_1PAD, 0x01), FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler @@ -92,16 +89,10 @@ const flexspi_nor_config_t qspiflash_config = { FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler // 6 Fast read quad mode - SDR - FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x6B, RADDR_SDR, FLEXSPI_1PAD, 24), - FLEXSPI_LUT_SEQ(DUMMY_SDR, FLEXSPI_4PAD, 0x08, READ_SDR, FLEXSPI_4PAD, 0x04), - FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler - FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler + EMPTY_LUT_SEQ // 7 Read ID - FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x90, DUMMY_SDR, FLEXSPI_1PAD, 24), - FLEXSPI_LUT_SEQ(READ_SDR, FLEXSPI_1PAD, 0x00, 0, 0, 0), - FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler - FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler + EMPTY_LUT_SEQ // 8 Erase Block (32k) FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x52, RADDR_SDR, FLEXSPI_1PAD, 24), @@ -127,14 +118,17 @@ const flexspi_nor_config_t qspiflash_config = { FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler - // 12 Empty LUT - EMPTY_LUT + // 12 Not used + EMPTY_LUT_SEQ // 13 READ SDFP - FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x5A, RADDR_SDR, FLEXSPI_1PAD, 24), - FLEXSPI_LUT_SEQ(DUMMY_SDR, FLEXSPI_1PAD, 8, READ_SDR, FLEXSPI_1PAD, 0x04), - FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler - FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler + EMPTY_LUT_SEQ + + // 14 Not used + EMPTY_LUT_SEQ + + // 15 Not used + EMPTY_LUT_SEQ }, }, .pageSize = 256u, From 0a433a02e1eab1c95e57129a1e828780c871be42 Mon Sep 17 00:00:00 2001 From: robert-hh Date: Tue, 4 Feb 2025 18:17:47 +0100 Subject: [PATCH 060/110] mimxrt/hal: Set the flexspi flash CLK frequency on boot. The flash clock frequency may have been set to a different value by a bootloader. Set the frequency according to the configured value. Use a table of pre-calculated dividers to get the closest value for the flash frequency, achieving for MIMXRT10xx: 30 -> 30.85 MHz 50 -> 49.65 MHz 60 -> 60 MHz 75 -> 75.13 MHz 80 -> 80 MHz 100 -> 99.31 Mhz 133 -> 132.92 MHz 166 -> 166.15 MHz for MIMXRT1176: 30 -> 31 MHz 50 -> 52.8 MJz 60 -> 58.7 MHz 75 -> 75.4 MHz 80 -> 75.4 MHz 100 -> 105.6 MHz 133 -> 132 MHz 166 -> 176 MHz Signed-off-by: robert-hh --- ports/mimxrt/flash.c | 3 +- ports/mimxrt/hal/flexspi_hyper_flash.c | 2 +- ports/mimxrt/hal/flexspi_hyper_flash.h | 2 +- ports/mimxrt/hal/flexspi_nor_flash.c | 93 +++++++++++++++++++++++++- ports/mimxrt/hal/flexspi_nor_flash.h | 3 +- 5 files changed, 95 insertions(+), 8 deletions(-) diff --git a/ports/mimxrt/flash.c b/ports/mimxrt/flash.c index cdcdc7483d833..57091b36ddcfd 100644 --- a/ports/mimxrt/flash.c +++ b/ports/mimxrt/flash.c @@ -29,7 +29,8 @@ void flash_init(void) { // Upload the custom flash configuration // And fix the entry for PAGEPROGRAM_QUAD - flexspi_nor_update_lut(); + // Update the flash CLK + flexspi_nor_update_lut_clk(MICROPY_HW_FLASH_CLK); // Configure FLEXSPI IP FIFO access. BOARD_FLEX_SPI->MCR0 &= ~(FLEXSPI_MCR0_ARDFEN_MASK); diff --git a/ports/mimxrt/hal/flexspi_hyper_flash.c b/ports/mimxrt/hal/flexspi_hyper_flash.c index c0cd63ca0be8e..9171640a9ed83 100644 --- a/ports/mimxrt/hal/flexspi_hyper_flash.c +++ b/ports/mimxrt/hal/flexspi_hyper_flash.c @@ -9,7 +9,7 @@ #include "fsl_clock.h" #include "flexspi_hyper_flash.h" -void flexspi_nor_update_lut(void) { +void flexspi_nor_update_lut_clk(uint32_t freq_index) { } // Copy of a few (pseudo-)functions from fsl_clock.h, which were nor reliably diff --git a/ports/mimxrt/hal/flexspi_hyper_flash.h b/ports/mimxrt/hal/flexspi_hyper_flash.h index a6454a1c9a234..40416d6e21d7f 100644 --- a/ports/mimxrt/hal/flexspi_hyper_flash.h +++ b/ports/mimxrt/hal/flexspi_hyper_flash.h @@ -47,7 +47,7 @@ extern flexspi_nor_config_t qspiflash_config; status_t flexspi_nor_hyperflash_cfi(FLEXSPI_Type *base); void flexspi_hyper_flash_init(void); -void flexspi_nor_update_lut(void); +void flexspi_nor_update_lut_clk(uint32_t freq_index); status_t flexspi_nor_flash_erase_sector(FLEXSPI_Type *base, uint32_t address); status_t flexspi_nor_flash_erase_block(FLEXSPI_Type *base, uint32_t address); status_t flexspi_nor_flash_page_program(FLEXSPI_Type *base, uint32_t address, const uint32_t *src, uint32_t size); diff --git a/ports/mimxrt/hal/flexspi_nor_flash.c b/ports/mimxrt/hal/flexspi_nor_flash.c index 6d5b830383f68..7fdb6b7c13df6 100644 --- a/ports/mimxrt/hal/flexspi_nor_flash.c +++ b/ports/mimxrt/hal/flexspi_nor_flash.c @@ -38,6 +38,9 @@ #include "flexspi_nor_flash.h" #include "flexspi_flash_config.h" +bool flash_busy_status_pol = 0; +bool flash_busy_status_offset = 0; + uint32_t LUT_pageprogram_quad[4] = { // 10 Page Program - quad mode FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x32, RADDR_SDR, FLEXSPI_1PAD, 24), @@ -54,16 +57,100 @@ uint32_t LUT_write_status[4] = { FLEXSPI_LUT_SEQ(0, 0, 0, 0, 0, 0), // Filler }; -void flexspi_nor_update_lut(void) { +#if defined(MIMXRT117x_SERIES) +static uint8_t div_table_mhz[] = { + 17, // Entry 0 is out of range + 17, // 30 -> 31 MHz + 10, // 50 -> 52.8 MHz + 9, // 60 -> 58.7 MHz + 7, // 75 -> 75.4 MHz + 7, // 80 -> 75.4 MHz + 5, // 100 -> 105.6 Mhz + 4, // 133 -> 132 MHz + 3 // 166 -> 176 MHz +}; + +#else +typedef struct _ps_div_t { + uint8_t pfd480_div; + uint8_t podf_div; +} ps_div_t; + +static ps_div_t div_table_mhz[] = { + { 35, 8 }, // Entry 0 is out of range + { 35, 8 }, // 30 -> 30.85 MHz + { 29, 6 }, // 50 -> 49.65 MHz + { 18, 8 }, // 60 -> 60 MHz + { 23, 5 }, // 75 -> 75.13 MHz + { 18, 6 }, // 80 -> 80 MHz + { 17, 5 }, // 100 -> 191 Mhz + { 13, 5 }, // 133 -> 132.92 MHz + { 13, 4 } // 166 -> 166.15 MHz +}; +#endif + +__attribute__((section(".ram_functions"))) void flexspi_nor_update_lut_clk(uint32_t freq_index) { + // Create a local copy of the LookupTable. Modify the entry for WRITESTATUSREG + // Add an entry for PAGEPROGRAM_QUAD. uint32_t lookuptable_copy[64]; memcpy(lookuptable_copy, (const uint32_t *)&qspiflash_config.memConfig.lookupTable, 64 * sizeof(uint32_t)); - // write WRITESTATUSREG code to entry 10 + // write local WRITESTATUSREG code to index 4 memcpy(&lookuptable_copy[NOR_CMD_LUT_SEQ_IDX_WRITESTATUSREG * 4], LUT_write_status, 4 * sizeof(uint32_t)); - // write PAGEPROGRAM_QUAD code to entry 10 + // write local PAGEPROGRAM_QUAD code to index 10 memcpy(&lookuptable_copy[NOR_CMD_LUT_SEQ_IDX_PAGEPROGRAM_QUAD * 4], LUT_pageprogram_quad, 4 * sizeof(uint32_t)); + // Update the LookupTable. FLEXSPI_UpdateLUT(BOARD_FLEX_SPI, 0, lookuptable_copy, 64); + + __DSB(); + __ISB(); + __disable_irq(); + SCB_DisableDCache(); + + #if defined(MIMXRT117x_SERIES) + volatile uint8_t pll2_div = div_table_mhz[freq_index] - 1; + + while (!FLEXSPI_GetBusIdleStatus(BOARD_FLEX_SPI)) { + } + FLEXSPI_Enable(BOARD_FLEX_SPI, false); + + // Disable FlexSPI clock + // Flexspi is clocked by PLL2. Only the divider can be changed. + CCM->LPCG[kCLOCK_Flexspi1].DIRECT = ((uint32_t)kCLOCK_Off & CCM_LPCG_DIRECT_ON_MASK); + // Change the PLL divider + CCM->CLOCK_ROOT[kCLOCK_Root_Flexspi1].CONTROL = (CCM->CLOCK_ROOT[kCLOCK_Root_Flexspi1].CONTROL & ~CCM_CLOCK_ROOT_CONTROL_DIV_MASK) | + CCM_CLOCK_ROOT_CONTROL_DIV(pll2_div); + // Re-enable FlexSPI clock + CCM->LPCG[kCLOCK_Flexspi1].DIRECT = ((uint32_t)kCLOCK_On & CCM_LPCG_DIRECT_ON_MASK); + + #else + + volatile uint8_t pfd480_div = div_table_mhz[freq_index].pfd480_div; + volatile uint8_t podf_div = div_table_mhz[freq_index].podf_div - 1; + + while (!FLEXSPI_GetBusIdleStatus(BOARD_FLEX_SPI)) { + } + FLEXSPI_Enable(BOARD_FLEX_SPI, false); + + // Disable FlexSPI clock + CCM->CCGR6 &= ~CCM_CCGR6_CG5_MASK; + // Changing the clock is OK now. + // Change the PFD + CCM_ANALOG->PFD_480 = (CCM_ANALOG->PFD_480 & ~CCM_ANALOG_PFD_480_TOG_PFD0_FRAC_MASK) | CCM_ANALOG_PFD_480_TOG_PFD0_FRAC(pfd480_div); + // Change the flexspi divider + CCM->CSCMR1 = (CCM->CSCMR1 & ~CCM_CSCMR1_FLEXSPI_PODF_MASK) | CCM_CSCMR1_FLEXSPI_PODF(podf_div); + // Re-enable FlexSPI + CCM->CCGR6 |= CCM_CCGR6_CG5_MASK; + #endif + + FLEXSPI_Enable(BOARD_FLEX_SPI, true); + FLEXSPI_SoftwareReset(BOARD_FLEX_SPI); + while (!FLEXSPI_GetBusIdleStatus(BOARD_FLEX_SPI)) { + } + + SCB_EnableDCache(); + __enable_irq(); } void flexspi_nor_reset(FLEXSPI_Type *base) __attribute__((section(".ram_functions"))); diff --git a/ports/mimxrt/hal/flexspi_nor_flash.h b/ports/mimxrt/hal/flexspi_nor_flash.h index 2e61effc2b7d8..dc5798d97f409 100644 --- a/ports/mimxrt/hal/flexspi_nor_flash.h +++ b/ports/mimxrt/hal/flexspi_nor_flash.h @@ -45,8 +45,7 @@ extern flexspi_nor_config_t qspiflash_config; status_t flexspi_nor_get_vendor_id(FLEXSPI_Type *base, uint8_t *vendorId); -status_t flexspi_nor_init(void); -void flexspi_nor_update_lut(void); +void flexspi_nor_update_lut_clk(uint32_t freq_index); status_t flexspi_nor_enable_quad_mode(FLEXSPI_Type *base); status_t flexspi_nor_flash_erase_sector(FLEXSPI_Type *base, uint32_t address); status_t flexspi_nor_flash_erase_block(FLEXSPI_Type *base, uint32_t address); From 2f646f93d375577d7bc5b500e6fe50c9381875c2 Mon Sep 17 00:00:00 2001 From: iabdalkader Date: Fri, 29 Nov 2024 16:22:51 +0100 Subject: [PATCH 061/110] mimxrt: Add optional MSC support. Add MSC support using internal flash storage or SD card. Note this is disabled by default, and can be enabled by boards if needed. Signed-off-by: iabdalkader --- ports/mimxrt/Makefile | 1 + ports/mimxrt/main.c | 15 ++++ ports/mimxrt/modmimxrt.c | 3 + ports/mimxrt/modmimxrt.h | 1 + ports/mimxrt/mpconfigport.h | 7 ++ ports/mimxrt/msc_disk.c | 153 ++++++++++++++++++++++++++++++++++++ 6 files changed, 180 insertions(+) create mode 100644 ports/mimxrt/msc_disk.c diff --git a/ports/mimxrt/Makefile b/ports/mimxrt/Makefile index 224b38384f718..8106ab4f3ff03 100644 --- a/ports/mimxrt/Makefile +++ b/ports/mimxrt/Makefile @@ -217,6 +217,7 @@ SRC_C += \ modmimxrt.c \ mphalport.c \ mpnetworkport.c \ + msc_disk.c \ network_lan.c \ pendsv.c \ pin.c \ diff --git a/ports/mimxrt/main.c b/ports/mimxrt/main.c index adb071a7fee8f..6b9d4fa17afe4 100644 --- a/ports/mimxrt/main.c +++ b/ports/mimxrt/main.c @@ -39,6 +39,7 @@ #include "led.h" #include "pendsv.h" #include "modmachine.h" +#include "modmimxrt.h" #if MICROPY_PY_LWIP #include "lwip/init.h" @@ -55,6 +56,7 @@ #include "systick.h" #include "extmod/modnetwork.h" +#include "extmod/vfs.h" extern uint8_t _sstack, _estack, _gc_heap_start, _gc_heap_end; @@ -113,6 +115,19 @@ int main(void) { // Execute _boot.py to set up the filesystem. pyexec_frozen_module("_boot.py", false); + #if MICROPY_HW_USB_MSC + // Set the USB medium to flash block device. + mimxrt_msc_medium = &mimxrt_flash_type; + + #if MICROPY_PY_MACHINE_SDCARD + const char *path = "/sdcard"; + // If SD is mounted, set the USB medium to SD. + if (mp_vfs_lookup_path(path, &path) != MP_VFS_NONE) { + mimxrt_msc_medium = &machine_sdcard_type; + } + #endif + #endif + // Execute user scripts. int ret = pyexec_file_if_exists("boot.py"); diff --git a/ports/mimxrt/modmimxrt.c b/ports/mimxrt/modmimxrt.c index 0f67538ce0d21..d699027388521 100644 --- a/ports/mimxrt/modmimxrt.c +++ b/ports/mimxrt/modmimxrt.c @@ -31,6 +31,9 @@ static const mp_rom_map_elem_t mimxrt_module_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_mimxrt) }, { MP_ROM_QSTR(MP_QSTR_Flash), MP_ROM_PTR(&mimxrt_flash_type) }, + #if MICROPY_HW_USB_MSC + { MP_ROM_QSTR(MP_QSTR_MSC), MP_ROM_TRUE }, + #endif }; static MP_DEFINE_CONST_DICT(mimxrt_module_globals, mimxrt_module_globals_table); diff --git a/ports/mimxrt/modmimxrt.h b/ports/mimxrt/modmimxrt.h index e047716691807..19cf799f88771 100644 --- a/ports/mimxrt/modmimxrt.h +++ b/ports/mimxrt/modmimxrt.h @@ -30,5 +30,6 @@ extern const mp_obj_type_t mimxrt_flash_type; extern const mp_obj_module_t mp_module_mimxrt; +extern const mp_obj_type_t *mimxrt_msc_medium; #endif // MICROPY_INCLUDED_MIMXRT_MODMIMXRT_H diff --git a/ports/mimxrt/mpconfigport.h b/ports/mimxrt/mpconfigport.h index b45f8408e6415..7da85f1aee579 100644 --- a/ports/mimxrt/mpconfigport.h +++ b/ports/mimxrt/mpconfigport.h @@ -151,8 +151,15 @@ uint32_t trng_random_u32(void); #endif #define MICROPY_HW_ENABLE_USBDEV (1) +// Enable USB-CDC serial port +#ifndef MICROPY_HW_USB_CDC #define MICROPY_HW_USB_CDC (1) #define MICROPY_HW_USB_CDC_1200BPS_TOUCH (1) +#endif +// Enable USB Mass Storage with FatFS filesystem. +#ifndef MICROPY_HW_USB_MSC +#define MICROPY_HW_USB_MSC (0) +#endif // Hooks to add builtins diff --git a/ports/mimxrt/msc_disk.c b/ports/mimxrt/msc_disk.c new file mode 100644 index 0000000000000..c3801f481e618 --- /dev/null +++ b/ports/mimxrt/msc_disk.c @@ -0,0 +1,153 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2020-2021 Damien P. George + * Copyright (c) 2024-2025 Ibrahim Abdelkader + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +#include "tusb.h" +#if CFG_TUD_MSC +#include "flash.h" +#include BOARD_FLASH_OPS_HEADER_H +#include "stdlib.h" +#include "modmimxrt.h" +#if MICROPY_PY_MACHINE_SDCARD +#include "sdcard.h" + +#ifndef MICROPY_HW_SDCARD_SDMMC +#define MICROPY_HW_SDCARD_SDMMC (1) +#endif + +#define MSC_SDCARD_INDEX (MICROPY_HW_SDCARD_SDMMC - 1) +#endif + +// This implementation does Not support Flash sector caching. +// MICROPY_FATFS_MAX_SS must be identical to SECTOR_SIZE_BYTES +#define BLOCK_SIZE (SECTOR_SIZE_BYTES) +#define BLOCK_COUNT (MICROPY_HW_FLASH_STORAGE_BYTES / BLOCK_SIZE) +#define FLASH_BASE_ADDR (MICROPY_HW_FLASH_STORAGE_BASE) + +static bool msc_ejected = false; + +const mp_obj_type_t *mimxrt_msc_medium = NULL; + +// Invoked when received SCSI_CMD_INQUIRY +// Application fill vendor id, product id and revision with string up to 8, 16, 4 characters respectively +void tud_msc_inquiry_cb(uint8_t lun, uint8_t vendor_id[8], uint8_t product_id[16], uint8_t product_rev[4]) { + memcpy(vendor_id, MICROPY_HW_USB_MSC_INQUIRY_VENDOR_STRING, MIN(strlen(MICROPY_HW_USB_MSC_INQUIRY_VENDOR_STRING), 8)); + memcpy(product_id, MICROPY_HW_USB_MSC_INQUIRY_PRODUCT_STRING, MIN(strlen(MICROPY_HW_USB_MSC_INQUIRY_PRODUCT_STRING), 16)); + memcpy(product_rev, MICROPY_HW_USB_MSC_INQUIRY_REVISION_STRING, MIN(strlen(MICROPY_HW_USB_MSC_INQUIRY_REVISION_STRING), 4)); +} + +// Invoked when received Test Unit Ready command. +// return true allowing host to read/write this LUN e.g SD card inserted +bool tud_msc_test_unit_ready_cb(uint8_t lun) { + if (msc_ejected || mimxrt_msc_medium == NULL) { + tud_msc_set_sense(lun, SCSI_SENSE_NOT_READY, 0x3a, 0x00); + return false; + } + return true; +} + +// Invoked when received SCSI_CMD_READ_CAPACITY_10 and SCSI_CMD_READ_FORMAT_CAPACITY to determine the disk size +// Application update block count and block size +void tud_msc_capacity_cb(uint8_t lun, uint32_t *block_count, uint16_t *block_size) { + if (mimxrt_msc_medium == &mimxrt_flash_type) { + *block_size = BLOCK_SIZE; + *block_count = BLOCK_COUNT; + #if MICROPY_PY_MACHINE_SDCARD + } else if (mimxrt_msc_medium == &machine_sdcard_type) { + mimxrt_sdcard_obj_t *card = &mimxrt_sdcard_objs[MSC_SDCARD_INDEX]; + *block_size = card->block_len; + *block_count = card->block_count; + #endif + } +} + +// Invoked when received Start Stop Unit command +// - Start = 0 : stopped power mode, if load_eject = 1 : unload disk storage +// - Start = 1 : active mode, if load_eject = 1 : load disk storage +bool tud_msc_start_stop_cb(uint8_t lun, uint8_t power_condition, bool start, bool load_eject) { + if (load_eject) { + if (start) { + // load disk storage + msc_ejected = false; + } else { + // unload disk storage + msc_ejected = true; + } + } + return true; +} + +// Callback invoked when received READ10 command. +// Copy disk's data to buffer (up to bufsize) and return number of copied bytes. +int32_t tud_msc_read10_cb(uint8_t lun, uint32_t lba, uint32_t offset, void *buffer, uint32_t bufsize) { + if (mimxrt_msc_medium == &mimxrt_flash_type) { + flash_read_block(FLASH_BASE_ADDR + lba * BLOCK_SIZE, buffer, bufsize); + #if MICROPY_PY_MACHINE_SDCARD + } else if (mimxrt_msc_medium == &machine_sdcard_type) { + mimxrt_sdcard_obj_t *card = &mimxrt_sdcard_objs[MSC_SDCARD_INDEX]; + sdcard_read(card, buffer, lba, bufsize / card->block_len); + #endif + } + return bufsize; +} + +// Callback invoked when received WRITE10 command. +// Process data in buffer to disk's storage and return number of written bytes +int32_t tud_msc_write10_cb(uint8_t lun, uint32_t lba, uint32_t offset, uint8_t *buffer, uint32_t bufsize) { + if (mimxrt_msc_medium == &mimxrt_flash_type) { + // Erase count sectors starting at lba + for (int n = 0; n < (bufsize / BLOCK_SIZE); n++) { + flash_erase_sector(FLASH_BASE_ADDR + (lba + n) * BLOCK_SIZE); + } + flash_write_block(FLASH_BASE_ADDR + lba * BLOCK_SIZE, buffer, bufsize); + #if MICROPY_PY_MACHINE_SDCARD + } else if (mimxrt_msc_medium == &machine_sdcard_type) { + mimxrt_sdcard_obj_t *card = &mimxrt_sdcard_objs[MSC_SDCARD_INDEX]; + sdcard_write(card, buffer, lba, bufsize / card->block_len); + + #endif + } + return bufsize; +} + +// Callback invoked when received an SCSI command not in built-in list below +// - READ_CAPACITY10, READ_FORMAT_CAPACITY, INQUIRY, MODE_SENSE6, REQUEST_SENSE +// - READ10 and WRITE10 has their own callbacks +int32_t tud_msc_scsi_cb(uint8_t lun, uint8_t const scsi_cmd[16], void *buffer, uint16_t bufsize) { + int32_t resplen = 0; + switch (scsi_cmd[0]) { + case SCSI_CMD_PREVENT_ALLOW_MEDIUM_REMOVAL: + // Sync the logical unit if needed. + break; + + default: + // Set Sense = Invalid Command Operation + tud_msc_set_sense(lun, SCSI_SENSE_ILLEGAL_REQUEST, 0x20, 0x00); + // negative means error -> tinyusb could stall and/or response with failed status + resplen = -1; + break; + } + return resplen; +} +#endif From cb417505f3c39a526162fe8db9365035ea415c98 Mon Sep 17 00:00:00 2001 From: iabdalkader Date: Fri, 7 Feb 2025 13:39:02 +0100 Subject: [PATCH 062/110] mimxrt/boards: Reduce stack size for 1011 and 1015 MCUs. Reduced to 16KBs to allow enabling MSC. Signed-off-by: iabdalkader --- ports/mimxrt/boards/MIMXRT1011.ld | 4 ++-- ports/mimxrt/boards/MIMXRT1015.ld | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ports/mimxrt/boards/MIMXRT1011.ld b/ports/mimxrt/boards/MIMXRT1011.ld index ab363bd56a025..0e961a49433f2 100644 --- a/ports/mimxrt/boards/MIMXRT1011.ld +++ b/ports/mimxrt/boards/MIMXRT1011.ld @@ -27,8 +27,8 @@ dtcm_size = 0x00008000; ocrm_start = 0x20200000; ocrm_size = 0x00010000; -/* 20kiB stack. */ -__stack_size__ = 0x5000; +/* 16kiB stack. */ +__stack_size__ = 0x4000; _estack = __StackTop; _sstack = __StackLimit; diff --git a/ports/mimxrt/boards/MIMXRT1015.ld b/ports/mimxrt/boards/MIMXRT1015.ld index 0237d348c2498..58b88e0fe3084 100644 --- a/ports/mimxrt/boards/MIMXRT1015.ld +++ b/ports/mimxrt/boards/MIMXRT1015.ld @@ -27,8 +27,8 @@ dtcm_size = 0x00008000; ocrm_start = 0x20200000; ocrm_size = 0x00010000; -/* 24kiB stack. */ -__stack_size__ = 0x5000; +/* 16kiB stack. */ +__stack_size__ = 0x4000; _estack = __StackTop; _sstack = __StackLimit; From 752c1672af7f516922416e8afe7f4cb42e151cae Mon Sep 17 00:00:00 2001 From: iabdalkader Date: Fri, 7 Feb 2025 13:45:44 +0100 Subject: [PATCH 063/110] tools/ci.sh: Build MIMXRT1060_EVK with MSC enabled as part of mimxrt CI. Signed-off-by: iabdalkader --- tools/ci.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/ci.sh b/tools/ci.sh index 2c647012f144d..ff362efd29e2d 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -230,6 +230,8 @@ function ci_mimxrt_build { make ${MAKEOPTS} -C ports/mimxrt BOARD=MIMXRT1020_EVK make ${MAKEOPTS} -C ports/mimxrt BOARD=TEENSY40 submodules make ${MAKEOPTS} -C ports/mimxrt BOARD=TEENSY40 + make ${MAKEOPTS} -C ports/mimxrt BOARD=MIMXRT1060_EVK submodules + make ${MAKEOPTS} -C ports/mimxrt BOARD=MIMXRT1060_EVK CFLAGS_EXTRA=-DMICROPY_HW_USB_MSC=1 } ######################################################################################## From 11c9656fad716d42a33e9f5bc65596300d141daa Mon Sep 17 00:00:00 2001 From: Glenn Moloney Date: Tue, 19 Sep 2023 11:41:46 +1000 Subject: [PATCH 064/110] tools/mpremote: Support mip install from package.json on local fs. Add support for `mpremote mip install package.json` where `package.json` is a json file on the local filesystem. Without this, package json files can only be loaded from http, https, github or gitlab URLs. This is useful for testing `package.json` files for pacages in development and for constructing one's own `package.json` files for Python packages which are not yet available for installation using mip. Signed-off-by: Glenn Moloney --- tools/mpremote/mpremote/mip.py | 40 ++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/tools/mpremote/mpremote/mip.py b/tools/mpremote/mpremote/mip.py index d23a0e2cbcb4f..4ce62b7f77bbb 100644 --- a/tools/mpremote/mpremote/mip.py +++ b/tools/mpremote/mpremote/mip.py @@ -13,6 +13,8 @@ _PACKAGE_INDEX = "https://micropython.org/pi/v2" +allowed_mip_url_prefixes = ("http://", "https://", "github:", "gitlab:") + # This implements os.makedirs(os.dirname(path)) def _ensure_path_exists(transport, path): @@ -78,16 +80,25 @@ def _download_file(transport, url, dest): def _install_json(transport, package_json_url, index, target, version, mpy): - try: - with urllib.request.urlopen(_rewrite_url(package_json_url, version)) as response: - package_json = json.load(response) - except urllib.error.HTTPError as e: - if e.status == 404: - raise CommandError(f"Package not found: {package_json_url}") - else: - raise CommandError(f"Error {e.status} requesting {package_json_url}") - except urllib.error.URLError as e: - raise CommandError(f"{e.reason} requesting {package_json_url}") + if package_json_url.startswith(allowed_mip_url_prefixes): + try: + with urllib.request.urlopen(_rewrite_url(package_json_url, version)) as response: + package_json = json.load(response) + except urllib.error.HTTPError as e: + if e.status == 404: + raise CommandError(f"Package not found: {package_json_url}") + else: + raise CommandError(f"Error {e.status} requesting {package_json_url}") + except urllib.error.URLError as e: + raise CommandError(f"{e.reason} requesting {package_json_url}") + elif package_json_url.endswith(".json"): + try: + with open(package_json_url, "r") as f: + package_json = json.load(f) + except OSError: + raise CommandError(f"Error opening {package_json_url}") + else: + raise CommandError(f"Invalid url for package: {package_json_url}") for target_path, short_hash in package_json.get("hashes", ()): fs_target_path = target + "/" + target_path file_url = f"{index}/file/{short_hash[:2]}/{short_hash}" @@ -100,12 +111,7 @@ def _install_json(transport, package_json_url, index, target, version, mpy): def _install_package(transport, package, index, target, version, mpy): - if ( - package.startswith("http://") - or package.startswith("https://") - or package.startswith("github:") - or package.startswith("gitlab:") - ): + if package.startswith(allowed_mip_url_prefixes): if package.endswith(".py") or package.endswith(".mpy"): print(f"Downloading {package} to {target}") _download_file( @@ -118,6 +124,8 @@ def _install_package(transport, package, index, target, version, mpy): package += "/" package += "package.json" print(f"Installing {package} to {target}") + elif package.endswith(".json"): + pass else: if not version: version = "latest" From b11ba39c57ef6dd4d5a865464229a2aede9c825e Mon Sep 17 00:00:00 2001 From: Neil Ludban Date: Sat, 25 Jan 2025 16:00:24 -0500 Subject: [PATCH 065/110] rp2/modules: Fix memory leak and logic bug in handling of _pio_funcs. The `rp2` package use a global dict `_pio_funcs` to populate a namespace for `@asm_pio` functions to be executed in. That dict is not cleaned up after use, keeping references to bound methods of a `PIOASMEmit`. By not setting/clearing all the functions, `asm_pio_encode` unintentionally allows the use of the old directives (harmless) as well as `jmp` (in general, produces the wrong output). Fix that by making sure `_pio_funcs` is returned to its original state after using it: - For `@asm_pio` update the target dict from `_pio_funcs` and then set additional functions as needed, leaving `_pio_funcs` unchanged. - For `asm_pio_encode`, borrow `_pio_funcs` to use as globals (avoiding a bunch of memory alloc/free) but delete the instruction entries after use. Signed-off-by: Neil Ludban --- ports/rp2/modules/rp2.py | 91 +++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 49 deletions(-) diff --git a/ports/rp2/modules/rp2.py b/ports/rp2/modules/rp2.py index e9be7dac8d58e..6068926036baf 100644 --- a/ports/rp2/modules/rp2.py +++ b/ports/rp2/modules/rp2.py @@ -215,49 +215,46 @@ def set(self, dest, data): # "block": see above "clear": 0x40, "rel": lambda x: x | 0x10, - # functions - "wrap_target": None, - "wrap": None, - "label": None, - "word": None, - "nop": None, - "jmp": None, - "wait": None, - "in_": None, - "out": None, - "push": None, - "pull": None, - "mov": None, - "irq": None, - "set": None, } +_pio_directives = ( + "wrap_target", + "wrap", + "label", +) + + +_pio_instructions = ( + "word", + "nop", + "jmp", + "wait", + "in_", + "out", + "push", + "pull", + "mov", + "irq", + "set", +) + + def asm_pio(**kw): emit = PIOASMEmit(**kw) def dec(f): nonlocal emit - gl = _pio_funcs - gl["wrap_target"] = emit.wrap_target - gl["wrap"] = emit.wrap - gl["label"] = emit.label - gl["word"] = emit.word - gl["nop"] = emit.nop - gl["jmp"] = emit.jmp - gl["wait"] = emit.wait - gl["in_"] = emit.in_ - gl["out"] = emit.out - gl["push"] = emit.push - gl["pull"] = emit.pull - gl["mov"] = emit.mov - gl["irq"] = emit.irq - gl["set"] = emit.set - - old_gl = f.__globals__.copy() - f.__globals__.clear() - f.__globals__.update(gl) + gl = f.__globals__ + old_gl = gl.copy() + gl.clear() + + gl.update(_pio_funcs) + for name in _pio_directives: + gl[name] = getattr(emit, name) + for name in _pio_instructions: + gl[name] = getattr(emit, name) emit.start_pass(0) f() @@ -265,8 +262,8 @@ def dec(f): emit.start_pass(1) f() - f.__globals__.clear() - f.__globals__.update(old_gl) + gl.clear() + gl.update(old_gl) return emit.prog @@ -284,19 +281,15 @@ def asm_pio_encode(instr, sideset_count, sideset_opt=False): emit.num_sideset = 0 gl = _pio_funcs - gl["word"] = emit.word - gl["nop"] = emit.nop - # gl["jmp"] = emit.jmp currently not supported - gl["wait"] = emit.wait - gl["in_"] = emit.in_ - gl["out"] = emit.out - gl["push"] = emit.push - gl["pull"] = emit.pull - gl["mov"] = emit.mov - gl["irq"] = emit.irq - gl["set"] = emit.set - - exec(instr, gl) + for name in _pio_instructions: + gl[name] = getattr(emit, name) + gl["jmp"] = None # emit.jmp currently not supported + + try: + exec(instr, gl) + finally: + for name in _pio_instructions: + del gl[name] if len(emit.prog[_PROG_DATA]) != 1: raise PIOASMError("expecting exactly 1 instruction") From 12dd9cb74598bb0144cb7ac7dcbc5f5631372afa Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Wed, 29 Jan 2025 15:18:31 +1100 Subject: [PATCH 066/110] docs/esp32: Add documentation for SPI Ethernet devices on esp32 port. Also cross-link with the other WIZNET5K driver, to avoid confusion. This work was funded through GitHub Sponsors. Signed-off-by: Angus Gratton --- docs/esp32/quickref.rst | 110 ++++++++++++++++++++++++++---- docs/library/network.LAN.rst | 7 +- docs/library/network.WIZNET5K.rst | 3 + 3 files changed, 104 insertions(+), 16 deletions(-) diff --git a/docs/esp32/quickref.rst b/docs/esp32/quickref.rst index c5c2dfb38e570..0780e01300801 100644 --- a/docs/esp32/quickref.rst +++ b/docs/esp32/quickref.rst @@ -121,11 +121,20 @@ calling ``wlan.config(reconnects=n)``, where n are the number of desired reconne attempts (0 means it won't retry, -1 will restore the default behaviour of trying to reconnect forever). +.. _esp32_network_lan: + LAN ^^^ -To use the wired interfaces via :class:`network.LAN` one has to specify the pins -and mode :: +Built-in MAC (original ESP32) +""""""""""""""""""""""""""""" + +The original ESP32 SoC has a built-in Ethernet MAC. Using this MAC requires an +external Ethernet PHY to be wired to the chip's EMAC pins. Most of the EMAC pin +assignments are fixed, consult the ESP32 datasheet for details. + +If the PHY is connected, the internal Ethernet MAC can be configured via +the :class:`network.LAN` constructor:: import network @@ -134,20 +143,33 @@ and mode :: lan.ipconfig('addr4') # get the interface's IPv4 addresses -The keyword arguments for the constructor defining the PHY type and interface are: +Required keyword arguments for the constructor: + +- ``mdc`` and ``mdio`` - :class:`machine.Pin` objects (or integers) specifying + the MDC and MDIO pins. +- ``phy_type`` - Select the PHY device type. Supported devices are + ``PHY_LAN8710``, ``PHY_LAN8720``, ``PHY_IP101``, ``PHY_RTL8201``, + ``PHY_DP83848``, ``PHY_KSZ8041`` and ``PHY_KSZ8081``. These values are all + constants defined in the ``network`` module. +- ``phy_addr`` - The address number of the PHY device. Must be an integer in the + range 0x00 to 0x1f, inclusive. Common values are ``0`` and ``1``. + +All of the above keyword arguments must be present to configure the interface. + +Optional keyword arguments: -- mdc=pin-object # set the mdc and mdio pins. -- mdio=pin-object -- reset=pin-object # set the reset pin of the PHY device. -- power=pin-object # set the pin which switches the power of the PHY device. -- phy_type= # Select the PHY device type. Supported devices are PHY_LAN8710, - PHY_LAN8720, PH_IP101, PHY_RTL8201, PHY_DP83848 and PHY_KSZ8041 -- phy_addr=number # The address number of the PHY device. -- ref_clk_mode=mode # Defines, whether the ref_clk at the ESP32 is an input - or output. Suitable values are Pin.IN and Pin.OUT. -- ref_clk=pin-object # defines the Pin used for ref_clk. +- ``reset`` - :class:`machine.Pin` object (or integer) specifying the PHY reset pin. +- ``power`` - :class:`machine.Pin` object (or integer) specifying a pin which + switches the power of the PHY device. +- ``ref_clk`` - :class:`machine.Pin` object (or integer) specifying the pin used + for the EMAC ``ref_clk`` signal. If not specified, the board default is used + (typically GPIO 0, but may be different if a particular board has Ethernet.) +- ``ref_clk_mode`` - Defines whether the EMAC ``ref_clk`` pin of the ESP32 + should be an input or an output. Suitable values are ``machine.Pin.IN`` and + ``machine.Pin.OUT``. If not specified, the board default is used + (typically input, but may be different if a particular board has Ethernet.) -These are working configurations for LAN interfaces of popular boards:: +These are working configurations for LAN interfaces of some popular ESP32 boards:: # Olimex ESP32-GATEWAY: power controlled by Pin(5) # Olimex ESP32 PoE and ESP32-PoE ISO: power controlled by Pin(12) @@ -172,6 +194,66 @@ These are working configurations for LAN interfaces of popular boards:: lan = network.LAN(id=0, mdc=Pin(23), mdio=Pin(18), power=Pin(5), phy_type=network.PHY_IP101, phy_addr=1) + +.. _esp32_spi_ethernet: + +SPI Ethernet Interface +"""""""""""""""""""""" + +All ESP32 SoCs support external SPI Ethernet interface chips. These are Ethernet +interfaces that connect via a SPI bus, rather than an Ethernet RMII interface. + +.. note:: The only exception is the ESP32 ``d2wd`` variant, where this feature is disabled + to save code size. + +SPI Ethernet uses the same :class:`network.LAN` constructor, with a different +set of keyword arguments:: + + import machine, network + + spi = machine.SPI(1, sck=SCK_PIN, mosi=MOSI_PIN, miso=MISO_PIN) + lan = network.LAN(spi=spi, cs=CS_PIN, ...) # Set the pin and mode configuration + lan.active(True) # activate the interface + lan.ipconfig('addr4') # get the interface's IPv4 addresses + +Required keyword arguments for the constructor: + +- ``spi`` - Should be a :class:`machine.SPI` object configured for this + connection. Note that any clock speed configured on the SPI object is ignored, + the SPI Ethernet clock speed is configured at compile time. +- ``cs`` - :class:`machine.Pin` object (or integer) specifying the CS pin + connected to the interface. +- ``int`` - :class:`machine.Pin` object (or integer) specifying the INT pin + connected to the interface. +- ``phy_type`` - Select the SPI Ethernet interface type. Supported devices are + ``PHY_KSZ8851SNL``, ``PHY_DM9051``, ``PHY_W5500``. These values are all + constants defined in the ``network`` module. +- ``phy_addr`` - The address number of the PHY device. Must be an integer in the + range 0x00 to 0x1f, inclusive. This is usually ``0`` for SPI Ethernet devices. + +All of the above keyword arguments must be present to configure the interface. + +Optional keyword arguments for the constructor: + +- ``reset`` - :class:`machine.Pin` object (or integer) specifying the SPI Ethernet + interface reset pin. +- ``power`` - :class:`machine.Pin` object (or integer) specifying a pin which + switches the power of the SPI Ethernet interface. + +Here is a sample configuration for a WIZNet W5500 chip connected to pins on +an ESP32-S3 development board:: + + import machine, network + from machine import Pin, SPI + + spi = SPI(1, sck=Pin(12), mosi=Pin(13), miso=Pin(14)) + lan = network.LAN(spi=spi, phy_type=network.PHY_W5500, phy_addr=0, + cs=Pin(10), int=Pin(11)) + +.. note:: WIZnet W5500 Ethernet is also supported on some other MicroPython + ports, but using a :ref:`different software interface + `. + Delay and timing ---------------- diff --git a/docs/library/network.LAN.rst b/docs/library/network.LAN.rst index 31ce8e98411c2..1b3c19c560b36 100644 --- a/docs/library/network.LAN.rst +++ b/docs/library/network.LAN.rst @@ -6,7 +6,7 @@ class LAN -- control an Ethernet module This class allows you to control the Ethernet interface. The PHY hardware type is board-specific. -Example usage:: +Example usage, for a board with built-in LAN support:: import network nic = network.LAN(0) @@ -32,7 +32,7 @@ Constructors - *phy_addr* specifies the address of the PHY interface. As with *phy_type*, the hardwired value has to be used for most boards and that value is the default. - *ref_clk_mode* specifies, whether the data clock is provided by the Ethernet controller or - the PYH interface. + the PHY interface. The default value is the one that matches the board. If set to ``LAN.OUT`` or ``Pin.OUT`` or ``True``, the clock is driven by the Ethernet controller, if set to ``LAN.IN`` or ``Pin.IN`` or ``False``, the clock is driven by the PHY interface. @@ -41,6 +41,9 @@ Constructors nic = LAN(0, phy_type=LAN.PHY_LAN8720, phy_addr=1, ref_clk_mode=Pin.IN) + .. note:: On esp32 port the constructor requires different arguments. See + :ref:`esp32 port reference `. + Methods ------- diff --git a/docs/library/network.WIZNET5K.rst b/docs/library/network.WIZNET5K.rst index a19bd45282399..d0778a7a37ce8 100644 --- a/docs/library/network.WIZNET5K.rst +++ b/docs/library/network.WIZNET5K.rst @@ -9,6 +9,9 @@ the W5200 and W5500 chipsets. The particular chipset that is supported by the firmware is selected at compile-time via the MICROPY_PY_NETWORK_WIZNET5K option. +.. note:: The esp32 port also supports WIZnet W5500 chipsets, but this port + uses the :ref:`network.LAN interface `. + Example usage:: import network From bab099826e956bcc000b8a3c45b144c97d870fe2 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 28 Jan 2025 11:54:57 +1100 Subject: [PATCH 067/110] docs: Note which ports have default or optional network.PPP support. Also add the default values of these macros to the respective `mpconfigport.h` files, to improve discoverability. Signed-off-by: Angus Gratton --- docs/library/network.PPP.rst | 7 ++++++- ports/rp2/mpconfigport.h | 4 ++++ ports/stm32/mpconfigport.h | 14 +++++++++++--- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/docs/library/network.PPP.rst b/docs/library/network.PPP.rst index 17a8d1c1cd89d..15ab1da193448 100644 --- a/docs/library/network.PPP.rst +++ b/docs/library/network.PPP.rst @@ -5,7 +5,12 @@ class PPP -- create network connections over serial PPP ======================================================= This class allows you to create a network connection over a serial port using -the PPP protocol. It is only available on selected ports and boards. +the PPP protocol. + +.. note:: Currently only the esp32 port has PPP support enabled in the default + firmware build. PPP support can be enabled in custom builds of the + stm32 and rp2 ports by enabling networking support and setting + ``MICROPY_PY_NETWORK_PPP_LWIP`` to 1. Example usage:: diff --git a/ports/rp2/mpconfigport.h b/ports/rp2/mpconfigport.h index bc289bcbf97cf..fe8287ba1d6e5 100644 --- a/ports/rp2/mpconfigport.h +++ b/ports/rp2/mpconfigport.h @@ -212,6 +212,10 @@ #ifndef MICROPY_PY_WEBREPL #define MICROPY_PY_WEBREPL (1) #endif + +#ifndef MICROPY_PY_NETWORK_PPP_LWIP +#define MICROPY_PY_NETWORK_PPP_LWIP (0) +#endif #endif #if MICROPY_PY_NETWORK_CYW43 diff --git a/ports/stm32/mpconfigport.h b/ports/stm32/mpconfigport.h index dc0a767fb07ee..2b57446ee8233 100644 --- a/ports/stm32/mpconfigport.h +++ b/ports/stm32/mpconfigport.h @@ -146,9 +146,6 @@ #define MICROPY_HW_SOFTSPI_MAX_BAUDRATE (HAL_RCC_GetSysClockFreq() / 48) #define MICROPY_PY_WEBSOCKET (MICROPY_PY_LWIP) #define MICROPY_PY_WEBREPL (MICROPY_PY_LWIP) -#ifndef MICROPY_PY_SOCKET -#define MICROPY_PY_SOCKET (1) -#endif #ifndef MICROPY_PY_NETWORK #define MICROPY_PY_NETWORK (1) #endif @@ -156,6 +153,17 @@ #define MICROPY_PY_ONEWIRE (1) #endif +// optional network features +#if MICROPY_PY_NETWORK +#ifndef MICROPY_PY_SOCKET +#define MICROPY_PY_SOCKET (1) +#endif + +#ifndef MICROPY_PY_NETWORK_PPP_LWIP +#define MICROPY_PY_NETWORK_PPP_LWIP (0) +#endif +#endif + // fatfs configuration used in ffconf.h #define MICROPY_FATFS_ENABLE_LFN (2) #define MICROPY_FATFS_LFN_CODE_PAGE 437 /* 1=SFN/ANSI 437=LFN/U.S.(OEM) */ From 0a55f1f40c8a0a69f34f550d45138d9ba7a1d467 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 4 Feb 2025 12:20:38 +1100 Subject: [PATCH 068/110] docs/reference: Add strings vs bytes to speed optimisation tips. Also add some additional context links, suggestions for alternative classes, etc. This work was funded through GitHub Sponsors. Signed-off-by: Angus Gratton --- docs/library/array.rst | 4 ++++ docs/library/builtins.rst | 4 ++++ docs/reference/speed_python.rst | 33 ++++++++++++++++++++++++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/docs/library/array.rst b/docs/library/array.rst index f417a7046e2ff..957260c2c7f92 100644 --- a/docs/library/array.rst +++ b/docs/library/array.rst @@ -19,6 +19,10 @@ Classes array are given by *iterable*. If it is not provided, an empty array is created. + In addition to the methods below, array objects also implement the buffer + protocol. This means the contents of the entire array can be accessed as raw + bytes via a `memoryview` or other interfaces which use this protocol. + .. method:: append(val) Append new element *val* to the end of array, growing it. diff --git a/docs/library/builtins.rst b/docs/library/builtins.rst index 5956aea7ab20f..b5d08ba7fed50 100644 --- a/docs/library/builtins.rst +++ b/docs/library/builtins.rst @@ -19,6 +19,8 @@ Functions and types .. class:: bytearray() + |see_cpython| `python:bytearray`. + .. class:: bytes() |see_cpython| `python:bytes`. @@ -104,6 +106,8 @@ Functions and types .. class:: memoryview() + |see_cpython| `python:memoryview`. + .. function:: min() .. function:: next() diff --git a/docs/reference/speed_python.rst b/docs/reference/speed_python.rst index 0c68dd60bdedc..a660432f6da37 100644 --- a/docs/reference/speed_python.rst +++ b/docs/reference/speed_python.rst @@ -57,6 +57,8 @@ and used in various methods. This is covered in further detail :ref:`Controlling garbage collection ` below. +.. _speed_buffers: + Buffers ~~~~~~~ @@ -69,6 +71,13 @@ example, objects which support stream interface (e.g., file or UART) provide ``r method which allocates new buffer for read data, but also a ``readinto()`` method to read data into an existing buffer. +Some useful classes for creating reusable buffer objects: + +- :class:`bytearray` +- :mod:`array` (:ref:`discussed below`) +- :class:`io.StringIO` and :class:`io.BytesIO` +- :class:`micropython.RingIO` + Floating point ~~~~~~~~~~~~~~ @@ -80,15 +89,20 @@ point to sections of the code where performance is not paramount. For example, capture ADC readings as integers values to an array in one quick go, and only then convert them to floating-point numbers for signal processing. +.. _speed_arrays: + Arrays ~~~~~~ Consider the use of the various types of array classes as an alternative to lists. -The `array` module supports various element types with 8-bit elements supported +The :mod:`array` module supports various element types with 8-bit elements supported by Python's built in `bytes` and `bytearray` classes. These data structures all store elements in contiguous memory locations. Once again to avoid memory allocation in critical code these should be pre-allocated and passed as arguments or as bound objects. +Memoryviews +~~~~~~~~~~~ + When passing slices of objects such as `bytearray` instances, Python creates a copy which involves allocation of the size proportional to the size of slice. This can be alleviated using a `memoryview` object. The `memoryview` itself @@ -118,6 +132,23 @@ of buffer and fills in entire buffer. What if you need to put data in the middle of existing buffer? Just create a memoryview into the needed section of buffer and pass it to ``readinto()``. +Strings vs Bytes +~~~~~~~~~~~~~~~~ + +MicroPython uses :ref:`string interning ` to save space when there are +multiple identical strings. Each time a new string is allocated at runtime (for +example, when two other strings are concatenated), MicroPython checks whether +the new string can be interned to save RAM. + +If you have code which performs performance-critical string operations then +consider using :class:`bytes` objects and literals (i.e. ``b"abc"``). This skips +the interning check, and can be several times faster than performing the same +operations with string objects. + +.. note:: The fastest performance will always be achieved by avoiding new object + creation entirely, for example with a reusable :ref:`buffer as described + above`. + Identifying the slowest section of code --------------------------------------- From 3b6252466139d5dfc7458eeea5d759b6c9850146 Mon Sep 17 00:00:00 2001 From: Alessandro Gatti Date: Mon, 10 Feb 2025 17:20:29 +0100 Subject: [PATCH 069/110] docs/library/espnow: Clarify usage of the "rate" configuration key. This commit adds a clarification for the ESPNow module's documentation regarding its "config" method. The original documentation for that method could be interpreted as having all its configuration keys being able to be queried, but the "rate" configuration key is actually write-only due to ESP-IDF's lack of a way to retrieve that bit of information from the radio's configuration. The documentation changes highlight the fact that said configuration key is actually write-only. Signed-off-by: Alessandro Gatti --- docs/library/espnow.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/library/espnow.rst b/docs/library/espnow.rst index 379e60486f224..84e9e9465de96 100644 --- a/docs/library/espnow.rst +++ b/docs/library/espnow.rst @@ -164,11 +164,13 @@ Configuration wait forever. The timeout can also be provided as arg to `recv()`/`irecv()`/`recvinto()`. - *rate*: (ESP32 only, IDF>=4.3.0 only) Set the transmission speed for + *rate*: (ESP32 only) Set the transmission speed for ESPNow packets. Must be set to a number from the allowed numeric values in `enum wifi_phy_rate_t - `_. + `_. This + parameter is actually *write-only* due to ESP-IDF not providing any + means for querying the radio interface's rate parameter. .. data:: Returns: From 372ecfef02eccc4e52a5d0abef068c7d25fc4315 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 21 Jan 2025 00:56:59 +1100 Subject: [PATCH 070/110] tests/run-tests.py: Give more information when CPython crashes. To make it easier to diagnose why CPython crashed. Signed-off-by: Damien George --- tests/run-tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/run-tests.py b/tests/run-tests.py index 1df1b35d23b83..b108b16395074 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -955,8 +955,8 @@ def run_one_test(test_file): cwd=os.path.dirname(test_file), stderr=subprocess.STDOUT, ) - except subprocess.CalledProcessError: - output_expected = b"CPYTHON3 CRASH" + except subprocess.CalledProcessError as er: + output_expected = b"CPYTHON3 CRASH:\n" + er.output # Canonical form for all host platforms is to use \n for end-of-line. output_expected = output_expected.replace(b"\r\n", b"\n") From 62e821ccb82fd8362a8198ad59ccb51b8a5c441e Mon Sep 17 00:00:00 2001 From: Damien George Date: Sun, 19 Jan 2025 23:30:59 +1100 Subject: [PATCH 071/110] py/objcode: Factor code object out into its own file. The `mp_obj_code_t` and `mp_type_code` code object was defined internally in both `py/builtinevex.c` and `py/profile.c`, with completely different implementations (the former very minimal, the latter quite complete). This commit factors these implementations into a new, separate source file, and allows the code object to have four different modes, selected at compile-time: - MICROPY_PY_BUILTINS_CODE_NONE: code object not included in the build. - MICROPY_PY_BUILTINS_CODE_MINIMUM: very simple code object that just holds a reference to the function that it represents. This level is used when MICROPY_PY_BUILTINS_COMPILE is enabled. - MICROPY_PY_BUILTINS_CODE_BASIC: simple code object that holds a reference to the proto-function and its constants. - MICROPY_PY_BUILTINS_CODE_FULL: almost complete implementation of the code object. This level is used when MICROPY_PY_SYS_SETTRACE is enabled. Signed-off-by: Damien George --- py/builtinevex.c | 51 +++++++++----- py/compile.c | 2 +- py/compile.h | 5 +- py/mpconfig.h | 9 +++ py/obj.h | 1 + py/objcode.c | 175 +++++++++++++++++++++++++++++++++++++++++++++++ py/objcode.h | 99 +++++++++++++++++++++++++++ py/profile.c | 140 ++----------------------------------- py/profile.h | 12 +--- py/py.cmake | 1 + py/py.mk | 1 + py/runtime.c | 7 +- 12 files changed, 337 insertions(+), 166 deletions(-) create mode 100644 py/objcode.c create mode 100644 py/objcode.h diff --git a/py/builtinevex.c b/py/builtinevex.c index e25cbd4d08502..74a4640492674 100644 --- a/py/builtinevex.c +++ b/py/builtinevex.c @@ -26,6 +26,7 @@ #include +#include "py/objcode.h" #include "py/objfun.h" #include "py/compile.h" #include "py/runtime.h" @@ -33,17 +34,6 @@ #if MICROPY_PY_BUILTINS_COMPILE -typedef struct _mp_obj_code_t { - mp_obj_base_t base; - mp_obj_t module_fun; -} mp_obj_code_t; - -static MP_DEFINE_CONST_OBJ_TYPE( - mp_type_code, - MP_QSTR_code, - MP_TYPE_FLAG_NONE - ); - static mp_obj_t code_execute(mp_obj_code_t *self, mp_obj_dict_t *globals, mp_obj_dict_t *locals) { // save context nlr_jump_callback_node_globals_locals_t ctx; @@ -57,19 +47,28 @@ static mp_obj_t code_execute(mp_obj_code_t *self, mp_obj_dict_t *globals, mp_obj // set exception handler to restore context if an exception is raised nlr_push_jump_callback(&ctx.callback, mp_globals_locals_set_from_nlr_jump_callback); + #if MICROPY_PY_BUILTINS_CODE >= MICROPY_PY_BUILTINS_CODE_BASIC + mp_module_context_t *module_context = m_new_obj(mp_module_context_t); + module_context->module.base.type = &mp_type_module; + module_context->module.globals = globals; + module_context->constants = *mp_code_get_constants(self); + mp_obj_t module_fun = mp_make_function_from_proto_fun(mp_code_get_proto_fun(self), module_context, NULL); + #else // The call to mp_parse_compile_execute() in mp_builtin_compile() below passes // NULL for the globals, so repopulate that entry now with the correct globals. + mp_obj_t module_fun = self->module_fun; if (mp_obj_is_type(self->module_fun, &mp_type_fun_bc) #if MICROPY_EMIT_NATIVE || mp_obj_is_type(self->module_fun, &mp_type_fun_native) #endif ) { - mp_obj_fun_bc_t *fun_bc = MP_OBJ_TO_PTR(self->module_fun); + mp_obj_fun_bc_t *fun_bc = MP_OBJ_TO_PTR(module_fun); ((mp_module_context_t *)fun_bc->context)->module.globals = globals; } + #endif // execute code - mp_obj_t ret = mp_call_function_0(self->module_fun); + mp_obj_t ret = mp_call_function_0(module_fun); // deregister exception handler and restore context nlr_pop_jump_callback(true); @@ -108,9 +107,29 @@ static mp_obj_t mp_builtin_compile(size_t n_args, const mp_obj_t *args) { mp_raise_ValueError(MP_ERROR_TEXT("bad compile mode")); } - mp_obj_code_t *code = mp_obj_malloc(mp_obj_code_t, &mp_type_code); - code->module_fun = mp_parse_compile_execute(lex, parse_input_kind, NULL, NULL); - return MP_OBJ_FROM_PTR(code); + #if MICROPY_PY_BUILTINS_CODE >= MICROPY_PY_BUILTINS_CODE_BASIC + + mp_parse_tree_t parse_tree = mp_parse(lex, parse_input_kind); + mp_module_context_t ctx; + ctx.module.globals = NULL; + mp_compiled_module_t cm; + cm.context = &ctx; + mp_compile_to_raw_code(&parse_tree, lex->source_name, parse_input_kind == MP_PARSE_SINGLE_INPUT, &cm); + + #if MICROPY_PY_BUILTINS_CODE >= MICROPY_PY_BUILTINS_CODE_FULL + mp_module_context_t *ctx_ptr = m_new_obj(mp_module_context_t); + *ctx_ptr = ctx; + return mp_obj_new_code(ctx_ptr, cm.rc, true); + #else + return mp_obj_new_code(ctx.constants, cm.rc); + #endif + + #else + + mp_obj_t module_fun = mp_parse_compile_execute(lex, parse_input_kind, NULL, NULL); + return mp_obj_new_code(module_fun); + + #endif } MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_builtin_compile_obj, 3, 6, mp_builtin_compile); diff --git a/py/compile.c b/py/compile.c index 60f06d7773e34..7a1151bcd66f0 100644 --- a/py/compile.c +++ b/py/compile.c @@ -3467,7 +3467,7 @@ static void scope_compute_things(scope_t *scope) { } } -#if !MICROPY_PERSISTENT_CODE_SAVE +#if !MICROPY_EXPOSE_MP_COMPILE_TO_RAW_CODE static #endif void mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl, mp_compiled_module_t *cm) { diff --git a/py/compile.h b/py/compile.h index f9970a521d644..64afc487d7c3d 100644 --- a/py/compile.h +++ b/py/compile.h @@ -30,6 +30,9 @@ #include "py/parse.h" #include "py/emitglue.h" +// Whether mp_compile_to_raw_code is exposed as a public function. +#define MICROPY_EXPOSE_MP_COMPILE_TO_RAW_CODE (MICROPY_PY_BUILTINS_CODE >= MICROPY_PY_BUILTINS_CODE_BASIC || MICROPY_PERSISTENT_CODE_SAVE) + #if MICROPY_COMP_ALLOW_TOP_LEVEL_AWAIT // set to `true` to allow top-level await expressions extern bool mp_compile_allow_top_level_await; @@ -40,7 +43,7 @@ extern bool mp_compile_allow_top_level_await; // mp_globals_get() will be used for the context mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl); -#if MICROPY_PERSISTENT_CODE_SAVE +#if MICROPY_EXPOSE_MP_COMPILE_TO_RAW_CODE // this has the same semantics as mp_compile void mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl, mp_compiled_module_t *cm); #endif diff --git a/py/mpconfig.h b/py/mpconfig.h index 64138a9ea7acc..76aff4681d32a 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -1129,6 +1129,15 @@ typedef double mp_float_t; #define MICROPY_PY_BUILTINS_BYTEARRAY (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) #endif +// Whether to support code objects, and how many features they have +#define MICROPY_PY_BUILTINS_CODE_NONE (0) +#define MICROPY_PY_BUILTINS_CODE_MINIMUM (1) +#define MICROPY_PY_BUILTINS_CODE_BASIC (2) +#define MICROPY_PY_BUILTINS_CODE_FULL (3) +#ifndef MICROPY_PY_BUILTINS_CODE +#define MICROPY_PY_BUILTINS_CODE (MICROPY_PY_SYS_SETTRACE ? MICROPY_PY_BUILTINS_CODE_FULL : (MICROPY_PY_BUILTINS_COMPILE ? MICROPY_PY_BUILTINS_CODE_MINIMUM : MICROPY_PY_BUILTINS_CODE_NONE)) +#endif + // Whether to support dict.fromkeys() class method #ifndef MICROPY_PY_BUILTINS_DICT_FROMKEYS #define MICROPY_PY_BUILTINS_DICT_FROMKEYS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) diff --git a/py/obj.h b/py/obj.h index bd31ef11d0928..93d1e0ee34349 100644 --- a/py/obj.h +++ b/py/obj.h @@ -840,6 +840,7 @@ extern const mp_obj_type_t mp_type_fun_bc; extern const mp_obj_type_t mp_type_fun_native; extern const mp_obj_type_t mp_type_fun_viper; extern const mp_obj_type_t mp_type_fun_asm; +extern const mp_obj_type_t mp_type_code; extern const mp_obj_type_t mp_type_module; extern const mp_obj_type_t mp_type_staticmethod; extern const mp_obj_type_t mp_type_classmethod; diff --git a/py/objcode.c b/py/objcode.c new file mode 100644 index 0000000000000..9b98a696798d4 --- /dev/null +++ b/py/objcode.c @@ -0,0 +1,175 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2025 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "py/objcode.h" + +#if MICROPY_PY_BUILTINS_CODE == MICROPY_PY_BUILTINS_CODE_NONE + +// Code object not implemented at this configuration level. + +#elif MICROPY_PY_BUILTINS_CODE <= MICROPY_PY_BUILTINS_CODE_BASIC + +MP_DEFINE_CONST_OBJ_TYPE( + mp_type_code, + MP_QSTR_code, + MP_TYPE_FLAG_NONE + ); + +#elif MICROPY_PY_BUILTINS_CODE <= MICROPY_PY_BUILTINS_CODE_FULL + +#include "py/profile.h" + +static void code_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { + (void)kind; + mp_obj_code_t *o = MP_OBJ_TO_PTR(o_in); + const mp_raw_code_t *rc = o->rc; + const mp_bytecode_prelude_t *prelude = &rc->prelude; + mp_printf(print, + "", + MP_CODE_QSTR_MAP(o->context, prelude->qstr_block_name_idx), + o, + MP_CODE_QSTR_MAP(o->context, 0), + rc->line_of_definition + ); +} + +static mp_obj_tuple_t *code_consts(const mp_module_context_t *context, const mp_raw_code_t *rc) { + mp_obj_tuple_t *consts = MP_OBJ_TO_PTR(mp_obj_new_tuple(rc->n_children + 1, NULL)); + + size_t const_no = 0; + for (size_t i = 0; i < rc->n_children; ++i) { + mp_obj_t code = mp_obj_new_code(context, rc->children[i], true); + consts->items[const_no++] = code; + } + consts->items[const_no++] = mp_const_none; + + return consts; +} + +static mp_obj_t raw_code_lnotab(const mp_raw_code_t *rc) { + // const mp_bytecode_prelude_t *prelude = &rc->prelude; + uint start = 0; + uint stop = rc->fun_data_len - start; + + uint last_lineno = mp_prof_bytecode_lineno(rc, start); + uint lasti = 0; + + const uint buffer_chunk_size = (stop - start) >> 2; // heuristic magic + uint buffer_size = buffer_chunk_size; + byte *buffer = m_new(byte, buffer_size); + uint buffer_index = 0; + + for (uint i = start; i < stop; ++i) { + uint lineno = mp_prof_bytecode_lineno(rc, i); + size_t line_diff = lineno - last_lineno; + if (line_diff > 0) { + uint instr_diff = (i - start) - lasti; + + assert(instr_diff < 256); + assert(line_diff < 256); + + if (buffer_index + 2 > buffer_size) { + buffer = m_renew(byte, buffer, buffer_size, buffer_size + buffer_chunk_size); + buffer_size = buffer_size + buffer_chunk_size; + } + last_lineno = lineno; + lasti = i - start; + buffer[buffer_index++] = instr_diff; + buffer[buffer_index++] = line_diff; + } + } + + mp_obj_t o = mp_obj_new_bytes(buffer, buffer_index); + m_del(byte, buffer, buffer_size); + return o; +} + +static void code_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { + if (dest[0] != MP_OBJ_NULL) { + // not load attribute + return; + } + mp_obj_code_t *o = MP_OBJ_TO_PTR(self_in); + const mp_raw_code_t *rc = o->rc; + const mp_bytecode_prelude_t *prelude = &rc->prelude; + switch (attr) { + case MP_QSTR_co_code: + dest[0] = mp_obj_new_bytes( + (void *)prelude->opcodes, + rc->fun_data_len - (prelude->opcodes - (const byte *)rc->fun_data) + ); + break; + case MP_QSTR_co_consts: + dest[0] = MP_OBJ_FROM_PTR(code_consts(o->context, rc)); + break; + case MP_QSTR_co_filename: + dest[0] = MP_OBJ_NEW_QSTR(MP_CODE_QSTR_MAP(o->context, 0)); + break; + case MP_QSTR_co_firstlineno: + dest[0] = MP_OBJ_NEW_SMALL_INT(mp_prof_bytecode_lineno(rc, 0)); + break; + case MP_QSTR_co_name: + dest[0] = MP_OBJ_NEW_QSTR(MP_CODE_QSTR_MAP(o->context, prelude->qstr_block_name_idx)); + break; + case MP_QSTR_co_names: + dest[0] = MP_OBJ_FROM_PTR(o->dict_locals); + break; + case MP_QSTR_co_lnotab: + if (!o->lnotab) { + o->lnotab = raw_code_lnotab(rc); + } + dest[0] = o->lnotab; + break; + } +} + +MP_DEFINE_CONST_OBJ_TYPE( + mp_type_code, + MP_QSTR_code, + MP_TYPE_FLAG_NONE, + print, code_print, + attr, code_attr + ); + +mp_obj_t mp_obj_new_code(const mp_module_context_t *context, const mp_raw_code_t *rc, bool result_required) { + mp_obj_code_t *o; + if (result_required) { + o = m_new_obj(mp_obj_code_t); + } else { + o = m_new_obj_maybe(mp_obj_code_t); + if (o == NULL) { + return MP_OBJ_NULL; + } + } + o->base.type = &mp_type_code; + o->context = context; + o->rc = rc; + o->dict_locals = mp_locals_get(); // this is a wrong! how to do this properly? + o->lnotab = MP_OBJ_NULL; + return MP_OBJ_FROM_PTR(o); +} + +#endif diff --git a/py/objcode.h b/py/objcode.h new file mode 100644 index 0000000000000..8db9a34b6e1c9 --- /dev/null +++ b/py/objcode.h @@ -0,0 +1,99 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2025 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef MICROPY_INCLUDED_PY_OBJCODE_H +#define MICROPY_INCLUDED_PY_OBJCODE_H + +#include "py/bc.h" + +#if MICROPY_PY_BUILTINS_CODE == MICROPY_PY_BUILTINS_CODE_NONE + +// Code object not implemented at this configuration level. + +#elif MICROPY_PY_BUILTINS_CODE <= MICROPY_PY_BUILTINS_CODE_MINIMUM + +typedef struct _mp_obj_code_t { + mp_obj_base_t base; + mp_obj_t module_fun; +} mp_obj_code_t; + +static inline mp_obj_t mp_obj_new_code(mp_obj_t module_fun) { + mp_obj_code_t *code = mp_obj_malloc(mp_obj_code_t, &mp_type_code); + code->module_fun = module_fun; + return MP_OBJ_FROM_PTR(code); +} + +#elif MICROPY_PY_BUILTINS_CODE <= MICROPY_PY_BUILTINS_CODE_BASIC + +typedef struct _mp_obj_code_t { + mp_obj_base_t base; + mp_module_constants_t constants; + const void *proto_fun; +} mp_obj_code_t; + +static inline mp_obj_t mp_obj_new_code(const mp_module_constants_t constants, const void *proto_fun) { + mp_obj_code_t *code = mp_obj_malloc(mp_obj_code_t, &mp_type_code); + code->constants = constants; + code->proto_fun = proto_fun; + return MP_OBJ_FROM_PTR(code); +} + +static inline const mp_module_constants_t *mp_code_get_constants(mp_obj_code_t *self) { + return &self->constants; +} + +static inline const void *mp_code_get_proto_fun(mp_obj_code_t *self) { + return self->proto_fun; +} + +#elif MICROPY_PY_BUILTINS_CODE <= MICROPY_PY_BUILTINS_CODE_FULL + +#include "py/emitglue.h" + +#define MP_CODE_QSTR_MAP(context, idx) (context->constants.qstr_table[idx]) + +typedef struct _mp_obj_code_t { + // TODO this was 4 words + mp_obj_base_t base; + const mp_module_context_t *context; + const mp_raw_code_t *rc; + mp_obj_dict_t *dict_locals; + mp_obj_t lnotab; +} mp_obj_code_t; + +mp_obj_t mp_obj_new_code(const mp_module_context_t *context, const mp_raw_code_t *rc, bool result_required); + +static inline const mp_module_constants_t *mp_code_get_constants(mp_obj_code_t *self) { + return &self->context->constants; +} + +static inline const void *mp_code_get_proto_fun(mp_obj_code_t *self) { + // A mp_raw_code_t is always a proto_fun (but not the other way around). + return self->rc; +} + +#endif + +#endif // MICROPY_INCLUDED_PY_OBJCODE_H diff --git a/py/profile.c b/py/profile.c index 92f414ace7c92..397d0291f9fad 100644 --- a/py/profile.c +++ b/py/profile.c @@ -38,9 +38,8 @@ #endif #define prof_trace_cb MP_STATE_THREAD(prof_trace_callback) -#define QSTR_MAP(context, idx) (context->constants.qstr_table[idx]) -static uint mp_prof_bytecode_lineno(const mp_raw_code_t *rc, size_t bc) { +uint mp_prof_bytecode_lineno(const mp_raw_code_t *rc, size_t bc) { const mp_bytecode_prelude_t *prelude = &rc->prelude; return mp_bytecode_get_source_line(prelude->line_info, prelude->line_info_top, bc); } @@ -68,137 +67,6 @@ void mp_prof_extract_prelude(const byte *bytecode, mp_bytecode_prelude_t *prelud prelude->line_info = ip; } -/******************************************************************************/ -// code object - -static void code_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { - (void)kind; - mp_obj_code_t *o = MP_OBJ_TO_PTR(o_in); - const mp_raw_code_t *rc = o->rc; - const mp_bytecode_prelude_t *prelude = &rc->prelude; - mp_printf(print, - "", - QSTR_MAP(o->context, prelude->qstr_block_name_idx), - o, - QSTR_MAP(o->context, 0), - rc->line_of_definition - ); -} - -static mp_obj_tuple_t *code_consts(const mp_module_context_t *context, const mp_raw_code_t *rc) { - mp_obj_tuple_t *consts = MP_OBJ_TO_PTR(mp_obj_new_tuple(rc->n_children + 1, NULL)); - - size_t const_no = 0; - for (size_t i = 0; i < rc->n_children; ++i) { - mp_obj_t code = mp_obj_new_code(context, rc->children[i]); - if (code == MP_OBJ_NULL) { - m_malloc_fail(sizeof(mp_obj_code_t)); - } - consts->items[const_no++] = code; - } - consts->items[const_no++] = mp_const_none; - - return consts; -} - -static mp_obj_t raw_code_lnotab(const mp_raw_code_t *rc) { - // const mp_bytecode_prelude_t *prelude = &rc->prelude; - uint start = 0; - uint stop = rc->fun_data_len - start; - - uint last_lineno = mp_prof_bytecode_lineno(rc, start); - uint lasti = 0; - - const uint buffer_chunk_size = (stop - start) >> 2; // heuristic magic - uint buffer_size = buffer_chunk_size; - byte *buffer = m_new(byte, buffer_size); - uint buffer_index = 0; - - for (uint i = start; i < stop; ++i) { - uint lineno = mp_prof_bytecode_lineno(rc, i); - size_t line_diff = lineno - last_lineno; - if (line_diff > 0) { - uint instr_diff = (i - start) - lasti; - - assert(instr_diff < 256); - assert(line_diff < 256); - - if (buffer_index + 2 > buffer_size) { - buffer = m_renew(byte, buffer, buffer_size, buffer_size + buffer_chunk_size); - buffer_size = buffer_size + buffer_chunk_size; - } - last_lineno = lineno; - lasti = i - start; - buffer[buffer_index++] = instr_diff; - buffer[buffer_index++] = line_diff; - } - } - - mp_obj_t o = mp_obj_new_bytes(buffer, buffer_index); - m_del(byte, buffer, buffer_size); - return o; -} - -static void code_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { - if (dest[0] != MP_OBJ_NULL) { - // not load attribute - return; - } - mp_obj_code_t *o = MP_OBJ_TO_PTR(self_in); - const mp_raw_code_t *rc = o->rc; - const mp_bytecode_prelude_t *prelude = &rc->prelude; - switch (attr) { - case MP_QSTR_co_code: - dest[0] = mp_obj_new_bytes( - (void *)prelude->opcodes, - rc->fun_data_len - (prelude->opcodes - (const byte *)rc->fun_data) - ); - break; - case MP_QSTR_co_consts: - dest[0] = MP_OBJ_FROM_PTR(code_consts(o->context, rc)); - break; - case MP_QSTR_co_filename: - dest[0] = MP_OBJ_NEW_QSTR(QSTR_MAP(o->context, 0)); - break; - case MP_QSTR_co_firstlineno: - dest[0] = MP_OBJ_NEW_SMALL_INT(mp_prof_bytecode_lineno(rc, 0)); - break; - case MP_QSTR_co_name: - dest[0] = MP_OBJ_NEW_QSTR(QSTR_MAP(o->context, prelude->qstr_block_name_idx)); - break; - case MP_QSTR_co_names: - dest[0] = MP_OBJ_FROM_PTR(o->dict_locals); - break; - case MP_QSTR_co_lnotab: - if (!o->lnotab) { - o->lnotab = raw_code_lnotab(rc); - } - dest[0] = o->lnotab; - break; - } -} - -MP_DEFINE_CONST_OBJ_TYPE( - mp_type_settrace_codeobj, - MP_QSTR_code, - MP_TYPE_FLAG_NONE, - print, code_print, - attr, code_attr - ); - -mp_obj_t mp_obj_new_code(const mp_module_context_t *context, const mp_raw_code_t *rc) { - mp_obj_code_t *o = m_new_obj_maybe(mp_obj_code_t); - if (o == NULL) { - return MP_OBJ_NULL; - } - o->base.type = &mp_type_settrace_codeobj; - o->context = context; - o->rc = rc; - o->dict_locals = mp_locals_get(); // this is a wrong! how to do this properly? - o->lnotab = MP_OBJ_NULL; - return MP_OBJ_FROM_PTR(o); -} - /******************************************************************************/ // frame object @@ -211,9 +79,9 @@ static void frame_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t mp_printf(print, "", frame, - QSTR_MAP(code->context, 0), + MP_CODE_QSTR_MAP(code->context, 0), frame->lineno, - QSTR_MAP(code->context, prelude->qstr_block_name_idx) + MP_CODE_QSTR_MAP(code->context, prelude->qstr_block_name_idx) ); } @@ -265,7 +133,7 @@ mp_obj_t mp_obj_new_frame(const mp_code_state_t *code_state) { return MP_OBJ_NULL; } - mp_obj_code_t *code = o->code = MP_OBJ_TO_PTR(mp_obj_new_code(code_state->fun_bc->context, code_state->fun_bc->rc)); + mp_obj_code_t *code = o->code = MP_OBJ_TO_PTR(mp_obj_new_code(code_state->fun_bc->context, code_state->fun_bc->rc, false)); if (code == NULL) { return MP_OBJ_NULL; } diff --git a/py/profile.h b/py/profile.h index 7f3f914034623..db72b9f076818 100644 --- a/py/profile.h +++ b/py/profile.h @@ -28,20 +28,12 @@ #define MICROPY_INCLUDED_PY_PROFILING_H #include "py/emitglue.h" +#include "py/objcode.h" #if MICROPY_PY_SYS_SETTRACE #define mp_prof_is_executing MP_STATE_THREAD(prof_callback_is_executing) -typedef struct _mp_obj_code_t { - // TODO this was 4 words - mp_obj_base_t base; - const mp_module_context_t *context; - const mp_raw_code_t *rc; - mp_obj_dict_t *dict_locals; - mp_obj_t lnotab; -} mp_obj_code_t; - typedef struct _mp_obj_frame_t { mp_obj_base_t base; const mp_code_state_t *code_state; @@ -53,9 +45,9 @@ typedef struct _mp_obj_frame_t { bool trace_opcodes; } mp_obj_frame_t; +uint mp_prof_bytecode_lineno(const mp_raw_code_t *rc, size_t bc); void mp_prof_extract_prelude(const byte *bytecode, mp_bytecode_prelude_t *prelude); -mp_obj_t mp_obj_new_code(const mp_module_context_t *mc, const mp_raw_code_t *rc); mp_obj_t mp_obj_new_frame(const mp_code_state_t *code_state); // This is the implementation for the sys.settrace diff --git a/py/py.cmake b/py/py.cmake index 0fee74ddcb489..1c81ed4c58f32 100644 --- a/py/py.cmake +++ b/py/py.cmake @@ -72,6 +72,7 @@ set(MICROPY_SOURCE_PY ${MICROPY_PY_DIR}/objattrtuple.c ${MICROPY_PY_DIR}/objbool.c ${MICROPY_PY_DIR}/objboundmeth.c + ${MICROPY_PY_DIR}/objcode.c ${MICROPY_PY_DIR}/objcell.c ${MICROPY_PY_DIR}/objclosure.c ${MICROPY_PY_DIR}/objcomplex.c diff --git a/py/py.mk b/py/py.mk index c0b7e1ac8b90e..e352d89792bfd 100644 --- a/py/py.mk +++ b/py/py.mk @@ -149,6 +149,7 @@ PY_CORE_O_BASENAME = $(addprefix py/,\ objboundmeth.o \ objcell.o \ objclosure.o \ + objcode.o \ objcomplex.o \ objdeque.o \ objdict.o \ diff --git a/py/runtime.c b/py/runtime.c index deb55bf283abc..bf26921f9ab9f 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -1620,10 +1620,13 @@ mp_obj_t mp_parse_compile_execute(mp_lexer_t *lex, mp_parse_input_kind_t parse_i mp_obj_t module_fun = mp_compile(&parse_tree, source_name, parse_input_kind == MP_PARSE_SINGLE_INPUT); mp_obj_t ret; - if (MICROPY_PY_BUILTINS_COMPILE && globals == NULL) { + #if MICROPY_PY_BUILTINS_COMPILE && MICROPY_PY_BUILTINS_CODE == MICROPY_PY_BUILTINS_CODE_MINIMUM + if (globals == NULL) { // for compile only, return value is the module function ret = module_fun; - } else { + } else + #endif + { // execute module function and get return value ret = mp_call_function_0(module_fun); } From ceb8ba60b4fea0c32e4977d0e45d5c0203b27b34 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 20 Jan 2025 22:23:48 +1100 Subject: [PATCH 072/110] py/objfun: Implement function.__code__ and function constructor. This allows retrieving the code object of a function using `function.__code__`, and then reconstructing a function from a code object using `FunctionType(code_object)`. This feature is controlled by `MICROPY_PY_FUNCTION_ATTRS_CODE` and is enabled at the full-features level. Signed-off-by: Damien George --- py/mpconfig.h | 7 ++- py/objfun.c | 47 +++++++++++++++++++ tests/basics/fun_code.py | 36 ++++++++++++++ tests/basics/fun_code_micropython.py | 19 ++++++++ tests/basics/fun_code_micropython.py.exp | 1 + tests/basics/subclass_native1.py | 6 +-- tests/micropython/native_fun_attrs_code.py | 21 +++++++++ .../micropython/native_fun_attrs_code.py.exp | 1 + 8 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 tests/basics/fun_code.py create mode 100644 tests/basics/fun_code_micropython.py create mode 100644 tests/basics/fun_code_micropython.py.exp create mode 100644 tests/micropython/native_fun_attrs_code.py create mode 100644 tests/micropython/native_fun_attrs_code.py.exp diff --git a/py/mpconfig.h b/py/mpconfig.h index 76aff4681d32a..5c4d19bb56f11 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -1041,6 +1041,11 @@ typedef double mp_float_t; #define MICROPY_PY_FUNCTION_ATTRS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif +// Whether to implement the __code__ attribute on functions, and function constructor +#ifndef MICROPY_PY_FUNCTION_ATTRS_CODE +#define MICROPY_PY_FUNCTION_ATTRS_CODE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_FULL_FEATURES) +#endif + // Whether to support the descriptors __get__, __set__, __delete__ // This costs some code size and makes load/store/delete of instance // attributes slower for the classes that use this feature @@ -1135,7 +1140,7 @@ typedef double mp_float_t; #define MICROPY_PY_BUILTINS_CODE_BASIC (2) #define MICROPY_PY_BUILTINS_CODE_FULL (3) #ifndef MICROPY_PY_BUILTINS_CODE -#define MICROPY_PY_BUILTINS_CODE (MICROPY_PY_SYS_SETTRACE ? MICROPY_PY_BUILTINS_CODE_FULL : (MICROPY_PY_BUILTINS_COMPILE ? MICROPY_PY_BUILTINS_CODE_MINIMUM : MICROPY_PY_BUILTINS_CODE_NONE)) +#define MICROPY_PY_BUILTINS_CODE (MICROPY_PY_SYS_SETTRACE ? MICROPY_PY_BUILTINS_CODE_FULL : (MICROPY_PY_FUNCTION_ATTRS_CODE ? MICROPY_PY_BUILTINS_CODE_BASIC : (MICROPY_PY_BUILTINS_COMPILE ? MICROPY_PY_BUILTINS_CODE_MINIMUM : MICROPY_PY_BUILTINS_CODE_NONE))) #endif // Whether to support dict.fromkeys() class method diff --git a/py/objfun.c b/py/objfun.c index 0b1b8c115f236..a742c5267254c 100644 --- a/py/objfun.c +++ b/py/objfun.c @@ -28,6 +28,8 @@ #include #include +#include "py/emitglue.h" +#include "py/objcode.h" #include "py/objtuple.h" #include "py/objfun.h" #include "py/runtime.h" @@ -151,6 +153,30 @@ qstr mp_obj_fun_get_name(mp_const_obj_t fun_in) { return name; } +#if MICROPY_PY_FUNCTION_ATTRS_CODE +static mp_obj_t fun_bc_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { + (void)type; + mp_arg_check_num(n_args, n_kw, 2, 2, false); + + if (!mp_obj_is_type(args[0], &mp_type_code)) { + mp_raise_TypeError(NULL); + } + if (!mp_obj_is_type(args[1], &mp_type_dict)) { + mp_raise_TypeError(NULL); + } + + mp_obj_code_t *code = MP_OBJ_TO_PTR(args[0]); + mp_obj_t globals = args[1]; + + mp_module_context_t *module_context = m_new_obj(mp_module_context_t); + module_context->module.base.type = &mp_type_module; + module_context->module.globals = MP_OBJ_TO_PTR(globals); + module_context->constants = *mp_code_get_constants(code); + + return mp_make_function_from_proto_fun(mp_code_get_proto_fun(code), module_context, NULL); +} +#endif + #if MICROPY_CPYTHON_COMPAT static void fun_bc_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { (void)kind; @@ -340,9 +366,29 @@ void mp_obj_fun_bc_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { mp_obj_fun_bc_t *self = MP_OBJ_TO_PTR(self_in); dest[0] = MP_OBJ_FROM_PTR(self->context->module.globals); } + #if MICROPY_PY_FUNCTION_ATTRS_CODE + if (attr == MP_QSTR___code__) { + const mp_obj_fun_bc_t *self = MP_OBJ_TO_PTR(self_in); + if ((self->base.type == &mp_type_fun_bc + || self->base.type == &mp_type_gen_wrap) + && self->child_table == NULL) { + #if MICROPY_PY_BUILTINS_CODE <= MICROPY_PY_BUILTINS_CODE_BASIC + dest[0] = mp_obj_new_code(self->context->constants, self->bytecode); + #else + dest[0] = mp_obj_new_code(self->context, self->rc, true); + #endif + } + } + #endif } #endif +#if MICROPY_PY_FUNCTION_ATTRS_CODE +#define FUN_BC_MAKE_NEW make_new, fun_bc_make_new, +#else +#define FUN_BC_MAKE_NEW +#endif + #if MICROPY_CPYTHON_COMPAT #define FUN_BC_TYPE_PRINT print, fun_bc_print, #else @@ -359,6 +405,7 @@ MP_DEFINE_CONST_OBJ_TYPE( mp_type_fun_bc, MP_QSTR_function, MP_TYPE_FLAG_BINDS_SELF, + FUN_BC_MAKE_NEW FUN_BC_TYPE_PRINT FUN_BC_TYPE_ATTR call, fun_bc_call diff --git a/tests/basics/fun_code.py b/tests/basics/fun_code.py new file mode 100644 index 0000000000000..59e1f7ec0483d --- /dev/null +++ b/tests/basics/fun_code.py @@ -0,0 +1,36 @@ +# Test function.__code__ attribute. + +try: + (lambda: 0).__code__ +except AttributeError: + print("SKIP") + raise SystemExit + + +def f(): + return a + + +ftype = type(f) + +# Test __code__ access and function constructor. +code = f.__code__ +print(type(ftype(code, {})) is ftype) + +# Test instantiating multiple code's with different globals dicts. +code = f.__code__ +f1 = ftype(code, {"a": 1}) +f2 = ftype(code, {"a": 2}) +print(f1(), f2()) + +# Test bad first argument type. +try: + ftype(None, {}) +except TypeError: + print("TypeError") + +# Test bad second argument type. +try: + ftype(f.__code__, None) +except TypeError: + print("TypeError") diff --git a/tests/basics/fun_code_micropython.py b/tests/basics/fun_code_micropython.py new file mode 100644 index 0000000000000..2c319a2db8c75 --- /dev/null +++ b/tests/basics/fun_code_micropython.py @@ -0,0 +1,19 @@ +# Test MicroPython-specific restrictions of function.__code__ attribute. + +try: + (lambda: 0).__code__ +except AttributeError: + print("SKIP") + raise SystemExit + + +def f_with_children(): + def g(): + pass + + +# Can't access __code__ when function has children. +try: + f_with_children.__code__ +except AttributeError: + print("AttributeError") diff --git a/tests/basics/fun_code_micropython.py.exp b/tests/basics/fun_code_micropython.py.exp new file mode 100644 index 0000000000000..d169edffb4cfb --- /dev/null +++ b/tests/basics/fun_code_micropython.py.exp @@ -0,0 +1 @@ +AttributeError diff --git a/tests/basics/subclass_native1.py b/tests/basics/subclass_native1.py index 288a686d1a756..74b377eac91b9 100644 --- a/tests/basics/subclass_native1.py +++ b/tests/basics/subclass_native1.py @@ -21,11 +21,9 @@ class mylist(list): # TODO: Faults #print(a + a) -def foo(): - print("hello from foo") - +# subclassing a type that doesn't have make_new at the C level (not allowed) try: - class myfunc(type(foo)): + class myfunc(type([].append)): pass except TypeError: print("TypeError") diff --git a/tests/micropython/native_fun_attrs_code.py b/tests/micropython/native_fun_attrs_code.py new file mode 100644 index 0000000000000..d277a7b9b2670 --- /dev/null +++ b/tests/micropython/native_fun_attrs_code.py @@ -0,0 +1,21 @@ +# Test MicroPython-specific restrictions of function.__code__ attribute. + +try: + (lambda: 0).__code__ +except AttributeError: + print("SKIP") + raise SystemExit + +import micropython + + +@micropython.native +def f_native(): + pass + + +# Can't access __code__ when function is native code. +try: + f_native.__code__ +except AttributeError: + print("AttributeError") diff --git a/tests/micropython/native_fun_attrs_code.py.exp b/tests/micropython/native_fun_attrs_code.py.exp new file mode 100644 index 0000000000000..d169edffb4cfb --- /dev/null +++ b/tests/micropython/native_fun_attrs_code.py.exp @@ -0,0 +1 @@ +AttributeError From a11ba7775e600b45c0e93443ca05dffb09a49389 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 17 Jul 2024 16:06:12 +1000 Subject: [PATCH 073/110] py/persistentcode: Add mp_raw_code_save_fun_to_bytes. Serialises a bytecode function/generator to a valid .mpy as bytes. Signed-off-by: Damien George --- py/mpconfig.h | 5 ++ py/persistentcode.c | 190 +++++++++++++++++++++++++++++++++++++++++++- py/persistentcode.h | 1 + 3 files changed, 194 insertions(+), 2 deletions(-) diff --git a/py/mpconfig.h b/py/mpconfig.h index 5c4d19bb56f11..a25d8cd32a2ba 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -342,6 +342,11 @@ #define MICROPY_PERSISTENT_CODE_SAVE_FILE (0) #endif +// Whether to support converting functions to persistent code (bytes) +#ifndef MICROPY_PERSISTENT_CODE_SAVE_FUN +#define MICROPY_PERSISTENT_CODE_SAVE_FUN (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) +#endif + // Whether generated code can persist independently of the VM/runtime instance // This is enabled automatically when needed by other features #ifndef MICROPY_PERSISTENT_CODE diff --git a/py/persistentcode.c b/py/persistentcode.c index 840ee49d3e4eb..2a42b904bc138 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -526,7 +526,7 @@ void mp_raw_code_load_file(qstr filename, mp_compiled_module_t *context) { #endif // MICROPY_PERSISTENT_CODE_LOAD -#if MICROPY_PERSISTENT_CODE_SAVE +#if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_PERSISTENT_CODE_SAVE_FUN #include "py/objstr.h" @@ -624,6 +624,10 @@ static void save_obj(mp_print_t *print, mp_obj_t o) { } } +#endif // MICROPY_PERSISTENT_CODE_SAVE || MICROPY_PERSISTENT_CODE_SAVE_FUN + +#if MICROPY_PERSISTENT_CODE_SAVE + static void save_raw_code(mp_print_t *print, const mp_raw_code_t *rc) { // Save function kind and data length mp_print_uint(print, (rc->fun_data_len << 3) | ((rc->n_children != 0) << 2) | (rc->kind - MP_CODE_BYTECODE)); @@ -693,6 +697,8 @@ void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print) { save_raw_code(print, cm->rc); } +#endif // MICROPY_PERSISTENT_CODE_SAVE + #if MICROPY_PERSISTENT_CODE_SAVE_FILE #include @@ -723,4 +729,184 @@ void mp_raw_code_save_file(mp_compiled_module_t *cm, qstr filename) { #endif // MICROPY_PERSISTENT_CODE_SAVE_FILE -#endif // MICROPY_PERSISTENT_CODE_SAVE +#if MICROPY_PERSISTENT_CODE_SAVE_FUN + +#include "py/bc0.h" +#include "py/objfun.h" +#include "py/smallint.h" +#include "py/gc.h" + +#define MP_BC_OPCODE_HAS_SIGNED_OFFSET(opcode) (MP_BC_UNWIND_JUMP <= (opcode) && (opcode) <= MP_BC_POP_JUMP_IF_FALSE) + +typedef struct _bit_vector_t { + size_t max_bit_set; + size_t alloc; + uintptr_t *bits; +} bit_vector_t; + +static void bit_vector_init(bit_vector_t *self) { + self->max_bit_set = 0; + self->alloc = 1; + self->bits = m_new(uintptr_t, self->alloc); +} + +static void bit_vector_clear(bit_vector_t *self) { + m_del(uintptr_t, self->bits, self->alloc); +} + +static bool bit_vector_is_set(bit_vector_t *self, size_t index) { + const size_t bits_size = sizeof(*self->bits) * MP_BITS_PER_BYTE; + return index / bits_size < self->alloc + && (self->bits[index / bits_size] & (1 << (index % bits_size))) != 0; +} + +static void bit_vector_set(bit_vector_t *self, size_t index) { + const size_t bits_size = sizeof(*self->bits) * MP_BITS_PER_BYTE; + self->max_bit_set = MAX(self->max_bit_set, index); + if (index / bits_size >= self->alloc) { + size_t new_alloc = self->alloc * 2; + self->bits = m_renew(uintptr_t, self->bits, self->alloc, new_alloc); + self->alloc = new_alloc; + } + self->bits[index / bits_size] |= 1 << (index % bits_size); +} + +typedef struct _mp_opcode_t { + uint8_t opcode; + uint8_t format; + uint8_t size; + mp_int_t arg; + uint8_t extra_arg; +} mp_opcode_t; + +static mp_opcode_t mp_opcode_decode(const uint8_t *ip) { + const uint8_t *ip_start = ip; + uint8_t opcode = *ip++; + uint8_t opcode_format = MP_BC_FORMAT(opcode); + mp_uint_t arg = 0; + uint8_t extra_arg = 0; + if (opcode_format == MP_BC_FORMAT_QSTR || opcode_format == MP_BC_FORMAT_VAR_UINT) { + arg = *ip & 0x7f; + if (opcode == MP_BC_LOAD_CONST_SMALL_INT && (arg & 0x40) != 0) { + arg |= (mp_uint_t)(-1) << 7; + } + while ((*ip & 0x80) != 0) { + arg = (arg << 7) | (*++ip & 0x7f); + } + ++ip; + } else if (opcode_format == MP_BC_FORMAT_OFFSET) { + if ((*ip & 0x80) == 0) { + arg = *ip++; + if (MP_BC_OPCODE_HAS_SIGNED_OFFSET(opcode)) { + arg -= 0x40; + } + } else { + arg = (ip[0] & 0x7f) | (ip[1] << 7); + ip += 2; + if (MP_BC_OPCODE_HAS_SIGNED_OFFSET(opcode)) { + arg -= 0x4000; + } + } + } + if ((opcode & MP_BC_MASK_EXTRA_BYTE) == 0) { + extra_arg = *ip++; + } + + mp_opcode_t op = { opcode, opcode_format, ip - ip_start, arg, extra_arg }; + return op; +} + +mp_obj_t mp_raw_code_save_fun_to_bytes(const mp_module_constants_t *consts, const uint8_t *bytecode) { + const uint8_t *fun_data = bytecode; + const uint8_t *fun_data_top = fun_data + gc_nbytes(fun_data); + + // Extract function information. + const byte *ip = fun_data; + MP_BC_PRELUDE_SIG_DECODE(ip); + MP_BC_PRELUDE_SIZE_DECODE(ip); + + // Track the qstrs used by the function. + bit_vector_t qstr_table_used; + bit_vector_init(&qstr_table_used); + + // Track the objects used by the function. + bit_vector_t obj_table_used; + bit_vector_init(&obj_table_used); + + const byte *ip_names = ip; + mp_uint_t simple_name = mp_decode_uint(&ip_names); + bit_vector_set(&qstr_table_used, simple_name); + for (size_t i = 0; i < n_pos_args + n_kwonly_args; ++i) { + mp_uint_t arg_name = mp_decode_uint(&ip_names); + bit_vector_set(&qstr_table_used, arg_name); + } + + // Skip pass source code info and cell info. + // Then ip points to the start of the opcodes. + ip += n_info + n_cell; + + // Decode bytecode. + while (ip < fun_data_top) { + mp_opcode_t op = mp_opcode_decode(ip); + if (op.opcode == MP_BC_BASE_RESERVED) { + // End of opcodes. + fun_data_top = ip; + } else if (op.opcode == MP_BC_LOAD_CONST_OBJ) { + bit_vector_set(&obj_table_used, op.arg); + } else if (op.format == MP_BC_FORMAT_QSTR) { + bit_vector_set(&qstr_table_used, op.arg); + } + ip += op.size; + } + + mp_uint_t fun_data_len = fun_data_top - fun_data; + + mp_print_t print; + vstr_t vstr; + vstr_init_print(&vstr, 64, &print); + + // Start with .mpy header. + const uint8_t header[4] = { 'M', MPY_VERSION, 0, MP_SMALL_INT_BITS }; + mp_print_bytes(&print, header, sizeof(header)); + + // Number of entries in constant table. + mp_print_uint(&print, qstr_table_used.max_bit_set + 1); + mp_print_uint(&print, obj_table_used.max_bit_set + 1); + + // Save qstrs. + for (size_t i = 0; i <= qstr_table_used.max_bit_set; ++i) { + if (bit_vector_is_set(&qstr_table_used, i)) { + save_qstr(&print, consts->qstr_table[i]); + } else { + save_qstr(&print, MP_QSTR_); + } + } + + // Save constant objects. + for (size_t i = 0; i <= obj_table_used.max_bit_set; ++i) { + if (bit_vector_is_set(&obj_table_used, i)) { + save_obj(&print, consts->obj_table[i]); + } else { + save_obj(&print, mp_const_none); + } + } + + bit_vector_clear(&qstr_table_used); + bit_vector_clear(&obj_table_used); + + // Save function kind and data length. + mp_print_uint(&print, fun_data_len << 3); + + // Save function code. + mp_print_bytes(&print, fun_data, fun_data_len); + + // Create and return bytes representing the .mpy data. + return mp_obj_new_bytes_from_vstr(&vstr); +} + +#endif // MICROPY_PERSISTENT_CODE_SAVE_FUN + +#if MICROPY_PERSISTENT_CODE_TRACK_RELOC_CODE +// An mp_obj_list_t that tracks relocated native code to prevent the GC from reclaiming them. +MP_REGISTER_ROOT_POINTER(mp_obj_t track_reloc_code_list); +#endif diff --git a/py/persistentcode.h b/py/persistentcode.h index f0b7f70f7d7fa..cf257a7ab1fb6 100644 --- a/py/persistentcode.h +++ b/py/persistentcode.h @@ -121,6 +121,7 @@ void mp_raw_code_load_file(qstr filename, mp_compiled_module_t *ctx); void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print); void mp_raw_code_save_file(mp_compiled_module_t *cm, qstr filename); +mp_obj_t mp_raw_code_save_fun_to_bytes(const mp_module_constants_t *consts, const uint8_t *bytecode); void mp_native_relocate(void *reloc, uint8_t *text, uintptr_t reloc_text); From c3a18d74ebebe1c68955c3dce3c782af949aa4c7 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 20 Jan 2025 22:24:10 +1100 Subject: [PATCH 074/110] extmod/modmarshal: Add new marshal module. This commit implements a small subset of the CPython `marshal` module. It implements `marshal.dumps()` and `marshal.loads()`, but only supports (un)marshalling code objects at this stage. The semantics match CPython, except that the actual marshalled bytes is not compatible with CPython's marshalled bytes. The module is enabled at the everything level (only on the unix coverage build at this stage). Signed-off-by: Damien George --- extmod/extmod.cmake | 1 + extmod/extmod.mk | 1 + extmod/modmarshal.c | 88 ++++++++++++++++++ ports/windows/msvc/sources.props | 1 + py/mpconfig.h | 7 +- tests/extmod/marshal_basic.py | 38 ++++++++ tests/extmod/marshal_micropython.py | 21 +++++ tests/extmod/marshal_stress.py | 122 +++++++++++++++++++++++++ tests/ports/unix/extra_coverage.py.exp | 10 +- 9 files changed, 283 insertions(+), 6 deletions(-) create mode 100644 extmod/modmarshal.c create mode 100644 tests/extmod/marshal_basic.py create mode 100644 tests/extmod/marshal_micropython.py create mode 100644 tests/extmod/marshal_stress.py diff --git a/extmod/extmod.cmake b/extmod/extmod.cmake index 532ce83f9075f..3643f1aee7bf5 100644 --- a/extmod/extmod.cmake +++ b/extmod/extmod.cmake @@ -24,6 +24,7 @@ set(MICROPY_SOURCE_EXTMOD ${MICROPY_EXTMOD_DIR}/modframebuf.c ${MICROPY_EXTMOD_DIR}/modlwip.c ${MICROPY_EXTMOD_DIR}/modmachine.c + ${MICROPY_EXTMOD_DIR}/modmarshal.c ${MICROPY_EXTMOD_DIR}/modnetwork.c ${MICROPY_EXTMOD_DIR}/modonewire.c ${MICROPY_EXTMOD_DIR}/modasyncio.c diff --git a/extmod/extmod.mk b/extmod/extmod.mk index 6d54ae2222e74..a510f3c54c25c 100644 --- a/extmod/extmod.mk +++ b/extmod/extmod.mk @@ -29,6 +29,7 @@ SRC_EXTMOD_C += \ extmod/modjson.c \ extmod/modlwip.c \ extmod/modmachine.c \ + extmod/modmarshal.c \ extmod/modnetwork.c \ extmod/modonewire.c \ extmod/modopenamp.c \ diff --git a/extmod/modmarshal.c b/extmod/modmarshal.c new file mode 100644 index 0000000000000..93d2bcf115065 --- /dev/null +++ b/extmod/modmarshal.c @@ -0,0 +1,88 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2025 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#include "py/objcode.h" +#include "py/objfun.h" +#include "py/persistentcode.h" +#include "py/runtime.h" + +#if MICROPY_PY_MARSHAL + +static mp_obj_t marshal_dumps(mp_obj_t value_in) { + if (mp_obj_is_type(value_in, &mp_type_code)) { + mp_obj_code_t *code = MP_OBJ_TO_PTR(value_in); + const void *proto_fun = mp_code_get_proto_fun(code); + const uint8_t *bytecode; + if (mp_proto_fun_is_bytecode(proto_fun)) { + bytecode = proto_fun; + } else { + const mp_raw_code_t *rc = proto_fun; + if (!(rc->kind == MP_CODE_BYTECODE && rc->children == NULL)) { + mp_raise_ValueError(MP_ERROR_TEXT("function must be bytecode with no children")); + } + bytecode = rc->fun_data; + } + return mp_raw_code_save_fun_to_bytes(mp_code_get_constants(code), bytecode); + } else { + mp_raise_ValueError(MP_ERROR_TEXT("unmarshallable object")); + } +} +static MP_DEFINE_CONST_FUN_OBJ_1(marshal_dumps_obj, marshal_dumps); + +static mp_obj_t marshal_loads(mp_obj_t data_in) { + mp_buffer_info_t bufinfo; + mp_get_buffer_raise(data_in, &bufinfo, MP_BUFFER_READ); + mp_module_context_t ctx; + ctx.module.globals = mp_globals_get(); + mp_compiled_module_t cm = { .context = &ctx }; + mp_raw_code_load_mem(bufinfo.buf, bufinfo.len, &cm); + #if MICROPY_PY_BUILTINS_CODE <= MICROPY_PY_BUILTINS_CODE_BASIC + return mp_obj_new_code(ctx.constants, cm.rc); + #else + mp_module_context_t *ctx_ptr = m_new_obj(mp_module_context_t); + *ctx_ptr = ctx; + return mp_obj_new_code(ctx_ptr, cm.rc, true); + #endif +} +static MP_DEFINE_CONST_FUN_OBJ_1(marshal_loads_obj, marshal_loads); + +static const mp_rom_map_elem_t mod_marshal_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_marshal) }, + { MP_ROM_QSTR(MP_QSTR_dumps), MP_ROM_PTR(&marshal_dumps_obj) }, + { MP_ROM_QSTR(MP_QSTR_loads), MP_ROM_PTR(&marshal_loads_obj) }, +}; + +static MP_DEFINE_CONST_DICT(mod_marshal_globals, mod_marshal_globals_table); + +const mp_obj_module_t mp_module_marshal = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&mod_marshal_globals, +}; + +MP_REGISTER_MODULE(MP_QSTR_marshal, mp_module_marshal); + +#endif // MICROPY_PY_MARSHAL diff --git a/ports/windows/msvc/sources.props b/ports/windows/msvc/sources.props index f7c4c6bcac01b..dcd10ddee913c 100644 --- a/ports/windows/msvc/sources.props +++ b/ports/windows/msvc/sources.props @@ -15,6 +15,7 @@ + diff --git a/py/mpconfig.h b/py/mpconfig.h index a25d8cd32a2ba..66b3d125e78de 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -344,7 +344,7 @@ // Whether to support converting functions to persistent code (bytes) #ifndef MICROPY_PERSISTENT_CODE_SAVE_FUN -#define MICROPY_PERSISTENT_CODE_SAVE_FUN (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) +#define MICROPY_PERSISTENT_CODE_SAVE_FUN (MICROPY_PY_MARSHAL) #endif // Whether generated code can persist independently of the VM/runtime instance @@ -1382,6 +1382,11 @@ typedef double mp_float_t; #define MICROPY_PY_COLLECTIONS_NAMEDTUPLE__ASDICT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) #endif +// Whether to provide "marshal" module +#ifndef MICROPY_PY_MARSHAL +#define MICROPY_PY_MARSHAL (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING) +#endif + // Whether to provide "math" module #ifndef MICROPY_PY_MATH #define MICROPY_PY_MATH (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES) diff --git a/tests/extmod/marshal_basic.py b/tests/extmod/marshal_basic.py new file mode 100644 index 0000000000000..9e7b70be4829d --- /dev/null +++ b/tests/extmod/marshal_basic.py @@ -0,0 +1,38 @@ +# Test the marshal module, basic functionality. + +try: + import marshal + + (lambda: 0).__code__ +except (AttributeError, ImportError): + print("SKIP") + raise SystemExit + +ftype = type(lambda: 0) + +# Test basic dumps and loads. +print(ftype(marshal.loads(marshal.dumps((lambda: a).__code__)), {"a": 4})()) + +# Test dumps of a result from compile(). +ftype(marshal.loads(marshal.dumps(compile("print(a)", "", "exec"))), {"print": print, "a": 5})() + +# Test marshalling a function with arguments. +print(ftype(marshal.loads(marshal.dumps((lambda x, y: x + y).__code__)), {})(1, 2)) + +# Test marshalling a function with default arguments. +print(ftype(marshal.loads(marshal.dumps((lambda x=0: x).__code__)), {})("arg")) + +# Test marshalling a function containing constant objects (a tuple). +print(ftype(marshal.loads(marshal.dumps((lambda: (None, ...)).__code__)), {})()) + +# Test instantiating multiple code's with different globals dicts. +code = marshal.loads(marshal.dumps((lambda: a).__code__)) +f1 = ftype(code, {"a": 1}) +f2 = ftype(code, {"a": 2}) +print(f1(), f2()) + +# Test unmarshallable object. +try: + marshal.dumps(type) +except ValueError: + print("ValueError") diff --git a/tests/extmod/marshal_micropython.py b/tests/extmod/marshal_micropython.py new file mode 100644 index 0000000000000..213b3bf31895d --- /dev/null +++ b/tests/extmod/marshal_micropython.py @@ -0,0 +1,21 @@ +# Test the marshal module, MicroPython-specific functionality. + +try: + import marshal +except ImportError: + print("SKIP") + raise SystemExit + +import unittest + + +class Test(unittest.TestCase): + def test_function_with_children(self): + # Can't marshal a function with children (in this case the module has a child function f). + code = compile("def f(): pass", "", "exec") + with self.assertRaises(ValueError): + marshal.dumps(code) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/extmod/marshal_stress.py b/tests/extmod/marshal_stress.py new file mode 100644 index 0000000000000..b52475c0361dc --- /dev/null +++ b/tests/extmod/marshal_stress.py @@ -0,0 +1,122 @@ +# Test the marshal module, stressing edge cases. + +try: + import marshal + + (lambda: 0).__code__ +except (AttributeError, ImportError): + print("SKIP") + raise SystemExit + +ftype = type(lambda: 0) + +# Test a large function. + + +def large_function(arg0, arg1, arg2, arg3): + # Arguments. + print(arg0, arg1, arg2, arg3) + + # Positive medium-sized integer (still a small-int though). + print(1234) + + # Negative small-ish integer. + print(-20) + + # More than 64 constant objects. + x = (0,) + x = (1,) + x = (2,) + x = (3,) + x = (4,) + x = (5,) + x = (6,) + x = (7,) + x = (8,) + x = (9,) + x = (10,) + x = (11,) + x = (12,) + x = (13,) + x = (14,) + x = (15,) + x = (16,) + x = (17,) + x = (18,) + x = (19,) + x = (20,) + x = (21,) + x = (22,) + x = (23,) + x = (24,) + x = (25,) + x = (26,) + x = (27,) + x = (28,) + x = (29,) + x = (30,) + x = (31,) + x = (32,) + x = (33,) + x = (34,) + x = (35,) + x = (36,) + x = (37,) + x = (38,) + x = (39,) + x = (40,) + x = (41,) + x = (42,) + x = (43,) + x = (44,) + x = (45,) + x = (46,) + x = (47,) + x = (48,) + x = (49,) + x = (50,) + x = (51,) + x = (52,) + x = (53,) + x = (54,) + x = (55,) + x = (56,) + x = (57,) + x = (58,) + x = (59,) + x = (60,) + x = (61,) + x = (62,) + x = (63,) + x = (64,) + + # Small jump. + x = 0 + while x < 2: + print("loop", x) + x += 1 + + # Large jump. + x = 0 + while x < 2: + try: + try: + try: + print + except Exception as e: + print + finally: + print + except Exception as e: + print + finally: + print + except Exception as e: + print + finally: + print("loop", x) + x += 1 + + +code = marshal.dumps(large_function.__code__) +ftype(marshal.loads(code), {"print": print})(0, 1, 2, 3) diff --git a/tests/ports/unix/extra_coverage.py.exp b/tests/ports/unix/extra_coverage.py.exp index 176db8e9f8479..5ff947e883d37 100644 --- a/tests/ports/unix/extra_coverage.py.exp +++ b/tests/ports/unix/extra_coverage.py.exp @@ -56,13 +56,13 @@ cmath collections cppexample cryptolib deflate errno example_package ffi framebuf gc hashlib heapq io json machine -math os platform random -re select socket struct -sys termios time tls -uctypes vfs websocket +marshal math os platform +random re select socket +struct sys termios time +tls uctypes vfs websocket me -micropython machine math +micropython machine marshal math argv atexit byteorder exc_info executable exit getsizeof implementation From e40a3fdb81fc7184e945754c06d42e169a296ab8 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 22 Jan 2025 15:13:13 +1100 Subject: [PATCH 075/110] docs/library/marshal: Document the marshal module. Signed-off-by: Damien George --- docs/library/index.rst | 1 + docs/library/marshal.rst | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 docs/library/marshal.rst diff --git a/docs/library/index.rst b/docs/library/index.rst index 4209a0781a60e..2919378ce13b2 100644 --- a/docs/library/index.rst +++ b/docs/library/index.rst @@ -69,6 +69,7 @@ library. heapq.rst io.rst json.rst + marshal.rst math.rst os.rst platform.rst diff --git a/docs/library/marshal.rst b/docs/library/marshal.rst new file mode 100644 index 0000000000000..6d3213e0a6ec8 --- /dev/null +++ b/docs/library/marshal.rst @@ -0,0 +1,28 @@ +:mod:`marshal` -- Python object serialization +============================================= + +.. module:: marshal + :synopsis: Convert Python objects to and from a binary format + +|see_cpython_module| :mod:`python:marshal`. + +This module implements conversion between Python objects and a binary format. +The format is specific to MicroPython but does not depend on the machine +architecture, so the data can be transferred and used on a different MicroPython +instance, as long as the version of the binary data matches (it's currently +versioned as the mpy file version, see :ref:`mpy_files`). + +Functions +--------- + +.. function:: dumps(value, /) + + Convert the given *value* to binary format and return a corresponding ``bytes`` + object. + + Currently, code objects are the only supported values that can be converted. + +.. function:: loads(data, /) + + Convert the given bytes-like *data* to its corresponding Python object, and + return it. From 30acb16ad34babf25d126a7b471616421167cb35 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 3 Jan 2025 12:02:12 +1100 Subject: [PATCH 076/110] extmod/vfs_rom: Remove ability to create VfsRom from an address. It's not necessary to support this, which allows an arbitrary memory address to be specified and potentially allows invalid memory accesses. Requiring an object with the buffer protocol is safer, and also means that the length of the region is always specified. Signed-off-by: Damien George --- extmod/vfs_rom.c | 12 +++++------- tests/extmod/vfs_rom.py | 3 ++- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/extmod/vfs_rom.c b/extmod/vfs_rom.c index b8b6f8e40d12b..99ddaba95ed79 100644 --- a/extmod/vfs_rom.c +++ b/extmod/vfs_rom.c @@ -198,15 +198,13 @@ static mp_obj_t vfs_rom_make_new(const mp_obj_type_t *type, size_t n_args, size_ self->base.type = type; self->memory = args[0]; + // Get the ROMFS memory region. mp_buffer_info_t bufinfo; - if (mp_get_buffer(self->memory, &bufinfo, MP_BUFFER_READ)) { - if (bufinfo.len < ROMFS_SIZE_MIN) { - mp_raise_OSError(MP_ENODEV); - } - self->filesystem = bufinfo.buf; - } else { - self->filesystem = (const uint8_t *)(uintptr_t)mp_obj_get_int_truncated(self->memory); + mp_get_buffer_raise(self->memory, &bufinfo, MP_BUFFER_READ); + if (bufinfo.len < ROMFS_SIZE_MIN) { + mp_raise_OSError(MP_ENODEV); } + self->filesystem = bufinfo.buf; // Verify it is a ROMFS. if (!(self->filesystem[0] == ROMFS_HEADER_BYTE0 diff --git a/tests/extmod/vfs_rom.py b/tests/extmod/vfs_rom.py index d416d83870558..f7958a93962d0 100644 --- a/tests/extmod/vfs_rom.py +++ b/tests/extmod/vfs_rom.py @@ -226,7 +226,8 @@ def test_unknown_record(self): class TestStandalone(TestBase): def test_constructor(self): self.assertIsInstance(vfs.VfsRom(self.romfs), vfs.VfsRom) - self.assertIsInstance(vfs.VfsRom(self.romfs_addr), vfs.VfsRom) + with self.assertRaises(TypeError): + vfs.VfsRom(self.romfs_addr) def test_mount(self): vfs.VfsRom(self.romfs).mount(True, False) From 1a67d720c786fd85c848ffba4b792048d262095f Mon Sep 17 00:00:00 2001 From: rufusclark <50201718+rufusclark@users.noreply.github.com> Date: Tue, 6 Aug 2024 00:32:35 +0100 Subject: [PATCH 077/110] tools/pyboard.py: Make get_time use machine.RTC instead of pyb.RTC. The current code evaluates `pyb.RTC().datetime()` resulting in a remote side exception, as `pyb` is not defined on most ports (only stm32). The code should evaluate `machine.RTC().datetime()` and hence return the current time. Signed-off-by: rufusclark <50201718+rufusclark@users.noreply.github.com> Signed-off-by: Damien George --- tools/pyboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pyboard.py b/tools/pyboard.py index dac51e7d6b896..0cf5b3d46596a 100755 --- a/tools/pyboard.py +++ b/tools/pyboard.py @@ -506,7 +506,7 @@ def execfile(self, filename): return self.exec_(pyfile) def get_time(self): - t = str(self.eval("pyb.RTC().datetime()"), encoding="utf8")[1:-1].split(", ") + t = str(self.eval("machine.RTC().datetime()"), encoding="utf8")[1:-1].split(", ") return int(t[4]) * 3600 + int(t[5]) * 60 + int(t[6]) def fs_exists(self, src): From b675c87992b8b62ca8b18a05862e9c7b9427b9de Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Wed, 5 Feb 2025 15:08:25 +1100 Subject: [PATCH 078/110] esp32/machine_sdcard: Fix invalid result of SDCard.read/writeblocks. Functions would return NULL instead of `mp_const_false` if failed to init. This work was funded through GitHub Sponsors. Signed-off-by: Angus Gratton --- ports/esp32/machine_sdcard.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ports/esp32/machine_sdcard.c b/ports/esp32/machine_sdcard.c index 92c6e64698c33..da6b0df282caf 100644 --- a/ports/esp32/machine_sdcard.c +++ b/ports/esp32/machine_sdcard.c @@ -364,7 +364,7 @@ static mp_obj_t machine_sdcard_readblocks(mp_obj_t self_in, mp_obj_t block_num, err = sdcard_ensure_card_init((sdcard_card_obj_t *)self, false); if (err != ESP_OK) { - return false; + return mp_const_false; } mp_get_buffer_raise(buf, &bufinfo, MP_BUFFER_WRITE); @@ -381,7 +381,7 @@ static mp_obj_t machine_sdcard_writeblocks(mp_obj_t self_in, mp_obj_t block_num, err = sdcard_ensure_card_init((sdcard_card_obj_t *)self, false); if (err != ESP_OK) { - return false; + return mp_const_false; } mp_get_buffer_raise(buf, &bufinfo, MP_BUFFER_READ); From 842e3617a0f152f837b2e8cc4e9d4efae6cb52a4 Mon Sep 17 00:00:00 2001 From: iabdalkader Date: Wed, 11 Dec 2024 13:17:47 +0100 Subject: [PATCH 079/110] renesas-ra/Makefile: Remove id_code section from binary file generation. The linker scripts for most of these microcontrollers contain a non-contiguous flash section for the ID code that results in big binary files, which exceed the flash size. This commit removes the ID code section from the main firmware binary, and outputs it to a separate binary, which can be deployed manually if ID code is enabled. Signed-off-by: iabdalkader --- ports/renesas-ra/Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ports/renesas-ra/Makefile b/ports/renesas-ra/Makefile index dd70f56d7ca63..03eb9b52b41b8 100644 --- a/ports/renesas-ra/Makefile +++ b/ports/renesas-ra/Makefile @@ -442,7 +442,8 @@ endef define GENERATE_BIN $(ECHO) "GEN $(1)" - $(Q)$(OBJCOPY) -I ihex -O binary $(2) $(1) + $(Q)$(OBJCOPY) -O binary -j .id_code $(2) $(BUILD)/id_code.bin + $(Q)$(OBJCOPY) -O binary --remove-section=.id_code $(2) $(1) endef define GENERATE_HEX @@ -452,7 +453,7 @@ endef .PHONY: -$(BUILD)/firmware.bin: $(BUILD)/firmware.hex +$(BUILD)/firmware.bin: $(BUILD)/firmware.elf $(call GENERATE_BIN,$@,$^) $(BUILD)/firmware.hex: $(BUILD)/firmware.elf From aef6705a321fbefb06288b5be1f5931bf8c42fe3 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Wed, 22 Jan 2025 17:17:03 -0600 Subject: [PATCH 080/110] extmod/lwip-include: Increase number of lwIP timers when mDNS enabled. Despite the code comments claiming one is sufficient, the mDNS application is capable of using up to twelve timers. Three per IP protocol are started at once in `mdns_start_multicast_timeouts_ipvX`, then another two per protocol can be started in `mdns_handle_question`. Further timers can be started for two additional callbacks. Having certain timers, such as `MDNS_MULTICAST_TIMEOUT`, fail to start due to none being free will break mDNS forever as the app will never realize it's safe to transmit a packet. Therefore, this commit goes somewhat overkill and allocates the maximal amount of timers; it's uncertain if all can run simultaneously, or how many callback timers are needed. Each timer struct is 16 bytes on standard 32 bit builds. Plus, say, 8 bytes of allocater overhead, that's 288 more bytes of RAM used which shouldn't be too horrible. Users who don't need mDNS can manually disable it to recover the RAM if necessary. This fixes mDNS on W5500_EVB_PICO (among other boards). Before, mDNS would work for a bit after connection until the host's cache expired a minute or two later. Then the board would never respond to further queries. With this patch, all works well. Signed-off-by: Thomas Watson --- extmod/lwip-include/lwipopts_common.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/extmod/lwip-include/lwipopts_common.h b/extmod/lwip-include/lwipopts_common.h index 8688db064ef6f..3e4230909499e 100644 --- a/extmod/lwip-include/lwipopts_common.h +++ b/extmod/lwip-include/lwipopts_common.h @@ -65,7 +65,9 @@ #define LWIP_NUM_NETIF_CLIENT_DATA LWIP_MDNS_RESPONDER #define MEMP_NUM_UDP_PCB (4 + LWIP_MDNS_RESPONDER) -#define MEMP_NUM_SYS_TIMEOUT (LWIP_NUM_SYS_TIMEOUT_INTERNAL + LWIP_MDNS_RESPONDER) + +// The mDNS responder requires 5 timers per IP version plus 2 others. Not having enough silently breaks it. +#define MEMP_NUM_SYS_TIMEOUT (LWIP_NUM_SYS_TIMEOUT_INTERNAL + (LWIP_MDNS_RESPONDER * (2 + (5 * (LWIP_IPV4 + LWIP_IPV6))))) #define SO_REUSE 1 #define TCP_LISTEN_BACKLOG 1 From 321b30ca564bb33c625292247d00f7dd29dc9559 Mon Sep 17 00:00:00 2001 From: Keenan Johnson Date: Thu, 13 Feb 2025 13:11:38 -0800 Subject: [PATCH 081/110] extmod/modtls_mbedtls: Wire in support for DTLS. This commit enables support for DTLS, i.e. TLS over datagram transport protocols like UDP. While support for DTLS is absent in CPython, it is worth supporting it in MicroPython because it is the basis of the ubiquitous CoAP protocol, used in many IoT projects. To select DTLS, a new set of "protocols" are added to SSLContext: - ssl.PROTOCOL_DTLS_CLIENT - ssl.PROTOCOL_DTLS_SERVER If one of these is set, the library assumes that the underlying socket is a datagram-like socket (i.e. UDP or similar). Our own timer callbacks are implemented because the out of the box implementation relies on `gettimeofday()`. This new DTLS feature is enabled on all ports that use mbedTLS. This commit is an update to a previous PR #10062. Addresses issue #5270 which requested DTLS support. Signed-off-by: Keenan Johnson --- docs/library/ssl.rst | 21 +++++++ extmod/mbedtls/mbedtls_config_common.h | 1 + extmod/modtls_mbedtls.c | 76 ++++++++++++++++++++++++-- ports/esp32/boards/sdkconfig.base | 3 + tests/extmod/tls_dtls.py | 51 +++++++++++++++++ tests/extmod/tls_dtls.py.exp | 3 + 6 files changed, 151 insertions(+), 4 deletions(-) create mode 100644 tests/extmod/tls_dtls.py create mode 100644 tests/extmod/tls_dtls.py.exp diff --git a/docs/library/ssl.rst b/docs/library/ssl.rst index dff90b8da58b9..4327c74bad6c8 100644 --- a/docs/library/ssl.rst +++ b/docs/library/ssl.rst @@ -117,11 +117,32 @@ Exceptions This exception does NOT exist. Instead its base class, OSError, is used. +DTLS support +------------ + +.. admonition:: Difference to CPython + :class: attention + + This is a MicroPython extension. + +This module supports DTLS in client and server mode via the `PROTOCOL_DTLS_CLIENT` +and `PROTOCOL_DTLS_SERVER` constants that can be used as the ``protocol`` argument +of `SSLContext`. + +In this case the underlying socket is expected to behave as a datagram socket (i.e. +like the socket opened with ``socket.socket`` with ``socket.AF_INET`` as ``af`` and +``socket.SOCK_DGRAM`` as ``type``). + +DTLS is only supported on ports that use mbed TLS, and it is not enabled by default: +it requires enabling ``MBEDTLS_SSL_PROTO_DTLS`` in the specific port configuration. + Constants --------- .. data:: ssl.PROTOCOL_TLS_CLIENT ssl.PROTOCOL_TLS_SERVER + ssl.PROTOCOL_DTLS_CLIENT (when DTLS support is enabled) + ssl.PROTOCOL_DTLS_SERVER (when DTLS support is enabled) Supported values for the *protocol* parameter. diff --git a/extmod/mbedtls/mbedtls_config_common.h b/extmod/mbedtls/mbedtls_config_common.h index 6ea8540af9925..6cd14befc3196 100644 --- a/extmod/mbedtls/mbedtls_config_common.h +++ b/extmod/mbedtls/mbedtls_config_common.h @@ -89,6 +89,7 @@ #define MBEDTLS_SHA384_C #define MBEDTLS_SHA512_C #define MBEDTLS_SSL_CLI_C +#define MBEDTLS_SSL_PROTO_DTLS #define MBEDTLS_SSL_SRV_C #define MBEDTLS_SSL_TLS_C #define MBEDTLS_X509_CRT_PARSE_C diff --git a/extmod/modtls_mbedtls.c b/extmod/modtls_mbedtls.c index 3fd416d72f5ef..6c34805da42cb 100644 --- a/extmod/modtls_mbedtls.c +++ b/extmod/modtls_mbedtls.c @@ -37,6 +37,7 @@ #include "py/stream.h" #include "py/objstr.h" #include "py/reader.h" +#include "py/mphal.h" #include "py/gc.h" #include "extmod/vfs.h" @@ -47,6 +48,9 @@ #include "mbedtls/pk.h" #include "mbedtls/entropy.h" #include "mbedtls/ctr_drbg.h" +#ifdef MBEDTLS_SSL_PROTO_DTLS +#include "mbedtls/timing.h" +#endif #include "mbedtls/debug.h" #include "mbedtls/error.h" #if MBEDTLS_VERSION_NUMBER >= 0x03000000 @@ -65,6 +69,14 @@ #define MP_STREAM_POLL_RDWR (MP_STREAM_POLL_RD | MP_STREAM_POLL_WR) +#define MP_ENDPOINT_IS_SERVER (1 << 0) +#define MP_TRANSPORT_IS_DTLS (1 << 1) + +#define MP_PROTOCOL_TLS_CLIENT 0 +#define MP_PROTOCOL_TLS_SERVER MP_ENDPOINT_IS_SERVER +#define MP_PROTOCOL_DTLS_CLIENT MP_TRANSPORT_IS_DTLS +#define MP_PROTOCOL_DTLS_SERVER MP_ENDPOINT_IS_SERVER | MP_TRANSPORT_IS_DTLS + // This corresponds to an SSLContext object. typedef struct _mp_obj_ssl_context_t { mp_obj_base_t base; @@ -91,6 +103,12 @@ typedef struct _mp_obj_ssl_socket_t { uintptr_t poll_mask; // Indicates which read or write operations the protocol needs next int last_error; // The last error code, if any + + #ifdef MBEDTLS_SSL_PROTO_DTLS + mp_uint_t timer_start_ms; + mp_uint_t timer_fin_ms; + mp_uint_t timer_int_ms; + #endif } mp_obj_ssl_socket_t; static const mp_obj_type_t ssl_context_type; @@ -242,7 +260,10 @@ static mp_obj_t ssl_context_make_new(const mp_obj_type_t *type_in, size_t n_args mp_arg_check_num(n_args, n_kw, 1, 1, false); // This is the "protocol" argument. - mp_int_t endpoint = mp_obj_get_int(args[0]); + mp_int_t protocol = mp_obj_get_int(args[0]); + + int endpoint = (protocol & MP_ENDPOINT_IS_SERVER) ? MBEDTLS_SSL_IS_SERVER : MBEDTLS_SSL_IS_CLIENT; + int transport = (protocol & MP_TRANSPORT_IS_DTLS) ? MBEDTLS_SSL_TRANSPORT_DATAGRAM : MBEDTLS_SSL_TRANSPORT_STREAM; // Create SSLContext object. #if MICROPY_PY_SSL_FINALISER @@ -282,7 +303,7 @@ static mp_obj_t ssl_context_make_new(const mp_obj_type_t *type_in, size_t n_args } ret = mbedtls_ssl_config_defaults(&self->conf, endpoint, - MBEDTLS_SSL_TRANSPORT_STREAM, MBEDTLS_SSL_PRESET_DEFAULT); + transport, MBEDTLS_SSL_PRESET_DEFAULT); if (ret != 0) { mbedtls_raise_error(ret); } @@ -525,6 +546,39 @@ static int _mbedtls_ssl_recv(void *ctx, byte *buf, size_t len) { } } +#ifdef MBEDTLS_SSL_PROTO_DTLS +static void _mbedtls_timing_set_delay(void *ctx, uint32_t int_ms, uint32_t fin_ms) { + mp_obj_ssl_socket_t *o = (mp_obj_ssl_socket_t *)ctx; + + o->timer_int_ms = int_ms; + o->timer_fin_ms = fin_ms; + + if (fin_ms != 0) { + o->timer_start_ms = mp_hal_ticks_ms(); + } +} + +static int _mbedtls_timing_get_delay(void *ctx) { + mp_obj_ssl_socket_t *o = (mp_obj_ssl_socket_t *)ctx; + + if (o->timer_fin_ms == 0) { + return -1; + } + + mp_uint_t elapsed_ms = mp_hal_ticks_ms() - o->timer_start_ms; + + if (elapsed_ms >= o->timer_fin_ms) { + return 2; + } + + if (elapsed_ms >= o->timer_int_ms) { + return 1; + } + + return 0; +} +#endif + static mp_obj_t ssl_socket_make_new(mp_obj_ssl_context_t *ssl_context, mp_obj_t sock, bool server_side, bool do_handshake_on_connect, mp_obj_t server_hostname) { @@ -577,6 +631,10 @@ static mp_obj_t ssl_socket_make_new(mp_obj_ssl_context_t *ssl_context, mp_obj_t mp_raise_ValueError(MP_ERROR_TEXT("CERT_REQUIRED requires server_hostname")); } + #ifdef MBEDTLS_SSL_PROTO_DTLS + mbedtls_ssl_set_timer_cb(&o->ssl, o, _mbedtls_timing_set_delay, _mbedtls_timing_get_delay); + #endif + mbedtls_ssl_set_bio(&o->ssl, &o->sock, _mbedtls_ssl_send, _mbedtls_ssl_recv, NULL); if (do_handshake_on_connect) { @@ -788,6 +846,12 @@ static const mp_rom_map_elem_t ssl_socket_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) }, { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) }, { MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) }, + #ifdef MBEDTLS_SSL_PROTO_DTLS + { MP_ROM_QSTR(MP_QSTR_recv), MP_ROM_PTR(&mp_stream_read1_obj) }, + { MP_ROM_QSTR(MP_QSTR_recv_into), MP_ROM_PTR(&mp_stream_readinto_obj) }, + { MP_ROM_QSTR(MP_QSTR_send), MP_ROM_PTR(&mp_stream_write1_obj) }, + { MP_ROM_QSTR(MP_QSTR_sendall), MP_ROM_PTR(&mp_stream_write_obj) }, + #endif { MP_ROM_QSTR(MP_QSTR_setblocking), MP_ROM_PTR(&socket_setblocking_obj) }, { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, #if MICROPY_PY_SSL_FINALISER @@ -879,8 +943,12 @@ static const mp_rom_map_elem_t mp_module_tls_globals_table[] = { // Constants. { MP_ROM_QSTR(MP_QSTR_MBEDTLS_VERSION), MP_ROM_PTR(&mbedtls_version_obj)}, - { MP_ROM_QSTR(MP_QSTR_PROTOCOL_TLS_CLIENT), MP_ROM_INT(MBEDTLS_SSL_IS_CLIENT) }, - { MP_ROM_QSTR(MP_QSTR_PROTOCOL_TLS_SERVER), MP_ROM_INT(MBEDTLS_SSL_IS_SERVER) }, + { MP_ROM_QSTR(MP_QSTR_PROTOCOL_TLS_CLIENT), MP_ROM_INT(MP_PROTOCOL_TLS_CLIENT) }, + { MP_ROM_QSTR(MP_QSTR_PROTOCOL_TLS_SERVER), MP_ROM_INT(MP_PROTOCOL_TLS_SERVER) }, + #ifdef MBEDTLS_SSL_PROTO_DTLS + { MP_ROM_QSTR(MP_QSTR_PROTOCOL_DTLS_CLIENT), MP_ROM_INT(MP_PROTOCOL_DTLS_CLIENT) }, + { MP_ROM_QSTR(MP_QSTR_PROTOCOL_DTLS_SERVER), MP_ROM_INT(MP_PROTOCOL_DTLS_SERVER) }, + #endif { MP_ROM_QSTR(MP_QSTR_CERT_NONE), MP_ROM_INT(MBEDTLS_SSL_VERIFY_NONE) }, { MP_ROM_QSTR(MP_QSTR_CERT_OPTIONAL), MP_ROM_INT(MBEDTLS_SSL_VERIFY_OPTIONAL) }, { MP_ROM_QSTR(MP_QSTR_CERT_REQUIRED), MP_ROM_INT(MBEDTLS_SSL_VERIFY_REQUIRED) }, diff --git a/ports/esp32/boards/sdkconfig.base b/ports/esp32/boards/sdkconfig.base index e20835c70c42d..530db427119ca 100644 --- a/ports/esp32/boards/sdkconfig.base +++ b/ports/esp32/boards/sdkconfig.base @@ -64,6 +64,9 @@ CONFIG_MBEDTLS_HAVE_TIME_DATE=y CONFIG_MBEDTLS_PLATFORM_TIME_ALT=y CONFIG_MBEDTLS_HAVE_TIME=y +# Enable DTLS +CONFIG_MBEDTLS_SSL_PROTO_DTLS=y + # Disable ALPN support as it's not implemented in MicroPython CONFIG_MBEDTLS_SSL_ALPN=n diff --git a/tests/extmod/tls_dtls.py b/tests/extmod/tls_dtls.py new file mode 100644 index 0000000000000..b2d716769d3f7 --- /dev/null +++ b/tests/extmod/tls_dtls.py @@ -0,0 +1,51 @@ +# Test DTLS functionality including timeout handling + +try: + from tls import PROTOCOL_DTLS_CLIENT, PROTOCOL_DTLS_SERVER, SSLContext, CERT_NONE + import io +except ImportError: + print("SKIP") + raise SystemExit + + +class DummySocket(io.IOBase): + def __init__(self): + self.write_buffer = bytearray() + self.read_buffer = bytearray() + + def write(self, data): + return len(data) + + def readinto(self, buf): + # This is a placeholder socket that doesn't actually read anything + # so the read buffer is always empty. + return None + + def ioctl(self, req, arg): + if req == 4: # MP_STREAM_CLOSE + return 0 + return -1 + + +# Create dummy sockets for testing +server_socket = DummySocket() +client_socket = DummySocket() + +# Wrap the DTLS Server +dtls_server_ctx = SSLContext(PROTOCOL_DTLS_SERVER) +dtls_server_ctx.verify_mode = CERT_NONE +dtls_server = dtls_server_ctx.wrap_socket(server_socket, do_handshake_on_connect=False) +print("Wrapped DTLS Server") + +# Wrap the DTLS Client +dtls_client_ctx = SSLContext(PROTOCOL_DTLS_CLIENT) +dtls_client_ctx.verify_mode = CERT_NONE +dtls_client = dtls_client_ctx.wrap_socket(client_socket, do_handshake_on_connect=False) +print("Wrapped DTLS Client") + +# Trigger the timing check multiple times with different elapsed times +for i in range(10): # Try multiple iterations to hit the timing window + dtls_client.write(b"test") + data = dtls_server.read(1024) # This should eventually hit the timing condition + +print("OK") diff --git a/tests/extmod/tls_dtls.py.exp b/tests/extmod/tls_dtls.py.exp new file mode 100644 index 0000000000000..78d72bff18816 --- /dev/null +++ b/tests/extmod/tls_dtls.py.exp @@ -0,0 +1,3 @@ +Wrapped DTLS Server +Wrapped DTLS Client +OK From 8987b39e0b772d22539022f0961c8ea0f7162751 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 14 Feb 2025 09:54:56 +1100 Subject: [PATCH 082/110] tests/multi_net: Add test for DTLS server and client. This adds a multi-test for DTLS server and client behaviour. It works on all ports that enable this feature (eg unix, esp32, rp2, stm32), but bare-metal ports that use lwIP are not reliable as the DTLS server because the lwIP bindings only support queuing one UDP packet at a time (that needs to be fixed). Also, to properly implement a DTLS server sockets need to support `socket.recvfrom(n, MSG_PEEK)`. That can be implemented in the future. Signed-off-by: Damien George --- tests/multi_net/tls_dtls_server_client.py | 89 +++++++++++++++++++ tests/multi_net/tls_dtls_server_client.py.exp | 14 +++ 2 files changed, 103 insertions(+) create mode 100644 tests/multi_net/tls_dtls_server_client.py create mode 100644 tests/multi_net/tls_dtls_server_client.py.exp diff --git a/tests/multi_net/tls_dtls_server_client.py b/tests/multi_net/tls_dtls_server_client.py new file mode 100644 index 0000000000000..d50deb354ed4d --- /dev/null +++ b/tests/multi_net/tls_dtls_server_client.py @@ -0,0 +1,89 @@ +# Test DTLS server and client, sending a small amount of data between them. + +try: + import socket + import tls +except ImportError: + print("SKIP") + raise SystemExit + +PORT = 8000 + +# These are test certificates. See tests/README.md for details. +certfile = "ec_cert.der" +keyfile = "ec_key.der" + +try: + with open(certfile, "rb") as cf: + cert = cadata = cf.read() + with open(keyfile, "rb") as kf: + key = kf.read() +except OSError: + print("SKIP") + raise SystemExit + + +# DTLS server. +def instance0(): + multitest.globals(IP=multitest.get_network_ip()) + + # Create a UDP socket and bind it to accept incoming connections. + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(socket.getaddrinfo("0.0.0.0", PORT)[0][-1]) + + multitest.next() + + # Wait for the client to connect. + data, client_addr = s.recvfrom(1) + print("incoming connection", data) + + # Connect back to the client, so the UDP socket can be used like a stream. + s.connect(client_addr) + + # Create the DTLS context and load the certificate. + ctx = tls.SSLContext(tls.PROTOCOL_DTLS_SERVER) + ctx.load_cert_chain(cert, key) + + # Wrap the UDP socket in server mode. + print("wrap socket") + s = ctx.wrap_socket(s, server_side=1) + + # Transfer some data. + for _ in range(4): + print(s.recv(16)) + s.send(b"server to client") + + # Close the DTLS and UDP connection. + s.close() + + +# DTLS client. +def instance1(): + multitest.next() + + # Create a UDP socket and connect to the server. + addr = socket.getaddrinfo(IP, PORT)[0][-1] + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + print("connect") + s.connect(addr) + + # Send one byte to indicate a connection, and so the server can obtain our address. + s.write("X") + + # Create a DTLS context and load the certificate. + ctx = tls.SSLContext(tls.PROTOCOL_DTLS_CLIENT) + ctx.verify_mode = tls.CERT_REQUIRED + ctx.load_verify_locations(cadata) + + # Wrap the UDP socket. + print("wrap socket") + s = ctx.wrap_socket(s, server_hostname="micropython.local") + + # Transfer some data. + for _ in range(4): + s.send(b"client to server") + print(s.recv(16)) + + # Close the DTLS and UDP connection. + s.close() diff --git a/tests/multi_net/tls_dtls_server_client.py.exp b/tests/multi_net/tls_dtls_server_client.py.exp new file mode 100644 index 0000000000000..f2ff396e181df --- /dev/null +++ b/tests/multi_net/tls_dtls_server_client.py.exp @@ -0,0 +1,14 @@ +--- instance0 --- +incoming connection b'X' +wrap socket +b'client to server' +b'client to server' +b'client to server' +b'client to server' +--- instance1 --- +connect +wrap socket +b'server to client' +b'server to client' +b'server to client' +b'server to client' From 71df9d0636c8b0b95ab009f82bde7d5b970835ec Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 11 Feb 2025 16:42:03 +1100 Subject: [PATCH 083/110] rp2: Fix build failure if threads are disabled. Regression in 3af006ef meant that pendsv.c no longer compiled if threads were disabled in the build config. Add an implementation based on the earlier one (simple counter) for the non-threads case. It seems like with the current usage patterns there's no need for the counter to be incremented/decremented atomically on a single core config. This work was funded through GitHub Sponsors. Signed-off-by: Angus Gratton --- ports/rp2/pendsv.c | 49 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/ports/rp2/pendsv.c b/ports/rp2/pendsv.c index 4ba1e81604b0f..2c086f89429be 100644 --- a/ports/rp2/pendsv.c +++ b/ports/rp2/pendsv.c @@ -43,10 +43,14 @@ static pendsv_dispatch_t pendsv_dispatch_table[PENDSV_DISPATCH_NUM_SLOTS]; +static inline void pendsv_resume_run_dispatch(void); + void PendSV_Handler(void); -// Using the nowait variant here as softtimer updates PendSV from the loop of mp_wfe_or_timeout(), -// where we don't want the CPU event bit to be set. +#if MICROPY_PY_THREAD + +// Important to use a 'nowait' mutex here as softtimer updates PendSV from the +// loop of mp_wfe_or_timeout(), where we don't want the CPU event bit to be set. static mp_thread_recursive_mutex_t pendsv_mutex; void pendsv_init(void) { @@ -62,7 +66,40 @@ void pendsv_suspend(void) { void pendsv_resume(void) { mp_thread_recursive_mutex_unlock(&pendsv_mutex); + pendsv_resume_run_dispatch(); +} + +static inline int pendsv_suspend_count(void) { + return pendsv_mutex.mutex.enter_count; +} + +#else + +// Without threads we don't include any pico-sdk mutex in the build, +// but also we don't need to worry about cross-thread contention (or +// races with interrupts that update this counter). +static int pendsv_lock; + +void pendsv_init(void) { +} + +void pendsv_suspend(void) { + pendsv_lock++; +} + +void pendsv_resume(void) { + assert(pendsv_lock > 0); + pendsv_lock--; + pendsv_resume_run_dispatch(); +} +static inline int pendsv_suspend_count(void) { + return pendsv_lock; +} + +#endif + +static inline void pendsv_resume_run_dispatch(void) { // Run pendsv if needed. Find an entry with a dispatch and call pendsv dispatch // with it. If pendsv runs it will service all slots. int count = PENDSV_DISPATCH_NUM_SLOTS; @@ -76,7 +113,7 @@ void pendsv_resume(void) { void pendsv_schedule_dispatch(size_t slot, pendsv_dispatch_t f) { pendsv_dispatch_table[slot] = f; - if (pendsv_mutex.mutex.enter_count == 0) { + if (pendsv_suspend_count() == 0) { #if PICO_ARM // There is a race here where other core calls pendsv_suspend() before // ISR can execute, but dispatch will happen later when other core @@ -97,6 +134,7 @@ void pendsv_schedule_dispatch(size_t slot, pendsv_dispatch_t f) { // PendSV interrupt handler to perform background processing. void PendSV_Handler(void) { + #if MICROPY_PY_THREAD if (!mp_thread_recursive_mutex_lock(&pendsv_mutex, 0)) { // Failure here means core 1 holds pendsv_mutex. ISR will // run again after core 1 calls pendsv_resume(). @@ -104,6 +142,9 @@ void PendSV_Handler(void) { } // Core 0 should not already have locked pendsv_mutex assert(pendsv_mutex.mutex.enter_count == 1); + #else + assert(pendsv_suspend_count() == 0); + #endif #if MICROPY_PY_NETWORK_CYW43 CYW43_STAT_INC(PENDSV_RUN_COUNT); @@ -117,5 +158,7 @@ void PendSV_Handler(void) { } } + #if MICROPY_PY_THREAD mp_thread_recursive_mutex_unlock(&pendsv_mutex); + #endif } From 516709be88571da0195125c840e772a3ec70fa66 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 11 Feb 2025 17:44:19 +1100 Subject: [PATCH 084/110] py/mkrules.cmake: Support passing CFLAGS_EXTRA in environment variable. This works similarly to the existing support in "bare metal" make ports, with the caveat that CMake will only set this value on a clean build and will reuse the previous value otherwise. This is slightly different to the CMake built-in support for CFLAGS, as this variable is used when evaluating source files for qstr generation, etc. This work was funded through GitHub Sponsors. Signed-off-by: Angus Gratton --- py/mkrules.cmake | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/py/mkrules.cmake b/py/mkrules.cmake index 3ee4c4c31aa57..907500dd67534 100644 --- a/py/mkrules.cmake +++ b/py/mkrules.cmake @@ -53,6 +53,15 @@ foreach(_arg ${MICROPY_CPP_DEF}) endforeach() list(APPEND MICROPY_CPP_FLAGS ${MICROPY_CPP_FLAGS_EXTRA}) +# Include anything passed in via CFLAGS_EXTRA +# in both MICROPY_CPP_FLAGS and CMAKE_C_FLAGS +if(DEFINED ENV{CFLAGS_EXTRA}) + set(CFLAGS_EXTRA $ENV{CFLAGS_EXTRA}) + string(APPEND CMAKE_C_FLAGS " ${CFLAGS_EXTRA}") # ... not a list + separate_arguments(CFLAGS_EXTRA) + list(APPEND MICROPY_CPP_FLAGS ${CFLAGS_EXTRA}) # ... a list +endif() + find_package(Python3 REQUIRED COMPONENTS Interpreter) target_sources(${MICROPY_TARGET} PRIVATE From 1034b17558b108b65aca480d3b5b8f933e128369 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 11 Feb 2025 17:46:33 +1100 Subject: [PATCH 085/110] tools/ci.sh: Build the W5100S_EVB_PICO board with no threads. Serves as a build test for a config we don't otherwise support. Signed-off-by: Angus Gratton --- tools/ci.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/ci.sh b/tools/ci.sh index ff362efd29e2d..d50af4a0ae2ee 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -342,7 +342,8 @@ function ci_rp2_build { make ${MAKEOPTS} -C ports/rp2 BOARD=RPI_PICO2 submodules make ${MAKEOPTS} -C ports/rp2 BOARD=RPI_PICO2 make ${MAKEOPTS} -C ports/rp2 BOARD=W5100S_EVB_PICO submodules - make ${MAKEOPTS} -C ports/rp2 BOARD=W5100S_EVB_PICO + # This build doubles as a build test for disabling threads in the config + make ${MAKEOPTS} -C ports/rp2 BOARD=W5100S_EVB_PICO CFLAGS_EXTRA=-DMICROPY_PY_THREAD=0 # Test building ninaw10 driver and NIC interface. make ${MAKEOPTS} -C ports/rp2 BOARD=ARDUINO_NANO_RP2040_CONNECT submodules From 6425c9ecc746eec797f7ba9a37d803244f7d2192 Mon Sep 17 00:00:00 2001 From: Alessandro Gatti Date: Fri, 14 Feb 2025 00:40:22 +0100 Subject: [PATCH 086/110] esp32: Remove unneeded "memory.h" header file. This commit removes "memory.h" from the ESP32 port tree, as it is no longer needed with recent ESP-IDF versions. Signed-off-by: Alessandro Gatti --- ports/esp32/memory.h | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 ports/esp32/memory.h diff --git a/ports/esp32/memory.h b/ports/esp32/memory.h deleted file mode 100644 index 1f07fe409d7eb..0000000000000 --- a/ports/esp32/memory.h +++ /dev/null @@ -1,2 +0,0 @@ -// this is needed for lib/crypto-algorithms/sha256.c -#include From 1e3cce1397c76f7714a5c4940b5021965944d268 Mon Sep 17 00:00:00 2001 From: robert-hh Date: Thu, 20 Feb 2025 09:19:33 +0100 Subject: [PATCH 087/110] mimxrt/boards/ADAFRUIT_METRO_M7: Reduce flash freq to 100MHz. It was set to 133Mhz, but that is not stable. Reduce to 100MHz. The UF2 bootloader runs at 100MHz, so no need for a change of the bootloader. Signed-off-by: robert-hh --- ports/mimxrt/boards/ADAFRUIT_METRO_M7/mpconfigboard.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ports/mimxrt/boards/ADAFRUIT_METRO_M7/mpconfigboard.mk b/ports/mimxrt/boards/ADAFRUIT_METRO_M7/mpconfigboard.mk index b9c1716a7f60c..520cea063fb7e 100644 --- a/ports/mimxrt/boards/ADAFRUIT_METRO_M7/mpconfigboard.mk +++ b/ports/mimxrt/boards/ADAFRUIT_METRO_M7/mpconfigboard.mk @@ -4,7 +4,7 @@ MCU_VARIANT = MIMXRT1011DAE5A MICROPY_FLOAT_IMPL = single MICROPY_HW_FLASH_TYPE ?= qspi_nor_flash MICROPY_HW_FLASH_SIZE ?= 0x800000 # 8MB -MICROPY_HW_FLASH_CLK = kFlexSpiSerialClk_133MHz +MICROPY_HW_FLASH_CLK = kFlexSpiSerialClk_100MHz MICROPY_HW_FLASH_QE_CMD = 0x31 MICROPY_HW_FLASH_QE_ARG = 0x02 From 4364d9411ad4c71bbf6de9bcc82683d6ee671cf5 Mon Sep 17 00:00:00 2001 From: robert-hh Date: Thu, 20 Feb 2025 09:19:33 +0100 Subject: [PATCH 088/110] mimxrt/hal/flexspi_nor_flash: Fix typo in comment about frequency. Signed-off-by: robert-hh --- ports/mimxrt/hal/flexspi_nor_flash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ports/mimxrt/hal/flexspi_nor_flash.c b/ports/mimxrt/hal/flexspi_nor_flash.c index 7fdb6b7c13df6..1476c1ec3a5db 100644 --- a/ports/mimxrt/hal/flexspi_nor_flash.c +++ b/ports/mimxrt/hal/flexspi_nor_flash.c @@ -83,7 +83,7 @@ static ps_div_t div_table_mhz[] = { { 18, 8 }, // 60 -> 60 MHz { 23, 5 }, // 75 -> 75.13 MHz { 18, 6 }, // 80 -> 80 MHz - { 17, 5 }, // 100 -> 191 Mhz + { 17, 5 }, // 100 -> 101 Mhz { 13, 5 }, // 133 -> 132.92 MHz { 13, 4 } // 166 -> 166.15 MHz }; From 2992e3495630a8b83683db28241d638dbc53c911 Mon Sep 17 00:00:00 2001 From: Glenn Moloney Date: Tue, 19 Sep 2023 13:04:46 +1000 Subject: [PATCH 089/110] tools/mpremote: Add support for relative urls in package.json files. URLs in `package.json` may now be specified relative to the base URL of the `package.json` file. Relative URLs wil work for `package.json` files installed from the web as well as local file paths. Docs: update `docs/reference/packages.rst` to add documentation for: - Installing packages from local filesystems (PR #12476); and - Using relative URLs in the `package.json` file (PR #12477); - Update the packaging example to encourage relative URLs as the default in `package.json`. Add `tools/mpremote/tests/test_mip_local_install.sh` to test the installation of a package from local files using relative URLs in the `package.json`. Signed-off-by: Glenn Moloney --- docs/reference/packages.rst | 48 +++++++++++-- tools/mpremote/mpremote/mip.py | 42 ++++++++---- .../mpremote/tests/test_mip_local_install.sh | 67 +++++++++++++++++++ .../tests/test_mip_local_install.sh.exp | 11 +++ 4 files changed, 149 insertions(+), 19 deletions(-) create mode 100755 tools/mpremote/tests/test_mip_local_install.sh create mode 100644 tools/mpremote/tests/test_mip_local_install.sh.exp diff --git a/docs/reference/packages.rst b/docs/reference/packages.rst index 86b8efaa1953d..5b5f626d45232 100644 --- a/docs/reference/packages.rst +++ b/docs/reference/packages.rst @@ -96,6 +96,18 @@ The ``--target=path``, ``--no-mpy``, and ``--index`` arguments can be set:: $ mpremote mip install --no-mpy pkgname $ mpremote mip install --index https://host/pi pkgname +:term:`mpremote` can also install packages from files stored on the host's local +filesystem:: + + $ mpremote mip install path/to/pkg.py + $ mpremote mip install path/to/app/package.json + $ mpremote mip install \\path\\to\\pkg.py + +This is especially useful for testing packages during development and for +installing packages from local clones of GitHub repositories. Note that URLs in +``package.json`` files must use forward slashes ("/") as directory separators, +even on Windows, so that they are compatible with installing from the web. + Installing packages manually ---------------------------- @@ -116,12 +128,25 @@ To write a "self-hosted" package that can be downloaded by ``mip`` or ``mpremote``, you need a static webserver (or GitHub) to host either a single .py file, or a ``package.json`` file alongside your .py files. -A typical ``package.json`` for an example ``mlx90640`` library looks like:: +An example ``mlx90640`` library hosted on GitHub could be installed with:: + + $ mpremote mip install github:org/micropython-mlx90640 + +The layout for the package on GitHub might look like:: + + https://github.com/org/micropython-mlx90640/ + package.json + mlx90640/ + __init__.py + utils.py + +The ``package.json`` specifies the location of files to be installed and other +dependencies:: { "urls": [ - ["mlx90640/__init__.py", "github:org/micropython-mlx90640/mlx90640/__init__.py"], - ["mlx90640/utils.py", "github:org/micropython-mlx90640/mlx90640/utils.py"] + ["mlx90640/__init__.py", "mlx90640/__init__.py"], + ["mlx90640/utils.py", "mlx90640/utils.py"] ], "deps": [ ["collections-defaultdict", "latest"], @@ -132,9 +157,20 @@ A typical ``package.json`` for an example ``mlx90640`` library looks like:: "version": "0.2" } -This includes two files, hosted at a GitHub repo named -``org/micropython-mlx90640``, which install into the ``mlx90640`` directory on -the device. It depends on ``collections-defaultdict`` and ``os-path`` which will +The ``urls`` list specifies the files to be installed according to:: + + "urls": [ + [destination_path, source_url] + ... + +where ``destination_path`` is the location and name of the file to be installed +on the device and ``source_url`` is the URL of the file to be installed. The +source URL would usually be specified relative to the directory containing the +``package.json`` file, but can also be an absolute URL, eg:: + + ["mlx90640/utils.py", "github:org/micropython-mlx90640/mlx90640/utils.py"] + +The package depends on ``collections-defaultdict`` and ``os-path`` which will be installed automatically from the :term:`micropython-lib`. The third dependency installs the content as defined by the ``package.json`` file of the ``main`` branch of the GitHub repo ``org/micropython-additions``. diff --git a/tools/mpremote/mpremote/mip.py b/tools/mpremote/mpremote/mip.py index 4ce62b7f77bbb..858b7933d11d8 100644 --- a/tools/mpremote/mpremote/mip.py +++ b/tools/mpremote/mpremote/mip.py @@ -7,6 +7,7 @@ import json import tempfile import os +import os.path from .commands import CommandError, show_progress_bar @@ -64,22 +65,33 @@ def _rewrite_url(url, branch=None): def _download_file(transport, url, dest): - try: - with urllib.request.urlopen(url) as src: - data = src.read() - print("Installing:", dest) - _ensure_path_exists(transport, dest) - transport.fs_writefile(dest, data, progress_callback=show_progress_bar) - except urllib.error.HTTPError as e: - if e.status == 404: - raise CommandError(f"File not found: {url}") - else: - raise CommandError(f"Error {e.status} requesting {url}") - except urllib.error.URLError as e: - raise CommandError(f"{e.reason} requesting {url}") + if url.startswith(allowed_mip_url_prefixes): + try: + with urllib.request.urlopen(url) as src: + data = src.read() + except urllib.error.HTTPError as e: + if e.status == 404: + raise CommandError(f"File not found: {url}") + else: + raise CommandError(f"Error {e.status} requesting {url}") + except urllib.error.URLError as e: + raise CommandError(f"{e.reason} requesting {url}") + else: + if "\\" in url: + raise CommandError(f'Use "/" instead of "\\" in file URLs: {url!r}\n') + try: + with open(url, "rb") as f: + data = f.read() + except OSError as e: + raise CommandError(f"{e.strerror} opening {url}") + + print("Installing:", dest) + _ensure_path_exists(transport, dest) + transport.fs_writefile(dest, data, progress_callback=show_progress_bar) def _install_json(transport, package_json_url, index, target, version, mpy): + base_url = "" if package_json_url.startswith(allowed_mip_url_prefixes): try: with urllib.request.urlopen(_rewrite_url(package_json_url, version)) as response: @@ -91,12 +103,14 @@ def _install_json(transport, package_json_url, index, target, version, mpy): raise CommandError(f"Error {e.status} requesting {package_json_url}") except urllib.error.URLError as e: raise CommandError(f"{e.reason} requesting {package_json_url}") + base_url = package_json_url.rpartition("/")[0] elif package_json_url.endswith(".json"): try: with open(package_json_url, "r") as f: package_json = json.load(f) except OSError: raise CommandError(f"Error opening {package_json_url}") + base_url = os.path.dirname(package_json_url) else: raise CommandError(f"Invalid url for package: {package_json_url}") for target_path, short_hash in package_json.get("hashes", ()): @@ -105,6 +119,8 @@ def _install_json(transport, package_json_url, index, target, version, mpy): _download_file(transport, file_url, fs_target_path) for target_path, url in package_json.get("urls", ()): fs_target_path = target + "/" + target_path + if base_url and not url.startswith(allowed_mip_url_prefixes): + url = f"{base_url}/{url}" # Relative URLs _download_file(transport, _rewrite_url(url, version), fs_target_path) for dep, dep_version in package_json.get("deps", ()): _install_package(transport, dep, index, target, dep_version, mpy) diff --git a/tools/mpremote/tests/test_mip_local_install.sh b/tools/mpremote/tests/test_mip_local_install.sh new file mode 100755 index 0000000000000..fb8c597bd1484 --- /dev/null +++ b/tools/mpremote/tests/test_mip_local_install.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +# This test the "mpremote mip install" from local files. It creates a package +# and "mip installs" it into a ramdisk. The package is then imported and +# executed. The package is a simple "Hello, world!" example. + +set -e + +PACKAGE=mip_example +PACKAGE_DIR=${TMP}/example +MODULE_DIR=${PACKAGE_DIR}/${PACKAGE} + +target=/__ramdisk +block_size=512 +num_blocks=50 + +# Create the smallest permissible ramdisk. +cat << EOF > "${TMP}/ramdisk.py" +class RAMBlockDev: + def __init__(self, block_size, num_blocks): + self.block_size = block_size + self.data = bytearray(block_size * num_blocks) + + def readblocks(self, block_num, buf): + for i in range(len(buf)): + buf[i] = self.data[block_num * self.block_size + i] + + def writeblocks(self, block_num, buf): + for i in range(len(buf)): + self.data[block_num * self.block_size + i] = buf[i] + + def ioctl(self, op, arg): + if op == 4: # get number of blocks + return len(self.data) // self.block_size + if op == 5: # get block size + return self.block_size + +import os + +bdev = RAMBlockDev(${block_size}, ${num_blocks}) +os.VfsFat.mkfs(bdev) +os.mount(bdev, '${target}') +EOF + +echo ----- Setup +mkdir -p ${MODULE_DIR} +echo "def hello(): print('Hello, world!')" > ${MODULE_DIR}/hello.py +echo "from .hello import hello" > ${MODULE_DIR}/__init__.py +cat > ${PACKAGE_DIR}/package.json < Date: Wed, 12 Feb 2025 14:03:31 +1100 Subject: [PATCH 090/110] github/workflows: Stop using ubuntu-20.04. For GitHub Actions, ubuntu-20.04 is deprecated and will be removed by 1st April 2025. See announcement at https://github.com/actions/runner-images/issues/11101 This commit changes actions that use ubuntu-20.04 to a newer image. Signed-off-by: Damien George --- .github/workflows/mpy_format.yml | 2 +- .github/workflows/ports_esp32.yml | 2 +- .github/workflows/ports_mimxrt.yml | 2 +- .github/workflows/ports_nrf.yml | 2 +- .github/workflows/ports_renesas-ra.yml | 2 +- .github/workflows/ports_stm32.yml | 2 +- .github/workflows/ports_unix.yml | 8 ++++---- tools/ci.sh | 17 +++++++++++------ 8 files changed, 21 insertions(+), 16 deletions(-) diff --git a/.github/workflows/mpy_format.yml b/.github/workflows/mpy_format.yml index baa02ce08d507..b6768a46c3cdc 100644 --- a/.github/workflows/mpy_format.yml +++ b/.github/workflows/mpy_format.yml @@ -15,7 +15,7 @@ concurrency: jobs: test: - runs-on: ubuntu-20.04 # use 20.04 to get python2 + runs-on: ubuntu-22.04 # use 22.04 to get python2 steps: - uses: actions/checkout@v4 - name: Install packages diff --git a/.github/workflows/ports_esp32.yml b/.github/workflows/ports_esp32.yml index 45808b659add7..0d76974f01946 100644 --- a/.github/workflows/ports_esp32.yml +++ b/.github/workflows/ports_esp32.yml @@ -25,7 +25,7 @@ jobs: ci_func: # names are functions in ci.sh - esp32_build_cmod_spiram_s2 - esp32_build_s3_c3 - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/ports_mimxrt.yml b/.github/workflows/ports_mimxrt.yml index 9e782bf63b31c..7743e036ab377 100644 --- a/.github/workflows/ports_mimxrt.yml +++ b/.github/workflows/ports_mimxrt.yml @@ -19,7 +19,7 @@ concurrency: jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest defaults: run: working-directory: 'micropython repo' # test build with space in path diff --git a/.github/workflows/ports_nrf.yml b/.github/workflows/ports_nrf.yml index d9cffb9778cba..76727c9d1f6bd 100644 --- a/.github/workflows/ports_nrf.yml +++ b/.github/workflows/ports_nrf.yml @@ -19,7 +19,7 @@ concurrency: jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Install packages diff --git a/.github/workflows/ports_renesas-ra.yml b/.github/workflows/ports_renesas-ra.yml index b1a30c2f11798..b9fa74331dc0b 100644 --- a/.github/workflows/ports_renesas-ra.yml +++ b/.github/workflows/ports_renesas-ra.yml @@ -19,7 +19,7 @@ concurrency: jobs: build_renesas_ra_board: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Install packages diff --git a/.github/workflows/ports_stm32.yml b/.github/workflows/ports_stm32.yml index f5e01dc1f620e..8800f145189b8 100644 --- a/.github/workflows/ports_stm32.yml +++ b/.github/workflows/ports_stm32.yml @@ -26,7 +26,7 @@ jobs: - stm32_pyb_build - stm32_nucleo_build - stm32_misc_build - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - name: Install packages diff --git a/.github/workflows/ports_unix.yml b/.github/workflows/ports_unix.yml index a095136b4a65e..2547015038e4d 100644 --- a/.github/workflows/ports_unix.yml +++ b/.github/workflows/ports_unix.yml @@ -98,7 +98,7 @@ jobs: run: tests/run-tests.py --print-failures coverage_32bit: - runs-on: ubuntu-20.04 # use 20.04 to get libffi-dev:i386 + runs-on: ubuntu-22.04 # use 22.04 to get libffi-dev:i386 steps: - uses: actions/checkout@v4 - name: Install packages @@ -116,7 +116,7 @@ jobs: run: tests/run-tests.py --print-failures nanbox: - runs-on: ubuntu-20.04 # use 20.04 to get python2, and libffi-dev:i386 + runs-on: ubuntu-22.04 # use 22.04 to get python2, and libffi-dev:i386 steps: - uses: actions/checkout@v4 - name: Install packages @@ -142,7 +142,7 @@ jobs: run: tests/run-tests.py --print-failures stackless_clang: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Install packages @@ -156,7 +156,7 @@ jobs: run: tests/run-tests.py --print-failures float_clang: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Install packages diff --git a/tools/ci.sh b/tools/ci.sh index d50af4a0ae2ee..9125b5bc9f135 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -106,12 +106,16 @@ function ci_code_size_build { # .mpy file format function ci_mpy_format_setup { + sudo apt-get update + sudo apt-get install python2.7 sudo pip3 install pyelftools + python2.7 --version + python3 --version } function ci_mpy_format_test { # Test mpy-tool.py dump feature on bytecode - python2 ./tools/mpy-tool.py -xd tests/frozen/frozentest.mpy + python2.7 ./tools/mpy-tool.py -xd tests/frozen/frozentest.mpy python3 ./tools/mpy-tool.py -xd tests/frozen/frozentest.mpy # Test mpy-tool.py dump feature on native code @@ -268,18 +272,18 @@ function ci_powerpc_build { # ports/qemu function ci_qemu_setup_arm { - ci_mpy_format_setup ci_gcc_arm_setup sudo apt-get update sudo apt-get install qemu-system + sudo pip3 install pyelftools qemu-system-arm --version } function ci_qemu_setup_rv32 { - ci_mpy_format_setup ci_gcc_riscv_setup sudo apt-get update sudo apt-get install qemu-system + sudo pip3 install pyelftools qemu-system-riscv32 --version } @@ -580,10 +584,11 @@ function ci_unix_coverage_run_native_mpy_tests { function ci_unix_32bit_setup { sudo dpkg --add-architecture i386 sudo apt-get update - sudo apt-get install gcc-multilib g++-multilib libffi-dev:i386 + sudo apt-get install gcc-multilib g++-multilib libffi-dev:i386 python2.7 sudo pip3 install setuptools sudo pip3 install pyelftools gcc --version + python2.7 --version python3 --version } @@ -602,12 +607,12 @@ function ci_unix_coverage_32bit_run_native_mpy_tests { function ci_unix_nanbox_build { # Use Python 2 to check that it can run the build scripts - ci_unix_build_helper PYTHON=python2 VARIANT=nanbox CFLAGS_EXTRA="-DMICROPY_PY_MATH_CONSTANTS=1" + ci_unix_build_helper PYTHON=python2.7 VARIANT=nanbox CFLAGS_EXTRA="-DMICROPY_PY_MATH_CONSTANTS=1" ci_unix_build_ffi_lib_helper gcc -m32 } function ci_unix_nanbox_run_tests { - ci_unix_run_tests_full_helper nanbox PYTHON=python2 + ci_unix_run_tests_full_helper nanbox PYTHON=python2.7 } function ci_unix_float_build { From 8b1ed4473dd215d564498f577cb6f3339c3c2e57 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Wed, 12 Feb 2025 16:46:43 +1100 Subject: [PATCH 091/110] github/workflows: Include the Python version in the ESP-IDF cache key. Changing runner OS can change Python version, and ESP-IDF installs are keyed on ESP-IDF and Python version together. Signed-off-by: Angus Gratton --- .github/workflows/ports_esp32.yml | 4 ++-- tools/ci.sh | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ports_esp32.yml b/.github/workflows/ports_esp32.yml index 0d76974f01946..4c07f89437757 100644 --- a/.github/workflows/ports_esp32.yml +++ b/.github/workflows/ports_esp32.yml @@ -30,8 +30,8 @@ jobs: - uses: actions/checkout@v4 - id: idf_ver - name: Read the ESP-IDF version - run: source tools/ci.sh && echo "IDF_VER=$IDF_VER" | tee "$GITHUB_OUTPUT" + name: Read the ESP-IDF version (including Python version) + run: source tools/ci.sh && echo "IDF_VER=${IDF_VER}-py${PYTHON_VER}" | tee "$GITHUB_OUTPUT" - name: Cached ESP-IDF install id: cache_esp_idf diff --git a/tools/ci.sh b/tools/ci.sh index 9125b5bc9f135..5b108d620258c 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -140,6 +140,7 @@ function ci_cc3200_build { # GitHub tag of ESP-IDF to use for CI (note: must be a tag or a branch) IDF_VER=v5.2.2 +PYTHON_VER=$(python --version | cut -d' ' -f2) export IDF_CCACHE_ENABLE=1 From 5e206fdeb5225ce24efb9411cc879bafaa04b86f Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 18 Feb 2025 13:24:11 +0100 Subject: [PATCH 092/110] all: Upgrade codespell to v2.4.1. This commit upgrades from codespell==2.2.6 to the current codespell==2.4.1, adding emac to the ignore-words-list. Signed-off-by: Christian Clauss --- .github/workflows/codespell.yml | 2 +- .pre-commit-config.yaml | 2 +- docs/reference/isr_rules.rst | 2 +- docs/reference/speed_python.rst | 2 +- examples/usercmodule/cexample/examplemodule.c | 2 +- extmod/modlwip.c | 2 +- ports/esp32/mpthreadport.c | 2 +- ports/mimxrt/boards/ADAFRUIT_METRO_M7/mpconfigboard.h | 2 +- ports/mimxrt/eth.c | 2 +- ports/mimxrt/hal/flexspi_flash_config.h | 2 +- ports/mimxrt/sdcard.c | 2 +- ports/nrf/boards/MICROBIT/modules/microbitfont.h | 2 +- ports/rp2/main.c | 2 +- ports/samd/mcu/samd21/mpconfigmcu.h | 2 +- ports/samd/mcu/samd51/mpconfigmcu.h | 2 +- ports/windows/windows_mphal.c | 2 +- py/reader.h | 2 +- pyproject.toml | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 2d8b4627aac2d..1d6b1dc9d2b27 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -8,6 +8,6 @@ jobs: steps: - uses: actions/checkout@v4 # codespell version should be kept in sync with .pre-commit-config.yml - - run: pip install --user codespell==2.2.6 tomli + - run: pip install --user codespell==2.4.1 tomli - run: codespell diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a1c811339adcc..193b2c6e83559 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: - id: ruff-format - repo: https://github.com/codespell-project/codespell # Version should be kept in sync with .github/workflows/codespell.yml - rev: v2.2.6 + rev: v2.4.1 hooks: - id: codespell name: Spellcheck for changed files (codespell) diff --git a/docs/reference/isr_rules.rst b/docs/reference/isr_rules.rst index 14010fb207fdd..5e8d6ad61201a 100644 --- a/docs/reference/isr_rules.rst +++ b/docs/reference/isr_rules.rst @@ -209,7 +209,7 @@ issue a further interrupt. It then schedules a callback to process the data. Scheduled callbacks should comply with the principles of interrupt handler design outlined below. This is to avoid problems resulting from I/O activity and the modification of shared data which can arise in any code -which pre-empts the main program loop. +which preempts the main program loop. Execution time needs to be considered in relation to the frequency with which interrupts can occur. If an interrupt occurs while the previous callback is executing, a further instance of the callback will be queued diff --git a/docs/reference/speed_python.rst b/docs/reference/speed_python.rst index a660432f6da37..64fd9df6c058a 100644 --- a/docs/reference/speed_python.rst +++ b/docs/reference/speed_python.rst @@ -219,7 +219,7 @@ process known as garbage collection reclaims the memory used by these redundant objects and the allocation is then tried again - a process which can take several milliseconds. -There may be benefits in pre-empting this by periodically issuing `gc.collect()`. +There may be benefits in preempting this by periodically issuing `gc.collect()`. Firstly doing a collection before it is actually required is quicker - typically on the order of 1ms if done frequently. Secondly you can determine the point in code where this time is used rather than have a longer delay occur at random points, diff --git a/examples/usercmodule/cexample/examplemodule.c b/examples/usercmodule/cexample/examplemodule.c index d13515e72cdd5..83cc3b27c058e 100644 --- a/examples/usercmodule/cexample/examplemodule.c +++ b/examples/usercmodule/cexample/examplemodule.c @@ -74,7 +74,7 @@ MP_DEFINE_CONST_OBJ_TYPE( // - A custom representation for __repr__ and __str__. // - Custom attribute handling to create a read/write "property". // -// It re-uses some of the elements of the basic Timer class. This is allowed +// It reuses some of the elements of the basic Timer class. This is allowed // because they both use example_Timer_obj_t as the instance structure. // Handles AdvancedTimer.__repr__, AdvancedTimer.__str__. diff --git a/extmod/modlwip.c b/extmod/modlwip.c index bcccd363e673c..f109e0029bb0f 100644 --- a/extmod/modlwip.c +++ b/extmod/modlwip.c @@ -476,7 +476,7 @@ static void _lwip_tcp_err_unaccepted(void *arg, err_t err) { // because it's only ever used by lwIP if tcp_connect is called on the TCP PCB. lwip_socket_obj_t *socket = (lwip_socket_obj_t *)pcb->connected; - // Array is not volatile because thiss callback is executed within the lwIP context + // Array is not volatile because this callback is executed within the lwIP context uint8_t alloc = socket->incoming.connection.alloc; struct tcp_pcb **tcp_array = (struct tcp_pcb **)lwip_socket_incoming_array(socket); diff --git a/ports/esp32/mpthreadport.c b/ports/esp32/mpthreadport.c index f436ebb80a7ad..962b5780d08dd 100644 --- a/ports/esp32/mpthreadport.c +++ b/ports/esp32/mpthreadport.c @@ -213,7 +213,7 @@ int mp_thread_mutex_lock(mp_thread_mutex_t *mutex, int wait) { void mp_thread_mutex_unlock(mp_thread_mutex_t *mutex) { xSemaphoreGive(mutex->handle); - // Python threads run at equal priority, so pre-emptively yield here to + // Python threads run at equal priority, so preemptively yield here to // prevent pathological imbalances where a thread unlocks and then // immediately re-locks a mutex before a context switch can occur, leaving // another thread waiting for an unbounded period of time. diff --git a/ports/mimxrt/boards/ADAFRUIT_METRO_M7/mpconfigboard.h b/ports/mimxrt/boards/ADAFRUIT_METRO_M7/mpconfigboard.h index 784373c939478..1ce06e7f408d7 100644 --- a/ports/mimxrt/boards/ADAFRUIT_METRO_M7/mpconfigboard.h +++ b/ports/mimxrt/boards/ADAFRUIT_METRO_M7/mpconfigboard.h @@ -52,7 +52,7 @@ { IOMUXC_GPIO_02_LPI2C1_SCL }, { IOMUXC_GPIO_01_LPI2C1_SDA }, \ { IOMUXC_GPIO_10_LPI2C2_SCL }, { IOMUXC_GPIO_09_LPI2C2_SDA }, -// Wifi Deinitions +// Wifi Definitions #define MICROPY_PY_NETWORK_HOSTNAME_DEFAULT "mpy-metro-m7" #define MICROPY_HW_WIFI_SPI_ID (0) diff --git a/ports/mimxrt/eth.c b/ports/mimxrt/eth.c index 3eb1a0cc35202..1ac19b83d6d44 100644 --- a/ports/mimxrt/eth.c +++ b/ports/mimxrt/eth.c @@ -327,7 +327,7 @@ static void eth_gpio_init(const iomux_table_t iomux_table[], size_t iomux_table_ } } -// eth_phy_init: Initilaize the PHY interface +// eth_phy_init: Initialize the PHY interface static void eth_phy_init(phy_handle_t *phyHandle, phy_config_t *phy_config, phy_speed_t *speed, phy_duplex_t *duplex, uint32_t phy_settle_time) { diff --git a/ports/mimxrt/hal/flexspi_flash_config.h b/ports/mimxrt/hal/flexspi_flash_config.h index d9aa5ed5eab69..239d7b3b4f868 100644 --- a/ports/mimxrt/hal/flexspi_flash_config.h +++ b/ports/mimxrt/hal/flexspi_flash_config.h @@ -119,7 +119,7 @@ enum kFlexSpiMiscOffset_WordAddressableEnable = 3, // !< Bit for Word Addressable enable kFlexSpiMiscOffset_SafeConfigFreqEnable = 4, // !< Bit for Safe Configuration Frequency enable kFlexSpiMiscOffset_PadSettingOverrideEnable = 5, // !< Bit for Pad setting override enable - kFlexSpiMiscOffset_DdrModeEnable = 6, // !< Bit for DDR clock confiuration indication. + kFlexSpiMiscOffset_DdrModeEnable = 6, // !< Bit for DDR clock configuration indication. }; // !@brief Flash Type Definition diff --git a/ports/mimxrt/sdcard.c b/ports/mimxrt/sdcard.c index ae52ed5d2cd09..84e2c98949d22 100644 --- a/ports/mimxrt/sdcard.c +++ b/ports/mimxrt/sdcard.c @@ -980,7 +980,7 @@ bool sdcard_power_on(mimxrt_sdcard_obj_t *card) { return false; } - // Finialize initialization + // Finalize initialization card->state->initialized = true; return true; } diff --git a/ports/nrf/boards/MICROBIT/modules/microbitfont.h b/ports/nrf/boards/MICROBIT/modules/microbitfont.h index 2ae0c8fab86ba..6e80acfe07214 100644 --- a/ports/nrf/boards/MICROBIT/modules/microbitfont.h +++ b/ports/nrf/boards/MICROBIT/modules/microbitfont.h @@ -36,7 +36,7 @@ DEALINGS IN THE SOFTWARE. * * Example: { 0x08, 0x08, 0x08, 0x0, 0x08 } * - * The above will produce an exclaimation mark on the second column in form the left. + * The above will produce an exclamation mark on the second column in form the left. * * We could compress further, but the complexity of decode would likely outweigh the gains. */ diff --git a/ports/rp2/main.c b/ports/rp2/main.c index d6bf448267152..58da63c06f12d 100644 --- a/ports/rp2/main.c +++ b/ports/rp2/main.c @@ -90,7 +90,7 @@ int main(int argc, char **argv) { // Set the MCU frequency and as a side effect the peripheral clock to 48 MHz. set_sys_clock_khz(SYS_CLK_KHZ, false); - // Hook for setting up anything that needs to be super early in the bootup process. + // Hook for setting up anything that needs to be super early in the boot-up process. MICROPY_BOARD_STARTUP(); #if MICROPY_HW_ENABLE_UART_REPL diff --git a/ports/samd/mcu/samd21/mpconfigmcu.h b/ports/samd/mcu/samd21/mpconfigmcu.h index 29965f50f63a8..f0a7a73e0c027 100644 --- a/ports/samd/mcu/samd21/mpconfigmcu.h +++ b/ports/samd/mcu/samd21/mpconfigmcu.h @@ -1,4 +1,4 @@ -// Deinitions common to all SAMD21 boards +// Definitions common to all SAMD21 boards #include "samd21.h" #define MICROPY_CONFIG_ROM_LEVEL (MICROPY_CONFIG_ROM_LEVEL_BASIC_FEATURES) diff --git a/ports/samd/mcu/samd51/mpconfigmcu.h b/ports/samd/mcu/samd51/mpconfigmcu.h index 9a7b8528f3573..831949bab56a0 100644 --- a/ports/samd/mcu/samd51/mpconfigmcu.h +++ b/ports/samd/mcu/samd51/mpconfigmcu.h @@ -1,4 +1,4 @@ -// Deinitions common to all SAMD51 boards +// Definitions common to all SAMD51 boards #include "samd51.h" #define MICROPY_CONFIG_ROM_LEVEL (MICROPY_CONFIG_ROM_LEVEL_FULL_FEATURES) diff --git a/ports/windows/windows_mphal.c b/ports/windows/windows_mphal.c index b636393f5401c..5a65b2fd2aeae 100644 --- a/ports/windows/windows_mphal.c +++ b/ports/windows/windows_mphal.c @@ -70,7 +70,7 @@ void mp_hal_stdio_mode_orig(void) { } // Handler to be installed by SetConsoleCtrlHandler, currently used only to handle Ctrl-C. -// This handler has to be installed just once (this has to be done elswhere in init code). +// This handler has to be installed just once (this has to be done elsewhere in init code). // Previous versions of the mp_hal code would install a handler whenever Ctrl-C input is // allowed and remove the handler again when it is not. That is not necessary though (1), // and it might introduce problems (2) because console notifications are delivered to the diff --git a/py/reader.h b/py/reader.h index 301c70ab3f0d4..6378457007cb3 100644 --- a/py/reader.h +++ b/py/reader.h @@ -48,7 +48,7 @@ void mp_reader_new_file(mp_reader_t *reader, qstr filename); void mp_reader_new_file_from_fd(mp_reader_t *reader, int fd, bool close_fd); // Try to efficiently read the given number of bytes from a ROM-based reader. -// Returns a valid, non-NULL pointer to the requseted data if the reader points to ROM. +// Returns a valid, non-NULL pointer to the requested data if the reader points to ROM. // Returns NULL if the reader does not point to ROM. const uint8_t *mp_reader_try_read_rom(mp_reader_t *reader, size_t len); diff --git a/pyproject.toml b/pyproject.toml index 1650bd088ea22..aeb25c8a03a2b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.codespell] count = "" ignore-regex = '\b[A-Z]{3}\b' -ignore-words-list = "ans,asend,deques,dout,extint,hsi,iput,mis,numer,shft,synopsys,technic,ure" +ignore-words-list = "ans,asend,deques,dout,emac,extint,hsi,iput,mis,numer,shft,synopsys,technic,ure" quiet-level = 3 skip = """ */build*,\ From 3f0dd13d931c88a759a779853dfeaee566f0e79a Mon Sep 17 00:00:00 2001 From: Ronald Weber Date: Mon, 24 Feb 2025 17:34:29 +0100 Subject: [PATCH 093/110] docs: Fix double 'the' in documentation. Signed-off-by: Ronald Weber --- docs/esp8266/tutorial/repl.rst | 2 +- docs/library/framebuf.rst | 2 +- docs/library/machine.UART.rst | 2 +- docs/reference/repl.rst | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/esp8266/tutorial/repl.rst b/docs/esp8266/tutorial/repl.rst index bc0142aaef5f9..db55d45facde8 100644 --- a/docs/esp8266/tutorial/repl.rst +++ b/docs/esp8266/tutorial/repl.rst @@ -38,7 +38,7 @@ browser. The latest versions of Firefox and Chrome are supported. For your convenience, WebREPL client is hosted at ``__. Alternatively, you can install it -locally from the the GitHub repository +locally from the GitHub repository ``__. Before connecting to WebREPL, you should set a password and enable it via diff --git a/docs/library/framebuf.rst b/docs/library/framebuf.rst index 149f4d6609be9..f22a3613bdbcb 100644 --- a/docs/library/framebuf.rst +++ b/docs/library/framebuf.rst @@ -114,7 +114,7 @@ Drawing text .. method:: FrameBuffer.text(s, x, y[, c]) - Write text to the FrameBuffer using the the coordinates as the upper-left + Write text to the FrameBuffer using the coordinates as the upper-left corner of the text. The color of the text can be defined by the optional argument but is otherwise a default value of 1. All characters have dimensions of 8x8 pixels and there is currently no way to change the font. diff --git a/docs/library/machine.UART.rst b/docs/library/machine.UART.rst index 4dcb4a1e7c4a6..5be79cccce884 100644 --- a/docs/library/machine.UART.rst +++ b/docs/library/machine.UART.rst @@ -83,7 +83,7 @@ Methods - *pins* is a 4 or 2 item list indicating the TX, RX, RTS and CTS pins (in that order). Any of the pins can be None if one wants the UART to operate with limited functionality. - If the RTS pin is given the the RX pin must be given as well. The same applies to CTS. + If the RTS pin is given the RX pin must be given as well. The same applies to CTS. When no pins are given, then the default set of TX and RX pins is taken, and hardware flow control will be disabled. If *pins* is ``None``, no pin assignment will be made. diff --git a/docs/reference/repl.rst b/docs/reference/repl.rst index be02ddebde8e2..2c019350f7e2a 100644 --- a/docs/reference/repl.rst +++ b/docs/reference/repl.rst @@ -213,7 +213,7 @@ echo turned off, and with optional flow control. Raw mode is entered using Ctrl-A. You then send your python code, followed by a Ctrl-D. The Ctrl-D will be acknowledged by 'OK' and then the python code will be compiled and executed. Any output (or errors) will be sent back. Entering -Ctrl-B will leave raw mode and return the the regular (aka friendly) REPL. +Ctrl-B will leave raw mode and return the regular (aka friendly) REPL. Raw-paste mode is an additional mode within the raw REPL that includes flow control, and which compiles code as it receives it. This makes it more robust for high-speed From dc2fcfcc5511a371ff684f7d7772e7a7b479246d Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 18 Feb 2025 10:01:09 +0100 Subject: [PATCH 094/110] all: Upgrade to ruff v0.9.6. Signed-off-by: Christian Clauss --- .github/workflows/ruff.yml | 2 +- .pre-commit-config.yaml | 2 +- ports/nrf/examples/seeed_tft.py | 1 + pyproject.toml | 22 ++++++++++++++----- tests/cpydiff/builtin_next_arg2.py | 1 + tests/cpydiff/core_class_delnotimpl.py | 1 + tests/cpydiff/core_fstring_concat.py | 2 ++ tests/cpydiff/core_function_argcount.py | 1 + tests/cpydiff/core_import_all.py | 1 + tests/cpydiff/core_import_path.py | 1 + tests/cpydiff/core_import_split_ns_pkgs.py | 1 + tests/cpydiff/core_locals_eval.py | 1 + tests/cpydiff/module_array_comparison.py | 1 + tests/cpydiff/module_array_constructor.py | 1 + tests/cpydiff/modules_array_containment.py | 1 + tests/cpydiff/modules_array_deletion.py | 1 + tests/cpydiff/modules_array_subscrstep.py | 1 + tests/cpydiff/modules_json_nonserializable.py | 1 + tests/cpydiff/modules_os_environ.py | 1 + tests/cpydiff/modules_os_getenv.py | 1 + tests/cpydiff/modules_struct_fewargs.py | 1 + tests/cpydiff/modules_struct_manyargs.py | 1 + .../modules_struct_whitespace_in_format.py | 1 + tests/cpydiff/modules_sys_stdassign.py | 1 + tests/cpydiff/syntax_assign_expr.py | 1 + tests/cpydiff/syntax_spaces.py | 1 + tests/cpydiff/syntax_unicode_nameesc.py | 1 + tests/cpydiff/types_bytearray_sliceassign.py | 1 + tests/cpydiff/types_bytes_format.py | 1 + tests/cpydiff/types_bytes_keywords.py | 1 + tests/cpydiff/types_bytes_subscrstep.py | 1 + tests/cpydiff/types_dict_keys_set.py | 1 + tests/cpydiff/types_exception_attrs.py | 1 + tests/cpydiff/types_exception_chaining.py | 1 + tests/cpydiff/types_exception_instancevar.py | 1 + tests/cpydiff/types_exception_loops.py | 1 + tests/cpydiff/types_float_rounding.py | 1 + tests/cpydiff/types_list_delete_subscrstep.py | 1 + tests/cpydiff/types_list_store_noniter.py | 1 + tests/cpydiff/types_list_store_subscrstep.py | 1 + tests/cpydiff/types_memoryview_invalid.py | 1 + tests/cpydiff/types_str_endswith.py | 1 + tests/cpydiff/types_str_formatsubscr.py | 1 + tests/cpydiff/types_str_keywords.py | 1 + tests/cpydiff/types_str_ljust_rjust.py | 1 + tests/cpydiff/types_str_rsplitnone.py | 1 + tests/cpydiff/types_str_subscrstep.py | 1 + tests/cpydiff/types_tuple_subscrstep.py | 1 + tests/ports/cc3200/pin.py | 1 + tests/unicode/unicode.py | 2 +- tools/gen-cpydiff.py | 6 ++--- tools/mpremote/mpremote/transport.py | 2 +- tools/pyboard.py | 2 +- tools/pydfu.py | 5 ++--- 54 files changed, 73 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index 29e9ddbf8b61a..9265c25bcba13 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -8,6 +8,6 @@ jobs: steps: - uses: actions/checkout@v4 # ruff version should be kept in sync with .pre-commit-config.yaml - - run: pip install --user ruff==0.1.3 + - run: pipx install ruff==0.9.6 - run: ruff check --output-format=github . - run: ruff format --diff . diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 193b2c6e83559..d07f9b0fda2a9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: stages: [commit-msg] - repo: https://github.com/charliermarsh/ruff-pre-commit # Version should be kept in sync with .github/workflows/ruff.yml - rev: v0.1.3 + rev: v0.9.6 hooks: - id: ruff - id: ruff-format diff --git a/ports/nrf/examples/seeed_tft.py b/ports/nrf/examples/seeed_tft.py index f53b560226363..263da7ae0be7f 100644 --- a/ports/nrf/examples/seeed_tft.py +++ b/ports/nrf/examples/seeed_tft.py @@ -44,6 +44,7 @@ tf = mount_tf() os.listdir() """ + import vfs import time import framebuf diff --git a/pyproject.toml b/pyproject.toml index aeb25c8a03a2b..d5e3d0e847464 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,17 @@ line-length = 99 target-version = "py37" [tool.ruff.lint] +exclude = [ # Ruff finds Python SyntaxError in these files + "tests/cmdline/repl_autocomplete.py", + "tests/cmdline/repl_autoindent.py", + "tests/cmdline/repl_basic.py", + "tests/cmdline/repl_cont.py", + "tests/cmdline/repl_emacs_keys.py", + "tests/cmdline/repl_words_move.py", + "tests/feature_check/repl_emacs_check.py", + "tests/feature_check/repl_words_move_check.py", + "tests/micropython/viper_args.py", +] extend-select = ["C9", "PLC"] ignore = [ "E401", @@ -37,14 +48,12 @@ ignore = [ "F401", "F403", "F405", - "PLC1901", + "PLC0206", ] +mccabe.max-complexity = 40 -[tool.ruff.mccabe] -max-complexity = 40 - -[tool.ruff.per-file-ignores] -# Exclude all tests from linting (does not apply to formatting). +[tool.ruff.lint.per-file-ignores] +# Exclude all tests from linting. "tests/**/*.py" = ["ALL"] "ports/cc3200/tools/uniflash.py" = ["E711"] # manifest.py files are evaluated with some global names pre-defined @@ -57,3 +66,4 @@ max-complexity = 40 # repl_: not real python files # viper_args: uses f(*) exclude = ["tests/basics/*.py", "tests/*/repl_*.py", "tests/micropython/viper_args.py"] +quote-style = "preserve" diff --git a/tests/cpydiff/builtin_next_arg2.py b/tests/cpydiff/builtin_next_arg2.py index 5df2d6e70f837..ed9565fe0fe77 100644 --- a/tests/cpydiff/builtin_next_arg2.py +++ b/tests/cpydiff/builtin_next_arg2.py @@ -9,4 +9,5 @@ except StopIteration: val = deflt """ + print(next(iter(range(0)), 42)) diff --git a/tests/cpydiff/core_class_delnotimpl.py b/tests/cpydiff/core_class_delnotimpl.py index 18c176e9bb1f7..6fac764ac9828 100644 --- a/tests/cpydiff/core_class_delnotimpl.py +++ b/tests/cpydiff/core_class_delnotimpl.py @@ -4,6 +4,7 @@ cause: Unknown workaround: Unknown """ + import gc diff --git a/tests/cpydiff/core_fstring_concat.py b/tests/cpydiff/core_fstring_concat.py index 83bf18820ece1..3daa13d75360e 100644 --- a/tests/cpydiff/core_fstring_concat.py +++ b/tests/cpydiff/core_fstring_concat.py @@ -6,7 +6,9 @@ """ x, y = 1, 2 +# fmt: off print("aa" f"{x}") # works print(f"{x}" "ab") # works print("a{}a" f"{x}") # fails print(f"{x}" "a{}b") # fails +# fmt: on diff --git a/tests/cpydiff/core_function_argcount.py b/tests/cpydiff/core_function_argcount.py index 5f3dca4dcdbe4..c257def726aca 100644 --- a/tests/cpydiff/core_function_argcount.py +++ b/tests/cpydiff/core_function_argcount.py @@ -4,6 +4,7 @@ cause: MicroPython counts "self" as an argument. workaround: Interpret error messages with the information above in mind. """ + try: [].append() except Exception as e: diff --git a/tests/cpydiff/core_import_all.py b/tests/cpydiff/core_import_all.py index 5adf9ae3eb1e6..0fbe9d4d4ecd6 100644 --- a/tests/cpydiff/core_import_all.py +++ b/tests/cpydiff/core_import_all.py @@ -4,6 +4,7 @@ cause: Not implemented. workaround: Manually import the sub-modules directly in __init__.py using ``from . import foo, bar``. """ + from modules3 import * foo.hello() diff --git a/tests/cpydiff/core_import_path.py b/tests/cpydiff/core_import_path.py index 959fd571f57ab..2028386be84b0 100644 --- a/tests/cpydiff/core_import_path.py +++ b/tests/cpydiff/core_import_path.py @@ -4,6 +4,7 @@ cause: MicroPython doesn't support namespace packages split across filesystem. Beyond that, MicroPython's import system is highly optimized for minimal memory usage. workaround: Details of import handling is inherently implementation dependent. Don't rely on such details in portable applications. """ + import modules print(modules.__path__) diff --git a/tests/cpydiff/core_import_split_ns_pkgs.py b/tests/cpydiff/core_import_split_ns_pkgs.py index 5c92b63124a50..a1cfd84893d4e 100644 --- a/tests/cpydiff/core_import_split_ns_pkgs.py +++ b/tests/cpydiff/core_import_split_ns_pkgs.py @@ -4,6 +4,7 @@ cause: MicroPython's import system is highly optimized for simplicity, minimal memory usage, and minimal filesystem search overhead. workaround: Don't install modules belonging to the same namespace package in different directories. For MicroPython, it's recommended to have at most 3-component module search paths: for your current application, per-user (writable), system-wide (non-writable). """ + import sys sys.path.append(sys.path[1] + "/modules") diff --git a/tests/cpydiff/core_locals_eval.py b/tests/cpydiff/core_locals_eval.py index 025d226372986..3f6ad2a1dbbc6 100644 --- a/tests/cpydiff/core_locals_eval.py +++ b/tests/cpydiff/core_locals_eval.py @@ -4,6 +4,7 @@ cause: MicroPython doesn't maintain symbolic local environment, it is optimized to an array of slots. Thus, local variables can't be accessed by a name. Effectively, ``eval(expr)`` in MicroPython is equivalent to ``eval(expr, globals(), globals())``. workaround: Unknown """ + val = 1 diff --git a/tests/cpydiff/module_array_comparison.py b/tests/cpydiff/module_array_comparison.py index a442af3f5bc96..4929b1e590dac 100644 --- a/tests/cpydiff/module_array_comparison.py +++ b/tests/cpydiff/module_array_comparison.py @@ -4,6 +4,7 @@ cause: Code size workaround: Compare individual elements """ + import array array.array("b", [1, 2]) == array.array("i", [1, 2]) diff --git a/tests/cpydiff/module_array_constructor.py b/tests/cpydiff/module_array_constructor.py index 08cf2ef2d1c3c..a53589d22ae76 100644 --- a/tests/cpydiff/module_array_constructor.py +++ b/tests/cpydiff/module_array_constructor.py @@ -4,6 +4,7 @@ cause: MicroPython implements implicit truncation in order to reduce code size and execution time workaround: If CPython compatibility is needed then mask the value explicitly """ + import array a = array.array("b", [257]) diff --git a/tests/cpydiff/modules_array_containment.py b/tests/cpydiff/modules_array_containment.py index 8f25b0d49149d..b29895aa75892 100644 --- a/tests/cpydiff/modules_array_containment.py +++ b/tests/cpydiff/modules_array_containment.py @@ -4,6 +4,7 @@ cause: Unknown workaround: Unknown """ + import array print(1 in array.array("B", b"12")) diff --git a/tests/cpydiff/modules_array_deletion.py b/tests/cpydiff/modules_array_deletion.py index 3376527373edf..2e7c31124b2da 100644 --- a/tests/cpydiff/modules_array_deletion.py +++ b/tests/cpydiff/modules_array_deletion.py @@ -4,6 +4,7 @@ cause: Unknown workaround: Unknown """ + import array a = array.array("b", (1, 2, 3)) diff --git a/tests/cpydiff/modules_array_subscrstep.py b/tests/cpydiff/modules_array_subscrstep.py index 24308bd9042d1..65b12332f54cd 100644 --- a/tests/cpydiff/modules_array_subscrstep.py +++ b/tests/cpydiff/modules_array_subscrstep.py @@ -4,6 +4,7 @@ cause: Unknown workaround: Unknown """ + import array a = array.array("b", (1, 2, 3)) diff --git a/tests/cpydiff/modules_json_nonserializable.py b/tests/cpydiff/modules_json_nonserializable.py index d6a5660cadfdb..d83e7beca2d5a 100644 --- a/tests/cpydiff/modules_json_nonserializable.py +++ b/tests/cpydiff/modules_json_nonserializable.py @@ -4,6 +4,7 @@ cause: Unknown workaround: Unknown """ + import json try: diff --git a/tests/cpydiff/modules_os_environ.py b/tests/cpydiff/modules_os_environ.py index 491d8a3101636..4edde16656781 100644 --- a/tests/cpydiff/modules_os_environ.py +++ b/tests/cpydiff/modules_os_environ.py @@ -4,6 +4,7 @@ cause: Unknown workaround: Use ``getenv``, ``putenv`` and ``unsetenv`` """ + import os try: diff --git a/tests/cpydiff/modules_os_getenv.py b/tests/cpydiff/modules_os_getenv.py index d1e828438e456..d5acd414eb3da 100644 --- a/tests/cpydiff/modules_os_getenv.py +++ b/tests/cpydiff/modules_os_getenv.py @@ -4,6 +4,7 @@ cause: The ``environ`` attribute is not implemented workaround: Unknown """ + import os print(os.getenv("NEW_VARIABLE")) diff --git a/tests/cpydiff/modules_struct_fewargs.py b/tests/cpydiff/modules_struct_fewargs.py index cb6b0fd874ec4..49b2a3213c898 100644 --- a/tests/cpydiff/modules_struct_fewargs.py +++ b/tests/cpydiff/modules_struct_fewargs.py @@ -4,6 +4,7 @@ cause: Unknown workaround: Unknown """ + import struct try: diff --git a/tests/cpydiff/modules_struct_manyargs.py b/tests/cpydiff/modules_struct_manyargs.py index 03395baad3b59..e3b78930f219f 100644 --- a/tests/cpydiff/modules_struct_manyargs.py +++ b/tests/cpydiff/modules_struct_manyargs.py @@ -4,6 +4,7 @@ cause: Unknown workaround: Unknown """ + import struct try: diff --git a/tests/cpydiff/modules_struct_whitespace_in_format.py b/tests/cpydiff/modules_struct_whitespace_in_format.py index a882b38569a95..a7a1d2facdfff 100644 --- a/tests/cpydiff/modules_struct_whitespace_in_format.py +++ b/tests/cpydiff/modules_struct_whitespace_in_format.py @@ -4,6 +4,7 @@ cause: MicroPython is optimised for code size. workaround: Don't use spaces in format strings. """ + import struct try: diff --git a/tests/cpydiff/modules_sys_stdassign.py b/tests/cpydiff/modules_sys_stdassign.py index 7d086078a93d6..29afe1b1203c9 100644 --- a/tests/cpydiff/modules_sys_stdassign.py +++ b/tests/cpydiff/modules_sys_stdassign.py @@ -4,6 +4,7 @@ cause: They are stored in read-only memory. workaround: Unknown """ + import sys sys.stdin = None diff --git a/tests/cpydiff/syntax_assign_expr.py b/tests/cpydiff/syntax_assign_expr.py index d4ed063b39ae7..58f57ca1fbe02 100644 --- a/tests/cpydiff/syntax_assign_expr.py +++ b/tests/cpydiff/syntax_assign_expr.py @@ -4,4 +4,5 @@ cause: MicroPython is optimised for code size and doesn't check this case. workaround: Do not rely on this behaviour if writing CPython compatible code. """ + print([i := -1 for i in range(4)]) diff --git a/tests/cpydiff/syntax_spaces.py b/tests/cpydiff/syntax_spaces.py index c308240a78da5..03d25d56199d1 100644 --- a/tests/cpydiff/syntax_spaces.py +++ b/tests/cpydiff/syntax_spaces.py @@ -4,6 +4,7 @@ cause: Unknown workaround: Unknown """ + try: print(eval("1and 0")) except SyntaxError: diff --git a/tests/cpydiff/syntax_unicode_nameesc.py b/tests/cpydiff/syntax_unicode_nameesc.py index 21628c974d80f..838394b6ebd09 100644 --- a/tests/cpydiff/syntax_unicode_nameesc.py +++ b/tests/cpydiff/syntax_unicode_nameesc.py @@ -4,4 +4,5 @@ cause: Unknown workaround: Unknown """ + print("\N{LATIN SMALL LETTER A}") diff --git a/tests/cpydiff/types_bytearray_sliceassign.py b/tests/cpydiff/types_bytearray_sliceassign.py index e4068b04b988b..353f61988fa04 100644 --- a/tests/cpydiff/types_bytearray_sliceassign.py +++ b/tests/cpydiff/types_bytearray_sliceassign.py @@ -4,6 +4,7 @@ cause: Unknown workaround: Unknown """ + b = bytearray(4) b[0:1] = [1, 2] print(b) diff --git a/tests/cpydiff/types_bytes_format.py b/tests/cpydiff/types_bytes_format.py index ad0498771165c..1a15e572c8ade 100644 --- a/tests/cpydiff/types_bytes_format.py +++ b/tests/cpydiff/types_bytes_format.py @@ -4,4 +4,5 @@ cause: MicroPython strives to be a more regular implementation, so if both `str` and `bytes` support ``__mod__()`` (the % operator), it makes sense to support ``format()`` for both too. Support for ``__mod__`` can also be compiled out, which leaves only ``format()`` for bytes formatting. workaround: If you are interested in CPython compatibility, don't use ``.format()`` on bytes objects. """ + print(b"{}".format(1)) diff --git a/tests/cpydiff/types_bytes_keywords.py b/tests/cpydiff/types_bytes_keywords.py index ade83d0a709eb..a459c94b41ed9 100644 --- a/tests/cpydiff/types_bytes_keywords.py +++ b/tests/cpydiff/types_bytes_keywords.py @@ -4,4 +4,5 @@ cause: Unknown workaround: Pass the encoding as a positional parameter, e.g. ``print(bytes('abc', 'utf-8'))`` """ + print(bytes("abc", encoding="utf8")) diff --git a/tests/cpydiff/types_bytes_subscrstep.py b/tests/cpydiff/types_bytes_subscrstep.py index 51b94cb710f1c..c566cbdb630aa 100644 --- a/tests/cpydiff/types_bytes_subscrstep.py +++ b/tests/cpydiff/types_bytes_subscrstep.py @@ -4,4 +4,5 @@ cause: MicroPython is highly optimized for memory usage. workaround: Use explicit loop for this very rare operation. """ + print(b"123"[0:3:2]) diff --git a/tests/cpydiff/types_dict_keys_set.py b/tests/cpydiff/types_dict_keys_set.py index 3a0849a35564c..a5f127962e6c2 100644 --- a/tests/cpydiff/types_dict_keys_set.py +++ b/tests/cpydiff/types_dict_keys_set.py @@ -4,4 +4,5 @@ cause: Not implemented. workaround: Explicitly convert keys to a set before using set operations. """ + print({1: 2, 3: 4}.keys() & {1}) diff --git a/tests/cpydiff/types_exception_attrs.py b/tests/cpydiff/types_exception_attrs.py index ad72b62a61a5d..5fed45451d2d9 100644 --- a/tests/cpydiff/types_exception_attrs.py +++ b/tests/cpydiff/types_exception_attrs.py @@ -4,6 +4,7 @@ cause: MicroPython is optimised to reduce code size. workaround: Only use ``value`` on ``StopIteration`` exceptions, and ``errno`` on ``OSError`` exceptions. Do not use or rely on these attributes on other exceptions. """ + e = Exception(1) print(e.value) print(e.errno) diff --git a/tests/cpydiff/types_exception_chaining.py b/tests/cpydiff/types_exception_chaining.py index 836c4eb3e73f5..cff68d4124aec 100644 --- a/tests/cpydiff/types_exception_chaining.py +++ b/tests/cpydiff/types_exception_chaining.py @@ -4,6 +4,7 @@ cause: Unknown workaround: Unknown """ + try: raise TypeError except TypeError: diff --git a/tests/cpydiff/types_exception_instancevar.py b/tests/cpydiff/types_exception_instancevar.py index adc353361f015..fb67771baf33e 100644 --- a/tests/cpydiff/types_exception_instancevar.py +++ b/tests/cpydiff/types_exception_instancevar.py @@ -4,6 +4,7 @@ cause: MicroPython is highly optimized for memory usage. workaround: Use user-defined exception subclasses. """ + e = Exception() e.x = 0 print(e.x) diff --git a/tests/cpydiff/types_exception_loops.py b/tests/cpydiff/types_exception_loops.py index 8d326cbbbb000..549f1dd0a5bf0 100644 --- a/tests/cpydiff/types_exception_loops.py +++ b/tests/cpydiff/types_exception_loops.py @@ -4,6 +4,7 @@ cause: Condition checks are optimized to happen at the end of loop body, and that line number is reported. workaround: Unknown """ + l = ["-foo", "-bar"] i = 0 diff --git a/tests/cpydiff/types_float_rounding.py b/tests/cpydiff/types_float_rounding.py index a5b591966b0de..206e359ed9be7 100644 --- a/tests/cpydiff/types_float_rounding.py +++ b/tests/cpydiff/types_float_rounding.py @@ -4,4 +4,5 @@ cause: Unknown workaround: Unknown """ + print("%.1g" % -9.9) diff --git a/tests/cpydiff/types_list_delete_subscrstep.py b/tests/cpydiff/types_list_delete_subscrstep.py index 36e6f526b3d23..3c801d84949e0 100644 --- a/tests/cpydiff/types_list_delete_subscrstep.py +++ b/tests/cpydiff/types_list_delete_subscrstep.py @@ -4,6 +4,7 @@ cause: Unknown workaround: Use explicit loop for this rare operation. """ + l = [1, 2, 3, 4] del l[0:4:2] print(l) diff --git a/tests/cpydiff/types_list_store_noniter.py b/tests/cpydiff/types_list_store_noniter.py index 1d69b4a823188..581b1369801ac 100644 --- a/tests/cpydiff/types_list_store_noniter.py +++ b/tests/cpydiff/types_list_store_noniter.py @@ -4,6 +4,7 @@ cause: RHS is restricted to be a tuple or list workaround: Use ``list()`` on RHS to convert the iterable to a list """ + l = [10, 20] l[0:1] = range(4) print(l) diff --git a/tests/cpydiff/types_list_store_subscrstep.py b/tests/cpydiff/types_list_store_subscrstep.py index 1460372bb1754..97590267f4a64 100644 --- a/tests/cpydiff/types_list_store_subscrstep.py +++ b/tests/cpydiff/types_list_store_subscrstep.py @@ -4,6 +4,7 @@ cause: Unknown workaround: Use explicit loop for this rare operation. """ + l = [1, 2, 3, 4] l[0:4:2] = [5, 6] print(l) diff --git a/tests/cpydiff/types_memoryview_invalid.py b/tests/cpydiff/types_memoryview_invalid.py index c995a2e8991c6..f288c50ab57b7 100644 --- a/tests/cpydiff/types_memoryview_invalid.py +++ b/tests/cpydiff/types_memoryview_invalid.py @@ -6,6 +6,7 @@ In the worst case scenario, resizing an object which is the target of a memoryview can cause the memoryview(s) to reference invalid freed memory (a use-after-free bug) and corrupt the MicroPython runtime. workaround: Do not change the size of any ``bytearray`` or ``io.bytesIO`` object that has a ``memoryview`` assigned to it. """ + b = bytearray(b"abcdefg") m = memoryview(b) b.extend(b"hijklmnop") diff --git a/tests/cpydiff/types_str_endswith.py b/tests/cpydiff/types_str_endswith.py index f222ac1cd3ad3..890c7ba5ef47f 100644 --- a/tests/cpydiff/types_str_endswith.py +++ b/tests/cpydiff/types_str_endswith.py @@ -4,4 +4,5 @@ cause: Unknown workaround: Unknown """ + print("abc".endswith("c", 1)) diff --git a/tests/cpydiff/types_str_formatsubscr.py b/tests/cpydiff/types_str_formatsubscr.py index 1b83cfff6cd34..5c59a1d200a9b 100644 --- a/tests/cpydiff/types_str_formatsubscr.py +++ b/tests/cpydiff/types_str_formatsubscr.py @@ -4,4 +4,5 @@ cause: Unknown workaround: Unknown """ + print("{a[0]}".format(a=[1, 2])) diff --git a/tests/cpydiff/types_str_keywords.py b/tests/cpydiff/types_str_keywords.py index 77a4eac1c1db2..640dfa6c39132 100644 --- a/tests/cpydiff/types_str_keywords.py +++ b/tests/cpydiff/types_str_keywords.py @@ -4,4 +4,5 @@ cause: Unknown workaround: Input the encoding format directly. eg ``print(bytes('abc', 'utf-8'))`` """ + print(str(b"abc", encoding="utf8")) diff --git a/tests/cpydiff/types_str_ljust_rjust.py b/tests/cpydiff/types_str_ljust_rjust.py index 72e5105e025b2..7f91c137e905f 100644 --- a/tests/cpydiff/types_str_ljust_rjust.py +++ b/tests/cpydiff/types_str_ljust_rjust.py @@ -4,4 +4,5 @@ cause: MicroPython is highly optimized for memory usage. Easy workarounds available. workaround: Instead of ``s.ljust(10)`` use ``"%-10s" % s``, instead of ``s.rjust(10)`` use ``"% 10s" % s``. Alternatively, ``"{:<10}".format(s)`` or ``"{:>10}".format(s)``. """ + print("abc".ljust(10)) diff --git a/tests/cpydiff/types_str_rsplitnone.py b/tests/cpydiff/types_str_rsplitnone.py index 5d334fea2f846..ae524620a08a8 100644 --- a/tests/cpydiff/types_str_rsplitnone.py +++ b/tests/cpydiff/types_str_rsplitnone.py @@ -4,4 +4,5 @@ cause: Unknown workaround: Unknown """ + print("a a a".rsplit(None, 1)) diff --git a/tests/cpydiff/types_str_subscrstep.py b/tests/cpydiff/types_str_subscrstep.py index 2d3245e5582f5..cd57f18ccb345 100644 --- a/tests/cpydiff/types_str_subscrstep.py +++ b/tests/cpydiff/types_str_subscrstep.py @@ -4,4 +4,5 @@ cause: Unknown workaround: Unknown """ + print("abcdefghi"[0:9:2]) diff --git a/tests/cpydiff/types_tuple_subscrstep.py b/tests/cpydiff/types_tuple_subscrstep.py index f90f3c5bf4d16..60ba31af4cf74 100644 --- a/tests/cpydiff/types_tuple_subscrstep.py +++ b/tests/cpydiff/types_tuple_subscrstep.py @@ -4,4 +4,5 @@ cause: Unknown workaround: Unknown """ + print((1, 2, 3, 4)[0:4:2]) diff --git a/tests/ports/cc3200/pin.py b/tests/ports/cc3200/pin.py index 43537bba5ce07..d7b0fa976cd2a 100644 --- a/tests/ports/cc3200/pin.py +++ b/tests/ports/cc3200/pin.py @@ -3,6 +3,7 @@ pull up or pull down connected. GP12 and GP17 must be connected together """ + from machine import Pin import os diff --git a/tests/unicode/unicode.py b/tests/unicode/unicode.py index 072e049fde416..58d406e63eb2a 100644 --- a/tests/unicode/unicode.py +++ b/tests/unicode/unicode.py @@ -5,7 +5,7 @@ # Test all three forms of Unicode escape, and # all blocks of UTF-8 byte patterns -s = "a\xA9\xFF\u0123\u0800\uFFEE\U0001F44C" +s = "a\xa9\xff\u0123\u0800\uffee\U0001f44c" for i in range(-len(s), len(s)): print("s[%d]: %s %X" % (i, s[i], ord(s[i]))) print("s[:%d]: %d chars, '%s'" % (i, len(s[:i]), s[:i])) diff --git a/tools/gen-cpydiff.py b/tools/gen-cpydiff.py index 578b6c136f568..c0b5734389cdf 100644 --- a/tools/gen-cpydiff.py +++ b/tools/gen-cpydiff.py @@ -22,9 +22,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -""" gen-cpydiff generates documentation which outlines operations that differ between MicroPython - and CPython. This script is called by the docs Makefile for html and Latex and may be run - manually using the command make gen-cpydiff. """ +"""gen-cpydiff generates documentation which outlines operations that differ between MicroPython +and CPython. This script is called by the docs Makefile for html and Latex and may be run +manually using the command make gen-cpydiff.""" import os import subprocess diff --git a/tools/mpremote/mpremote/transport.py b/tools/mpremote/mpremote/transport.py index d845796953a29..c8fdc54a8891a 100644 --- a/tools/mpremote/mpremote/transport.py +++ b/tools/mpremote/mpremote/transport.py @@ -73,7 +73,7 @@ def fs_listdir(self, src=""): def repr_consumer(b): buf.extend(b.replace(b"\x04", b"")) - cmd = "import os\nfor f in os.ilistdir(%s):\n" " print(repr(f), end=',')" % ( + cmd = "import os\nfor f in os.ilistdir(%s):\n print(repr(f), end=',')" % ( ("'%s'" % src) if src else "" ) try: diff --git a/tools/pyboard.py b/tools/pyboard.py index 0cf5b3d46596a..d49365f617efd 100755 --- a/tools/pyboard.py +++ b/tools/pyboard.py @@ -530,7 +530,7 @@ def fs_listdir(self, src=""): def repr_consumer(b): buf.extend(b.replace(b"\x04", b"")) - cmd = "import os\nfor f in os.ilistdir(%s):\n" " print(repr(f), end=',')" % ( + cmd = "import os\nfor f in os.ilistdir(%s):\n print(repr(f), end=',')" % ( ("'%s'" % src) if src else "" ) try: diff --git a/tools/pydfu.py b/tools/pydfu.py index e1e4074a6baff..cd7354818cdea 100755 --- a/tools/pydfu.py +++ b/tools/pydfu.py @@ -344,8 +344,7 @@ def read_dfu_file(filename): # B uint8_t targets Number of targets dfu_prefix, data = consume("<5sBIB", data, "signature version size targets") print( - " %(signature)s v%(version)d, image size: %(size)d, " - "targets: %(targets)d" % dfu_prefix + " %(signature)s v%(version)d, image size: %(size)d, targets: %(targets)d" % dfu_prefix ) for target_idx in range(dfu_prefix["targets"]): # Decode the Image Prefix @@ -359,7 +358,7 @@ def read_dfu_file(filename): # I uint32_t size Size of image (without prefix) # I uint32_t elements Number of elements in the image img_prefix, data = consume( - "<6sBI255s2I", data, "signature altsetting named name " "size elements" + "<6sBI255s2I", data, "signature altsetting named name size elements" ) img_prefix["num"] = target_idx if img_prefix["named"]: From 8ce7a58be21eb77532b109594abcc4e47aa8d887 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 25 Feb 2025 10:43:01 +0100 Subject: [PATCH 095/110] tests: Four typos in tests directory. Found by codespell. Signed-off-by: Christian Clauss --- tests/README.md | 2 +- tests/extmod_hardware/machine_pwm.py | 2 +- tests/ports/esp32/esp32_nvs.py | 2 +- tests/run-tests.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/README.md b/tests/README.md index 891668c8b8737..21e14eee5e128 100644 --- a/tests/README.md +++ b/tests/README.md @@ -13,7 +13,7 @@ target platform and run the appropriate set of tests for that platform. For exa That will run tests on the `/dev/ttyACM0` serial port. You can also use shortcut device names like `a` for `/dev/ttyACM` and `c` for `COM`. Use -`./run-tests.py --help` to see all of the device possibilites, and other options. +`./run-tests.py --help` to see all of the device possibilities, and other options. There are three kinds of tests: diff --git a/tests/extmod_hardware/machine_pwm.py b/tests/extmod_hardware/machine_pwm.py index 014030be5c601..e27da32548659 100644 --- a/tests/extmod_hardware/machine_pwm.py +++ b/tests/extmod_hardware/machine_pwm.py @@ -1,4 +1,4 @@ -# Test machine.PWM, frequncy and duty cycle (using machine.time_pulse_us). +# Test machine.PWM, frequency and duty cycle (using machine.time_pulse_us). # # IMPORTANT: This test requires hardware connections: the PWM-output and pulse-input # pins must be wired together (see the variable `pwm_pulse_pins`). diff --git a/tests/ports/esp32/esp32_nvs.py b/tests/ports/esp32/esp32_nvs.py index fd8b152ca71d8..e6c308cde2f30 100644 --- a/tests/ports/esp32/esp32_nvs.py +++ b/tests/ports/esp32/esp32_nvs.py @@ -4,7 +4,7 @@ nvs = NVS("mp-test") -# test setting and gettin an integer kv +# test setting and getting an integer kv nvs.set_i32("key1", 1234) print(nvs.get_i32("key1")) nvs.set_i32("key2", -503) diff --git a/tests/run-tests.py b/tests/run-tests.py index b108b16395074..1af3dfc2973fc 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -848,7 +848,7 @@ def run_one_test(test_file): test_file_abspath = os.path.abspath(test_file).replace("\\", "/") if args.filters: - # Default verdict is the opposit of the first action + # Default verdict is the opposite of the first action verdict = "include" if args.filters[0][0] == "exclude" else "exclude" for action, pat in args.filters: if pat.search(test_file): From 731f7adf4f4541a36b32d59087148f78b39d7a27 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 18 Feb 2025 16:11:13 +1100 Subject: [PATCH 096/110] stm32/sdcard: Fix unchecked uint32_t overflow in SD card driver. Manifests as `readblocks(-1, buf)` failing. The ST HAL does a bounds check, but it checks `(block_num + num_blocks)` is within bounds, so if these values overflow then it allows the read which seems to hang some SD Cards (but not all). Fix by explicitly testing for overflow in our layer of the driver. This work was funded through GitHub Sponsors. Signed-off-by: Angus Gratton --- ports/stm32/sdcard.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/ports/stm32/sdcard.c b/ports/stm32/sdcard.c index a7d0629399651..eef6548ddf4ed 100644 --- a/ports/stm32/sdcard.c +++ b/ports/stm32/sdcard.c @@ -498,13 +498,27 @@ static HAL_StatusTypeDef sdcard_wait_finished(uint32_t timeout) { return HAL_OK; } -mp_uint_t sdcard_read_blocks(uint8_t *dest, uint32_t block_num, uint32_t num_blocks) { +static HAL_StatusTypeDef sdcard_common_checks(uint32_t block_num, uint32_t num_blocks) { // check that SD card is initialised if (!(pyb_sdmmc_flags & PYB_SDMMC_FLAG_ACTIVE)) { return HAL_ERROR; } - HAL_StatusTypeDef err = HAL_OK; + // check that adding block_num & num_blocks don't overflow + // (the ST HAL does a bounds check, but only after adding them...) + uint32_t end_block; + if (__builtin_add_overflow(block_num, num_blocks, &end_block)) { + return HAL_ERROR; + } + + return HAL_OK; +} + +mp_uint_t sdcard_read_blocks(uint8_t *dest, uint32_t block_num, uint32_t num_blocks) { + HAL_StatusTypeDef err = sdcard_common_checks(block_num, num_blocks); + if (err != HAL_OK) { + return err; + } // check that dest pointer is aligned on a 4-byte boundary uint8_t *orig_dest = NULL; @@ -595,13 +609,11 @@ mp_uint_t sdcard_read_blocks(uint8_t *dest, uint32_t block_num, uint32_t num_blo } mp_uint_t sdcard_write_blocks(const uint8_t *src, uint32_t block_num, uint32_t num_blocks) { - // check that SD card is initialised - if (!(pyb_sdmmc_flags & PYB_SDMMC_FLAG_ACTIVE)) { - return HAL_ERROR; + HAL_StatusTypeDef err = sdcard_common_checks(block_num, num_blocks); + if (err != HAL_OK) { + return err; } - HAL_StatusTypeDef err = HAL_OK; - // check that src pointer is aligned on a 4-byte boundary if (((uint32_t)src & 3) != 0) { // pointer is not aligned, so allocate a temporary block to do the write From 78728dc94abf9482d07efb3056d03ff6d41d7b1a Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 18 Feb 2025 16:43:58 +1100 Subject: [PATCH 097/110] stm32/sdcard: Drop the pyb.SDCard timeout from 60 to 30 seconds. 60 seconds is long enough that the USB serial connection drops out before it times out (at least on my computer). Also refactor out the timeout argument from sdcard_wait_finished, to try and save a little code size. This work was funded through GitHub Sponsors. Signed-off-by: Angus Gratton --- ports/stm32/sdcard.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/ports/stm32/sdcard.c b/ports/stm32/sdcard.c index eef6548ddf4ed..706d6315c44d4 100644 --- a/ports/stm32/sdcard.c +++ b/ports/stm32/sdcard.c @@ -158,6 +158,8 @@ static uint8_t pyb_sdmmc_flags; +#define TIMEOUT_MS 30000 + // TODO: I think that as an optimization, we can allocate these dynamically // if an sd card is detected. This will save approx 260 bytes of RAM // when no sdcard was being used. @@ -441,7 +443,7 @@ static void sdcard_reset_periph(void) { SDIO->ICR = SDMMC_STATIC_FLAGS; } -static HAL_StatusTypeDef sdcard_wait_finished(uint32_t timeout) { +static HAL_StatusTypeDef sdcard_wait_finished(void) { // Wait for HAL driver to be ready (eg for DMA to finish) uint32_t start = HAL_GetTick(); for (;;) { @@ -463,7 +465,7 @@ static HAL_StatusTypeDef sdcard_wait_finished(uint32_t timeout) { } __WFI(); enable_irq(irq_state); - if (HAL_GetTick() - start >= timeout) { + if (HAL_GetTick() - start >= TIMEOUT_MS) { return HAL_TIMEOUT; } } @@ -490,7 +492,7 @@ static HAL_StatusTypeDef sdcard_wait_finished(uint32_t timeout) { if (!(state == HAL_SD_CARD_SENDING || state == HAL_SD_CARD_RECEIVING || state == HAL_SD_CARD_PROGRAMMING)) { return HAL_ERROR; } - if (HAL_GetTick() - start >= timeout) { + if (HAL_GetTick() - start >= TIMEOUT_MS) { return HAL_TIMEOUT; } __WFI(); @@ -569,7 +571,7 @@ mp_uint_t sdcard_read_blocks(uint8_t *dest, uint32_t block_num, uint32_t num_blo err = HAL_SD_ReadBlocks_DMA(&sdmmc_handle.sd, dest, block_num, num_blocks); } if (err == HAL_OK) { - err = sdcard_wait_finished(60000); + err = sdcard_wait_finished(); } #if SDIO_USE_GPDMA @@ -588,14 +590,14 @@ mp_uint_t sdcard_read_blocks(uint8_t *dest, uint32_t block_num, uint32_t num_blo } else { #if MICROPY_HW_ENABLE_MMCARD if (pyb_sdmmc_flags & PYB_SDMMC_FLAG_MMC) { - err = HAL_MMC_ReadBlocks(&sdmmc_handle.mmc, dest, block_num, num_blocks, 60000); + err = HAL_MMC_ReadBlocks(&sdmmc_handle.mmc, dest, block_num, num_blocks, TIMEOUT_MS); } else #endif { - err = HAL_SD_ReadBlocks(&sdmmc_handle.sd, dest, block_num, num_blocks, 60000); + err = HAL_SD_ReadBlocks(&sdmmc_handle.sd, dest, block_num, num_blocks, TIMEOUT_MS); } if (err == HAL_OK) { - err = sdcard_wait_finished(60000); + err = sdcard_wait_finished(); } } @@ -662,7 +664,7 @@ mp_uint_t sdcard_write_blocks(const uint8_t *src, uint32_t block_num, uint32_t n err = HAL_SD_WriteBlocks_DMA(&sdmmc_handle.sd, (uint8_t *)src, block_num, num_blocks); } if (err == HAL_OK) { - err = sdcard_wait_finished(60000); + err = sdcard_wait_finished(); } #if SDIO_USE_GPDMA @@ -681,14 +683,14 @@ mp_uint_t sdcard_write_blocks(const uint8_t *src, uint32_t block_num, uint32_t n } else { #if MICROPY_HW_ENABLE_MMCARD if (pyb_sdmmc_flags & PYB_SDMMC_FLAG_MMC) { - err = HAL_MMC_WriteBlocks(&sdmmc_handle.mmc, (uint8_t *)src, block_num, num_blocks, 60000); + err = HAL_MMC_WriteBlocks(&sdmmc_handle.mmc, (uint8_t *)src, block_num, num_blocks, TIMEOUT_MS); } else #endif { - err = HAL_SD_WriteBlocks(&sdmmc_handle.sd, (uint8_t *)src, block_num, num_blocks, 60000); + err = HAL_SD_WriteBlocks(&sdmmc_handle.sd, (uint8_t *)src, block_num, num_blocks, TIMEOUT_MS); } if (err == HAL_OK) { - err = sdcard_wait_finished(60000); + err = sdcard_wait_finished(); } } From e3101ce1b3782955ea2d103922f0dc19bc0331f6 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 25 Feb 2025 15:14:08 +1100 Subject: [PATCH 098/110] qemu/boards/SABRELITE: Increase MicroPython heap to 160k. Signed-off-by: Damien George --- ports/qemu/boards/SABRELITE/mpconfigboard.mk | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ports/qemu/boards/SABRELITE/mpconfigboard.mk b/ports/qemu/boards/SABRELITE/mpconfigboard.mk index 80ea08f12cf76..e56dba712ff44 100644 --- a/ports/qemu/boards/SABRELITE/mpconfigboard.mk +++ b/ports/qemu/boards/SABRELITE/mpconfigboard.mk @@ -12,6 +12,10 @@ LDSCRIPT = mcu/arm/imx6.ld SRC_BOARD_O = shared/runtime/gchelper_generic.o +# Use a larger heap than the default so tests run with the native emitter have +# enough memory (because emitted ARM machine code is larger than Thumb2 code). +MICROPY_HEAP_SIZE ?= 163840 + # It's really armv7a but closest supported value is armv6. MPY_CROSS_FLAGS += -march=armv6 From 14ba32bb205fa1f7d6ac456879b08aadb4e1aaf7 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 24 Feb 2025 23:14:48 +1100 Subject: [PATCH 099/110] extmod/vfs_rom: Add bounds checking for all filesystem accesses. Testing with ROMFS shows that it is relatively easy to end up with a corrupt filesystem on the device -- eg due to the ROMFS deploy process stopping half way through -- which could lead to hard crashes. Notably, there can be boot loops trying to mount a corrupt filesystem, crashes when importing modules like `os` that first scan the filesystem for `os.py`, and crashing when deploying a new ROMFS in certain cases because the old one is removed while still mounted. The main problem is that `mp_decode_uint()` has an loop that keeps going as long as it reads 0xff byte values, which can happen in the case of erased and unwritten flash. This commit adds full bounds checking in the new `mp_decode_uint_checked()` function, and that makes all ROMFS filesystem accesses robust. Signed-off-by: Damien George --- extmod/vfs_rom.c | 114 +++++++++++++++++++++++++++++++--------- tests/extmod/vfs_rom.py | 73 +++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 24 deletions(-) diff --git a/extmod/vfs_rom.c b/extmod/vfs_rom.c index 99ddaba95ed79..ff3652d2ce3a2 100644 --- a/extmod/vfs_rom.c +++ b/extmod/vfs_rom.c @@ -91,6 +91,7 @@ #define ROMFS_RECORD_KIND_DATA_POINTER (3) #define ROMFS_RECORD_KIND_DIRECTORY (4) #define ROMFS_RECORD_KIND_FILE (5) +#define ROMFS_RECORD_KIND_FILESYSTEM (0x14a6b1) typedef mp_uint_t record_kind_t; @@ -101,34 +102,72 @@ struct _mp_obj_vfs_rom_t { const uint8_t *filesystem_end; }; -static record_kind_t extract_record(const uint8_t **fs, const uint8_t **fs_next) { - record_kind_t record_kind = mp_decode_uint(fs); - mp_uint_t record_len = mp_decode_uint(fs); +// Returns 0 for success, -1 for failure. +static int mp_decode_uint_checked(const uint8_t **ptr, const uint8_t *ptr_max, mp_uint_t *value_out) { + mp_uint_t unum = 0; + byte val; + const uint8_t *p = *ptr; + do { + if (p >= ptr_max) { + return -1; + } + val = *p++; + unum = (unum << 7) | (val & 0x7f); + } while ((val & 0x80) != 0); + *ptr = p; + *value_out = unum; + return 0; +} + +static record_kind_t extract_record(const uint8_t **fs, const uint8_t **fs_next, const uint8_t *fs_max) { + mp_uint_t record_kind; + if (mp_decode_uint_checked(fs, fs_max, &record_kind) != 0) { + return ROMFS_RECORD_KIND_UNUSED; + } + mp_uint_t record_len; + if (mp_decode_uint_checked(fs, fs_max, &record_len) != 0) { + return ROMFS_RECORD_KIND_UNUSED; + } *fs_next = *fs + record_len; return record_kind; } -static void extract_data(mp_obj_vfs_rom_t *self, const uint8_t *fs, const uint8_t *fs_top, size_t *size_out, const uint8_t **data_out) { - *size_out = 0; - *data_out = NULL; +// Returns 0 for success, a negative integer for failure. +static int extract_data(mp_obj_vfs_rom_t *self, const uint8_t *fs, const uint8_t *fs_top, size_t *size_out, const uint8_t **data_out) { while (fs < fs_top) { const uint8_t *fs_next; - record_kind_t record_kind = extract_record(&fs, &fs_next); - if (record_kind == ROMFS_RECORD_KIND_DATA_VERBATIM) { - // Verbatim data. - *size_out = fs_next - fs; - *data_out = fs; + record_kind_t record_kind = extract_record(&fs, &fs_next, fs_top); + if (record_kind == ROMFS_RECORD_KIND_UNUSED) { + // Corrupt filesystem. break; + } else if (record_kind == ROMFS_RECORD_KIND_DATA_VERBATIM) { + // Verbatim data. + if (size_out != NULL) { + *size_out = fs_next - fs; + *data_out = fs; + } + return 0; } else if (record_kind == ROMFS_RECORD_KIND_DATA_POINTER) { // Pointer to data. - *size_out = mp_decode_uint(&fs); - *data_out = self->filesystem + mp_decode_uint(&fs); - break; + mp_uint_t size; + if (mp_decode_uint_checked(&fs, fs_next, &size) != 0) { + break; + } + mp_uint_t offset; + if (mp_decode_uint_checked(&fs, fs_next, &offset) != 0) { + break; + } + if (size_out != NULL) { + *size_out = size; + *data_out = self->filesystem + offset; + } + return 0; } else { // Skip this record. fs = fs_next; } } + return -MP_EIO; } // Searches for `path` in the filesystem. @@ -144,10 +183,17 @@ mp_import_stat_t mp_vfs_rom_search_filesystem(mp_obj_vfs_rom_t *self, const char } while (path_len > 0 && fs < fs_top) { const uint8_t *fs_next; - record_kind_t record_kind = extract_record(&fs, &fs_next); - if (record_kind == ROMFS_RECORD_KIND_DIRECTORY || record_kind == ROMFS_RECORD_KIND_FILE) { + record_kind_t record_kind = extract_record(&fs, &fs_next, fs_top); + if (record_kind == ROMFS_RECORD_KIND_UNUSED) { + // Corrupt filesystem. + return MP_IMPORT_STAT_NO_EXIST; + } else if (record_kind == ROMFS_RECORD_KIND_DIRECTORY || record_kind == ROMFS_RECORD_KIND_FILE) { // A directory or file record. - mp_uint_t name_len = mp_decode_uint(&fs); + mp_uint_t name_len; + if (mp_decode_uint_checked(&fs, fs_next, &name_len) != 0) { + // Corrupt filesystem. + return MP_IMPORT_STAT_NO_EXIST; + } if ((name_len == path_len || (name_len < path_len && path[name_len] == '/')) && memcmp(path, fs, name_len) == 0) { @@ -167,8 +213,9 @@ mp_import_stat_t mp_vfs_rom_search_filesystem(mp_obj_vfs_rom_t *self, const char if (path_len != 0) { return MP_IMPORT_STAT_NO_EXIST; } - if (size_out != NULL) { - extract_data(self, fs, fs_top, size_out, data_out); + if (extract_data(self, fs, fs_top, size_out, data_out) != 0) { + // Corrupt filesystem. + return MP_IMPORT_STAT_NO_EXIST; } return MP_IMPORT_STAT_FILE; } @@ -214,7 +261,15 @@ static mp_obj_t vfs_rom_make_new(const mp_obj_type_t *type, size_t n_args, size_ } // The ROMFS is a record itself, so enter into it and compute its limit. - extract_record(&self->filesystem, &self->filesystem_end); + record_kind_t record_kind = extract_record(&self->filesystem, &self->filesystem_end, self->filesystem + bufinfo.len); + if (record_kind != ROMFS_RECORD_KIND_FILESYSTEM) { + mp_raise_OSError(MP_ENODEV); + } + + // Check the filesystem is within the limits of the input buffer. + if (self->filesystem_end > (const uint8_t *)bufinfo.buf + bufinfo.len) { + mp_raise_OSError(MP_ENODEV); + } return MP_OBJ_FROM_PTR(self); } @@ -259,13 +314,21 @@ static mp_obj_t vfs_rom_ilistdir_it_iternext(mp_obj_t self_in) { while (self->index < self->index_top) { const uint8_t *index_next; - record_kind_t record_kind = extract_record(&self->index, &index_next); + record_kind_t record_kind = extract_record(&self->index, &index_next, self->index_top); uint32_t type; mp_uint_t name_len; size_t data_len; - if (record_kind == ROMFS_RECORD_KIND_DIRECTORY || record_kind == ROMFS_RECORD_KIND_FILE) { + if (record_kind == ROMFS_RECORD_KIND_UNUSED) { + // Corrupt filesystem. + self->index = self->index_top; + break; + } else if (record_kind == ROMFS_RECORD_KIND_DIRECTORY || record_kind == ROMFS_RECORD_KIND_FILE) { // A directory or file record. - name_len = mp_decode_uint(&self->index); + if (mp_decode_uint_checked(&self->index, index_next, &name_len) != 0) { + // Corrupt filesystem. + self->index = self->index_top; + break; + } if (record_kind == ROMFS_RECORD_KIND_DIRECTORY) { // A directory. type = MP_S_IFDIR; @@ -274,7 +337,10 @@ static mp_obj_t vfs_rom_ilistdir_it_iternext(mp_obj_t self_in) { // A file. type = MP_S_IFREG; const uint8_t *data_value; - extract_data(self->vfs_rom, self->index + name_len, index_next, &data_len, &data_value); + if (extract_data(self->vfs_rom, self->index + name_len, index_next, &data_len, &data_value) != 0) { + // Corrupt filesystem. + break; + } } } else { // Skip this record. diff --git a/tests/extmod/vfs_rom.py b/tests/extmod/vfs_rom.py index f7958a93962d0..dc88481c028ac 100644 --- a/tests/extmod/vfs_rom.py +++ b/tests/extmod/vfs_rom.py @@ -223,6 +223,79 @@ def test_unknown_record(self): self.assertEqual(f.read(), b"contents") +class TestCorrupt(unittest.TestCase): + def test_corrupt_filesystem(self): + # Make the filesystem length bigger than the buffer. + romfs = bytearray(make_romfs(())) + romfs[3] = 0x01 + with self.assertRaises(OSError): + vfs.VfsRom(romfs) + + # Corrupt the filesystem length. + romfs = bytearray(make_romfs(())) + romfs[3] = 0xFF + with self.assertRaises(OSError): + vfs.VfsRom(romfs) + + # Corrupt the contents of the filesystem. + romfs = bytearray(make_romfs(())) + romfs[3] = 0x01 + romfs.extend(b"\xff\xff") + fs = vfs.VfsRom(romfs) + with self.assertRaises(OSError): + fs.stat("a") + self.assertEqual(list(fs.ilistdir("")), []) + + def test_corrupt_file_entry(self): + romfs = make_romfs((("file", b"data"),)) + + # Corrupt the length of filename. + romfs_corrupt = bytearray(romfs) + romfs_corrupt[7:] = b"\xff" * (len(romfs) - 7) + fs = vfs.VfsRom(romfs_corrupt) + with self.assertRaises(OSError): + fs.stat("file") + self.assertEqual(list(fs.ilistdir("")), []) + + # Erase the data record (change it to a padding record). + romfs_corrupt = bytearray(romfs) + romfs_corrupt[12] = VfsRomWriter.ROMFS_RECORD_KIND_PADDING + fs = vfs.VfsRom(romfs_corrupt) + with self.assertRaises(OSError): + fs.stat("file") + self.assertEqual(list(fs.ilistdir("")), []) + + # Corrupt the header of the data record. + romfs_corrupt = bytearray(romfs) + romfs_corrupt[12:] = b"\xff" * (len(romfs) - 12) + fs = vfs.VfsRom(romfs_corrupt) + with self.assertRaises(OSError): + fs.stat("file") + + # Corrupt the interior of the data record. + romfs_corrupt = bytearray(romfs) + romfs_corrupt[13:] = b"\xff" * (len(romfs) - 13) + fs = vfs.VfsRom(romfs_corrupt) + with self.assertRaises(OSError): + fs.stat("file") + + # Change the data record to an indirect pointer and corrupt the length. + romfs_corrupt = bytearray(romfs) + romfs_corrupt[12] = VfsRomWriter.ROMFS_RECORD_KIND_DATA_POINTER + romfs_corrupt[14:18] = b"\xff\xff\xff\xff" + fs = vfs.VfsRom(romfs_corrupt) + with self.assertRaises(OSError): + fs.stat("file") + + # Change the data record to an indirect pointer and corrupt the offset. + romfs_corrupt = bytearray(romfs) + romfs_corrupt[12] = VfsRomWriter.ROMFS_RECORD_KIND_DATA_POINTER + romfs_corrupt[14:18] = b"\x00\xff\xff\xff" + fs = vfs.VfsRom(romfs_corrupt) + with self.assertRaises(OSError): + fs.stat("file") + + class TestStandalone(TestBase): def test_constructor(self): self.assertIsInstance(vfs.VfsRom(self.romfs), vfs.VfsRom) From dc2c33b07f83ac4fcbcbcbb294e5726a95507490 Mon Sep 17 00:00:00 2001 From: Alessandro Gatti Date: Sat, 4 Jan 2025 12:51:18 +0100 Subject: [PATCH 100/110] py/emitinlinerv32: Fix compilation with ESP-IDF v5.2 and later. This commit fixes a compilation warning (turned error) about a potentially uninitialised variable being used. The warning can be ignored as the variable in question is always written to, but the code has been changed to silence that warning. Signed-off-by: Alessandro Gatti --- py/emitinlinerv32.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/py/emitinlinerv32.c b/py/emitinlinerv32.c index 179e289c67b05..ba171b4e6fae0 100644 --- a/py/emitinlinerv32.c +++ b/py/emitinlinerv32.c @@ -709,9 +709,11 @@ static bool handle_load_store_opcode_with_offset(emit_inline_asm_t *emit, qstr o return false; } - mp_uint_t rd; - mp_uint_t rs1; - parse_register_node(nodes[0], &rd, opcode_data->argument1_kind & C); + mp_uint_t rd = 0; + mp_uint_t rs1 = 0; + if (!parse_register_node(nodes[0], &rd, opcode_data->argument1_kind & C)) { + return false; + } if (!parse_register_node(nodes[1], &rs1, opcode_data->argument3_kind & C)) { return false; } From 8633abc08291d98035c65904dc4dab9983ddada3 Mon Sep 17 00:00:00 2001 From: Alessandro Gatti Date: Sat, 4 Jan 2025 12:54:44 +0100 Subject: [PATCH 101/110] py/emitinlinerv32: Reduce the footprint of compiled code. This commit introduces a few changes aimed at reducing the amount of space taken by the inline assembler once compiled: * The register string table uses 2 bytes for each qstr rather than the usual 4 * The opcode table uses 2 bytes for each qstr rather than the usual 4 * Opcode masks are not embedded in each opcode entry but looked up via an additional smaller table, reducing the number of bytes taken by an opcode's masks from 12 to 2 (with a fixed overhead of 24 bytes for the the masks themselves stored elsewhere) * Some error messages had a trailing period, now removed * Error messages have been parameterised when possible, and the overall text length is smaller. Signed-off-by: Alessandro Gatti --- py/emitinlinerv32.c | 332 ++++++++++++++++++++++++-------------------- 1 file changed, 181 insertions(+), 151 deletions(-) diff --git a/py/emitinlinerv32.c b/py/emitinlinerv32.c index ba171b4e6fae0..a9a81ddf16c4c 100644 --- a/py/emitinlinerv32.c +++ b/py/emitinlinerv32.c @@ -60,7 +60,7 @@ struct _emit_inline_asm_t { qstr *label_lookup; }; -static const qstr REGISTERS_QSTR_TABLE[] = { +static const qstr_short_t REGISTERS_QSTR_TABLE[] = { MP_QSTR_zero, MP_QSTR_ra, MP_QSTR_sp, MP_QSTR_gp, MP_QSTR_tp, MP_QSTR_t0, MP_QSTR_t1, MP_QSTR_t2, MP_QSTR_s0, MP_QSTR_s1, MP_QSTR_a0, MP_QSTR_a1, MP_QSTR_a2, MP_QSTR_a3, MP_QSTR_a4, MP_QSTR_a5, MP_QSTR_a6, MP_QSTR_a7, MP_QSTR_s2, MP_QSTR_s3, MP_QSTR_s4, MP_QSTR_s5, MP_QSTR_s6, MP_QSTR_s7, @@ -185,18 +185,21 @@ static bool emit_inline_rv32_label(emit_inline_asm_t *emit, mp_uint_t label_num, return true; } -#define CALL_RRR 0 // Register, Register, Register -#define CALL_RR 1 // Register, Register -#define CALL_RRI 2 // Register, Register, Immediate -#define CALL_RRL 3 // Register, Register, Label -#define CALL_RI 4 // Register, Immediate -#define CALL_L 5 // Label -#define CALL_R 6 // Register -#define CALL_RL 7 // Register, Label -#define CALL_N 8 // No arguments -#define CALL_I 9 // Immediate -#define CALL_RII 10 // Register, Immediate, Immediate -#define CALL_RIR 11 // Register, Immediate(Register) +typedef enum { + CALL_RRR, // Opcode Register, Register, Register + CALL_RR, // Opcode Register, Register + CALL_RRI, // Opcode Register, Register, Immediate + CALL_RRL, // Opcode Register, Register, Label + CALL_RI, // Opcode Register, Immediate + CALL_L, // Opcode Label + CALL_R, // Opcode Register + CALL_RL, // Opcode Register, Label + CALL_N, // Opcode + CALL_I, // Opcode Immediate + CALL_RII, // Opcode Register, Register, Immediate + CALL_RIR, // Opcode Register, Immediate(Register) + CALL_COUNT +} call_convention_t; #define N 0 // No argument #define R 1 // Register @@ -217,18 +220,21 @@ typedef void (*call_r_t)(asm_rv32_t *state, mp_uint_t rd); typedef void (*call_n_t)(asm_rv32_t *state); typedef struct _opcode_t { - qstr qstring; - void *emitter; + qstr_short_t qstring; + uint16_t argument1_mask : 4; + uint16_t argument2_mask : 4; + uint16_t argument3_mask : 4; + uint16_t arguments_count : 2; + // 2 bits available here uint32_t calling_convention : 4; uint32_t argument1_kind : 4; - uint32_t argument1_shift : 5; + uint32_t argument1_shift : 4; uint32_t argument2_kind : 4; - uint32_t argument2_shift : 5; + uint32_t argument2_shift : 4; uint32_t argument3_kind : 4; - uint32_t argument3_shift : 5; - uint32_t argument1_mask; - uint32_t argument2_mask; - uint32_t argument3_mask; + uint32_t argument3_shift : 4; + // 4 bits available here + void *emitter; } opcode_t; #define opcode_li asm_rv32_emit_optimised_load_immediate @@ -249,89 +255,129 @@ static void opcode_la(asm_rv32_t *state, mp_uint_t rd, mp_int_t displacement) { #define IZ (I | Z) #define IUZ (I | U | Z) +#define MASK_NOT_USED 0 + +enum { + MASK_FFFFFFFF, + MASK_00000FFF, + MASK_FFFFF000, + MASK_00001FFE, + MASK_0000001F, + MASK_FFFFFFFE, + MASK_0000003F, + MASK_0000FF00, + MASK_000003FC, + MASK_000001FE, + MASK_00000FFE, + MASK_FFFFFFFA, + MASK_0001F800, + MASK_0000007C, + MASK_000000FC, + MASK_001FFFFE, +}; + +static const uint32_t OPCODE_MASKS[] = { + [MASK_FFFFFFFF] = 0xFFFFFFFF, + [MASK_00000FFF] = 0x00000FFF, + [MASK_FFFFF000] = 0xFFFFF000, + [MASK_00001FFE] = 0x00001FFE, + [MASK_0000001F] = 0x0000001F, + [MASK_FFFFFFFE] = 0xFFFFFFFE, + [MASK_0000003F] = 0x0000003F, + [MASK_0000FF00] = 0x0000FF00, + [MASK_000003FC] = 0x000003FC, + [MASK_000001FE] = 0x000001FE, + [MASK_00000FFE] = 0x00000FFE, + [MASK_FFFFFFFA] = 0xFFFFFFFA, + [MASK_0001F800] = 0x0001F800, + [MASK_0000007C] = 0x0000007C, + [MASK_000000FC] = 0x000000FC, + [MASK_001FFFFE] = 0x001FFFFE, +}; + static const opcode_t OPCODES[] = { - { MP_QSTR_add, asm_rv32_opcode_add, CALL_RRR, R, 0, R, 0, R, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }, - { MP_QSTR_addi, asm_rv32_opcode_addi, CALL_RRI, R, 0, R, 0, I, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000FFF }, - { MP_QSTR_and_, asm_rv32_opcode_and, CALL_RRR, R, 0, R, 0, R, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }, - { MP_QSTR_andi, asm_rv32_opcode_andi, CALL_RRI, R, 0, R, 0, I, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000FFF }, - { MP_QSTR_auipc, asm_rv32_opcode_auipc, CALL_RI, R, 0, I, 12, N, 0, 0xFFFFFFFF, 0xFFFFF000, 0x00000000 }, - { MP_QSTR_beq, asm_rv32_opcode_beq, CALL_RRL, R, 0, R, 0, L, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x00001FFE }, - { MP_QSTR_bge, asm_rv32_opcode_bge, CALL_RRL, R, 0, R, 0, L, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x00001FFE }, - { MP_QSTR_bgeu, asm_rv32_opcode_bgeu, CALL_RRL, R, 0, R, 0, L, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x00001FFE }, - { MP_QSTR_blt, asm_rv32_opcode_blt, CALL_RRL, R, 0, R, 0, L, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x00001FFE }, - { MP_QSTR_bltu, asm_rv32_opcode_bltu, CALL_RRL, R, 0, R, 0, L, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x00001FFE }, - { MP_QSTR_bne, asm_rv32_opcode_bne, CALL_RRL, R, 0, R, 0, L, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x00001FFE }, - { MP_QSTR_csrrc, asm_rv32_opcode_csrrc, CALL_RRI, R, 0, R, 0, IU, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000FFF }, - { MP_QSTR_csrrs, asm_rv32_opcode_csrrs, CALL_RRI, R, 0, R, 0, IU, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000FFF }, - { MP_QSTR_csrrw, asm_rv32_opcode_csrrw, CALL_RRI, R, 0, R, 0, IU, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000FFF }, - { MP_QSTR_csrrci, asm_rv32_opcode_csrrci, CALL_RII, R, 0, IU, 0, IU, 0, 0xFFFFFFFF, 0x00000FFF, 0x0000001F }, - { MP_QSTR_csrrsi, asm_rv32_opcode_csrrsi, CALL_RII, R, 0, IU, 0, IU, 0, 0xFFFFFFFF, 0x00000FFF, 0x0000001F }, - { MP_QSTR_csrrwi, asm_rv32_opcode_csrrwi, CALL_RII, R, 0, IU, 0, IU, 0, 0xFFFFFFFF, 0x00000FFF, 0x0000001F }, - { MP_QSTR_c_add, asm_rv32_opcode_cadd, CALL_RR, R, 0, R, 0, N, 0, 0xFFFFFFFE, 0xFFFFFFFE, 0x00000000 }, - { MP_QSTR_c_addi, asm_rv32_opcode_caddi, CALL_RI, R, 0, IZ, 0, N, 0, 0xFFFFFFFE, 0x0000003F, 0x00000000 }, - { MP_QSTR_c_addi4spn, asm_rv32_opcode_caddi4spn, CALL_RI, R, 0, IUZ, 0, N, 0, 0x0000FF00, 0x000003FC, 0x00000000 }, - { MP_QSTR_c_and, asm_rv32_opcode_cand, CALL_RR, RC, 0, RC, 0, N, 0, 0x0000FF00, 0x0000FF00, 0x00000000 }, - { MP_QSTR_c_andi, asm_rv32_opcode_candi, CALL_RI, RC, 0, I, 0, N, 0, 0x0000FF00, 0x0000003F, 0x00000000 }, - { MP_QSTR_c_beqz, asm_rv32_opcode_cbeqz, CALL_RL, RC, 0, L, 0, N, 0, 0x0000FF00, 0x000001FE, 0x00000000 }, - { MP_QSTR_c_bnez, asm_rv32_opcode_cbnez, CALL_RL, RC, 0, L, 0, N, 0, 0x0000FF00, 0x000001FE, 0x00000000 }, - { MP_QSTR_c_ebreak, asm_rv32_opcode_cebreak, CALL_N, N, 0, N, 0, N, 0, 0x00000000, 0x00000000, 0x00000000 }, - { MP_QSTR_c_j, asm_rv32_opcode_cj, CALL_L, L, 0, N, 0, N, 0, 0x00000FFE, 0x00000000, 0x00000000 }, - { MP_QSTR_c_jal, asm_rv32_opcode_cjal, CALL_L, L, 0, N, 0, N, 0, 0x00000FFE, 0x00000000, 0x00000000 }, - { MP_QSTR_c_jalr, asm_rv32_opcode_cjalr, CALL_R, R, 0, N, 0, N, 0, 0xFFFFFFFE, 0x00000000, 0x00000000 }, - { MP_QSTR_c_jr, asm_rv32_opcode_cjr, CALL_R, R, 0, N, 0, N, 0, 0xFFFFFFFE, 0x00000000, 0x00000000 }, - { MP_QSTR_c_li, asm_rv32_opcode_cli, CALL_RI, R, 0, I, 0, N, 0, 0xFFFFFFFE, 0x0000003F, 0x00000000 }, - { MP_QSTR_c_lui, asm_rv32_opcode_clui, CALL_RI, R, 0, IUZ, 12, N, 0, 0xFFFFFFFA, 0x0001F800, 0x00000000 }, - { MP_QSTR_c_lw, asm_rv32_opcode_clw, CALL_RIR, RC, 0, I, 0, RC, 0, 0x0000FF00, 0x0000007C, 0x0000FF00 }, - { MP_QSTR_c_lwsp, asm_rv32_opcode_clwsp, CALL_RI, R, 0, I, 0, N, 0, 0xFFFFFFFE, 0x000000FC, 0x00000000 }, - { MP_QSTR_c_mv, asm_rv32_opcode_cmv, CALL_RR, R, 0, R, 0, N, 0, 0xFFFFFFFE, 0xFFFFFFFE, 0x00000000 }, - { MP_QSTR_c_nop, asm_rv32_opcode_cnop, CALL_N, N, 0, N, 0, N, 0, 0x00000000, 0x00000000, 0x00000000 }, - { MP_QSTR_c_or, asm_rv32_opcode_cor, CALL_RR, RC, 0, RC, 0, N, 0, 0x0000FF00, 0x0000FF00, 0x00000000 }, - { MP_QSTR_c_slli, asm_rv32_opcode_cslli, CALL_RI, R, 0, IU, 0, N, 0, 0xFFFFFFFE, 0x0000001F, 0x00000000 }, - { MP_QSTR_c_srai, asm_rv32_opcode_csrai, CALL_RI, RC, 0, IU, 0, N, 0, 0x0000FF00, 0x0000001F, 0x00000000 }, - { MP_QSTR_c_srli, asm_rv32_opcode_csrli, CALL_RI, RC, 0, IU, 0, N, 0, 0x0000FF00, 0x0000001F, 0x00000000 }, - { MP_QSTR_c_sub, asm_rv32_opcode_csub, CALL_RR, RC, 0, RC, 0, N, 0, 0x0000FF00, 0x0000FF00, 0x00000000 }, - { MP_QSTR_c_sw, asm_rv32_opcode_csw, CALL_RIR, RC, 0, I, 0, RC, 0, 0x0000FF00, 0x0000007C, 0x0000FF00 }, - { MP_QSTR_c_swsp, asm_rv32_opcode_cswsp, CALL_RI, R, 0, I, 0, N, 0, 0xFFFFFFFF, 0x000000FC, 0x00000000 }, - { MP_QSTR_c_xor, asm_rv32_opcode_cxor, CALL_RR, RC, 0, RC, 0, N, 0, 0x0000FF00, 0x0000FF00, 0x00000000 }, - { MP_QSTR_div, asm_rv32_opcode_div, CALL_RRR, R, 0, R, 0, R, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }, - { MP_QSTR_divu, asm_rv32_opcode_divu, CALL_RRR, R, 0, R, 0, R, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }, - { MP_QSTR_ebreak, asm_rv32_opcode_ebreak, CALL_N, N, 0, N, 0, N, 0, 0x00000000, 0x00000000, 0x00000000 }, - { MP_QSTR_ecall, asm_rv32_opcode_ecall, CALL_N, N, 0, N, 0, N, 0, 0x00000000, 0x00000000, 0x00000000 }, - { MP_QSTR_jal, asm_rv32_opcode_jal, CALL_RL, R, 0, L, 0, N, 0, 0xFFFFFFFF, 0x001FFFFE, 0x00000000 }, - { MP_QSTR_jalr, asm_rv32_opcode_jalr, CALL_RRI, R, 0, R, 0, I, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000FFF }, - { MP_QSTR_la, opcode_la, CALL_RL, R, 0, L, 0, N, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000 }, - { MP_QSTR_lb, asm_rv32_opcode_lb, CALL_RIR, R, 0, I, 0, R, 0, 0xFFFFFFFF, 0x00000FFF, 0xFFFFFFFF }, - { MP_QSTR_lbu, asm_rv32_opcode_lbu, CALL_RIR, R, 0, I, 0, R, 0, 0xFFFFFFFF, 0x00000FFF, 0xFFFFFFFF }, - { MP_QSTR_lh, asm_rv32_opcode_lh, CALL_RIR, R, 0, I, 0, R, 0, 0xFFFFFFFF, 0x00000FFF, 0xFFFFFFFF }, - { MP_QSTR_lhu, asm_rv32_opcode_lhu, CALL_RIR, R, 0, I, 0, R, 0, 0xFFFFFFFF, 0x00000FFF, 0xFFFFFFFF }, - { MP_QSTR_li, opcode_li, CALL_RI, R, 0, I, 0, N, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000 }, - { MP_QSTR_lui, asm_rv32_opcode_lui, CALL_RI, R, 0, I, 12, N, 0, 0xFFFFFFFF, 0xFFFFF000, 0x00000000 }, - { MP_QSTR_lw, asm_rv32_opcode_lw, CALL_RIR, R, 0, I, 0, R, 0, 0xFFFFFFFF, 0x00000FFF, 0xFFFFFFFF }, - { MP_QSTR_mv, asm_rv32_opcode_cmv, CALL_RR, R, 0, R, 0, N, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000 }, - { MP_QSTR_mul, asm_rv32_opcode_mul, CALL_RRR, R, 0, R, 0, R, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }, - { MP_QSTR_mulh, asm_rv32_opcode_mulh, CALL_RRR, R, 0, R, 0, R, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }, - { MP_QSTR_mulhsu, asm_rv32_opcode_mulhsu, CALL_RRR, R, 0, R, 0, R, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }, - { MP_QSTR_mulhu, asm_rv32_opcode_mulhu, CALL_RRR, R, 0, R, 0, R, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }, - { MP_QSTR_or_, asm_rv32_opcode_or, CALL_RRR, R, 0, R, 0, R, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }, - { MP_QSTR_ori, asm_rv32_opcode_ori, CALL_RRI, R, 0, R, 0, I, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000FFF }, - { MP_QSTR_rem, asm_rv32_opcode_rem, CALL_RRR, R, 0, R, 0, R, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }, - { MP_QSTR_remu, asm_rv32_opcode_remu, CALL_RRR, R, 0, R, 0, R, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }, - { MP_QSTR_sb, asm_rv32_opcode_sb, CALL_RIR, R, 0, I, 0, R, 0, 0xFFFFFFFF, 0x00000FFF, 0xFFFFFFFF }, - { MP_QSTR_sh, asm_rv32_opcode_sh, CALL_RIR, R, 0, I, 0, R, 0, 0xFFFFFFFF, 0x00000FFF, 0xFFFFFFFF }, - { MP_QSTR_sll, asm_rv32_opcode_sll, CALL_RRR, R, 0, R, 0, R, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }, - { MP_QSTR_slli, asm_rv32_opcode_slli, CALL_RRI, R, 0, R, 0, IU, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x0000001F }, - { MP_QSTR_slt, asm_rv32_opcode_slt, CALL_RRR, R, 0, R, 0, R, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }, - { MP_QSTR_slti, asm_rv32_opcode_slti, CALL_RRI, R, 0, R, 0, I, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000FFF }, - { MP_QSTR_sltiu, asm_rv32_opcode_sltiu, CALL_RRI, R, 0, R, 0, I, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000FFF }, - { MP_QSTR_sltu, asm_rv32_opcode_sltu, CALL_RRR, R, 0, R, 0, R, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }, - { MP_QSTR_sra, asm_rv32_opcode_sra, CALL_RRR, R, 0, R, 0, R, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }, - { MP_QSTR_srai, asm_rv32_opcode_srai, CALL_RRI, R, 0, R, 0, IU, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x0000001F }, - { MP_QSTR_srl, asm_rv32_opcode_srl, CALL_RRR, R, 0, R, 0, R, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }, - { MP_QSTR_srli, asm_rv32_opcode_srli, CALL_RRI, R, 0, R, 0, IU, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x0000001F }, - { MP_QSTR_sub, asm_rv32_opcode_sub, CALL_RRR, R, 0, R, 0, R, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }, - { MP_QSTR_sw, asm_rv32_opcode_sw, CALL_RIR, R, 0, I, 0, R, 0, 0xFFFFFFFF, 0x00000FFF, 0xFFFFFFFF }, - { MP_QSTR_xor, asm_rv32_opcode_xor, CALL_RRR, R, 0, R, 0, R, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }, - { MP_QSTR_xori, asm_rv32_opcode_xori, CALL_RRI, R, 0, R, 0, I, 0, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000FFF }, + { MP_QSTR_add, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_add }, + { MP_QSTR_addi, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, asm_rv32_opcode_addi }, + { MP_QSTR_and_, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_and }, + { MP_QSTR_andi, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, asm_rv32_opcode_andi }, + { MP_QSTR_auipc, MASK_FFFFFFFF, MASK_FFFFF000, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 12, N, 0, asm_rv32_opcode_auipc }, + { MP_QSTR_beq, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, asm_rv32_opcode_beq }, + { MP_QSTR_bge, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, asm_rv32_opcode_bge }, + { MP_QSTR_bgeu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, asm_rv32_opcode_bgeu }, + { MP_QSTR_blt, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, asm_rv32_opcode_blt }, + { MP_QSTR_bltu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, asm_rv32_opcode_bltu }, + { MP_QSTR_bne, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00001FFE, 3, CALL_RRL, R, 0, R, 0, L, 0, asm_rv32_opcode_bne }, + { MP_QSTR_csrrc, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, IU, 0, asm_rv32_opcode_csrrc }, + { MP_QSTR_csrrs, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, IU, 0, asm_rv32_opcode_csrrs }, + { MP_QSTR_csrrw, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, IU, 0, asm_rv32_opcode_csrrw }, + { MP_QSTR_csrrci, MASK_FFFFFFFF, MASK_00000FFF, MASK_0000001F, 3, CALL_RII, R, 0, IU, 0, IU, 0, asm_rv32_opcode_csrrci }, + { MP_QSTR_csrrsi, MASK_FFFFFFFF, MASK_00000FFF, MASK_0000001F, 3, CALL_RII, R, 0, IU, 0, IU, 0, asm_rv32_opcode_csrrsi }, + { MP_QSTR_csrrwi, MASK_FFFFFFFF, MASK_00000FFF, MASK_0000001F, 3, CALL_RII, R, 0, IU, 0, IU, 0, asm_rv32_opcode_csrrwi }, + { MP_QSTR_c_add, MASK_FFFFFFFE, MASK_FFFFFFFE, MASK_NOT_USED, 2, CALL_RR, R, 0, R, 0, N, 0, asm_rv32_opcode_cadd }, + { MP_QSTR_c_addi, MASK_FFFFFFFE, MASK_0000003F, MASK_NOT_USED, 2, CALL_RI, R, 0, IZ, 0, N, 0, asm_rv32_opcode_caddi }, + { MP_QSTR_c_addi4spn, MASK_0000FF00, MASK_000003FC, MASK_NOT_USED, 2, CALL_RI, R, 0, IUZ, 0, N, 0, asm_rv32_opcode_caddi4spn }, + { MP_QSTR_c_and, MASK_0000FF00, MASK_0000FF00, MASK_NOT_USED, 2, CALL_RR, RC, 0, RC, 0, N, 0, asm_rv32_opcode_cand }, + { MP_QSTR_c_andi, MASK_0000FF00, MASK_0000003F, MASK_NOT_USED, 2, CALL_RI, RC, 0, I, 0, N, 0, asm_rv32_opcode_candi }, + { MP_QSTR_c_beqz, MASK_0000FF00, MASK_000001FE, MASK_NOT_USED, 2, CALL_RL, RC, 0, L, 0, N, 0, asm_rv32_opcode_cbeqz }, + { MP_QSTR_c_bnez, MASK_0000FF00, MASK_000001FE, MASK_NOT_USED, 2, CALL_RL, RC, 0, L, 0, N, 0, asm_rv32_opcode_cbnez }, + { MP_QSTR_c_ebreak, MASK_NOT_USED, MASK_NOT_USED, MASK_NOT_USED, 0, CALL_N, N, 0, N, 0, N, 0, asm_rv32_opcode_cebreak }, + { MP_QSTR_c_j, MASK_00000FFE, MASK_NOT_USED, MASK_NOT_USED, 1, CALL_L, L, 0, N, 0, N, 0, asm_rv32_opcode_cj }, + { MP_QSTR_c_jal, MASK_00000FFE, MASK_NOT_USED, MASK_NOT_USED, 1, CALL_L, L, 0, N, 0, N, 0, asm_rv32_opcode_cjal }, + { MP_QSTR_c_jalr, MASK_FFFFFFFE, MASK_NOT_USED, MASK_NOT_USED, 1, CALL_R, R, 0, N, 0, N, 0, asm_rv32_opcode_cjalr }, + { MP_QSTR_c_jr, MASK_FFFFFFFE, MASK_NOT_USED, MASK_NOT_USED, 1, CALL_R, R, 0, N, 0, N, 0, asm_rv32_opcode_cjr }, + { MP_QSTR_c_li, MASK_FFFFFFFE, MASK_0000003F, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 0, N, 0, asm_rv32_opcode_cli }, + { MP_QSTR_c_lui, MASK_FFFFFFFA, MASK_0001F800, MASK_NOT_USED, 2, CALL_RI, R, 0, IUZ, 12, N, 0, asm_rv32_opcode_clui }, + { MP_QSTR_c_lw, MASK_0000FF00, MASK_0000007C, MASK_0000FF00, 3, CALL_RIR, RC, 0, I, 0, RC, 0, asm_rv32_opcode_clw }, + { MP_QSTR_c_lwsp, MASK_FFFFFFFE, MASK_000000FC, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 0, N, 0, asm_rv32_opcode_clwsp }, + { MP_QSTR_c_mv, MASK_FFFFFFFE, MASK_FFFFFFFE, MASK_NOT_USED, 2, CALL_RR, R, 0, R, 0, N, 0, asm_rv32_opcode_cmv }, + { MP_QSTR_c_nop, MASK_NOT_USED, MASK_NOT_USED, MASK_NOT_USED, 0, CALL_N, N, 0, N, 0, N, 0, asm_rv32_opcode_cnop }, + { MP_QSTR_c_or, MASK_0000FF00, MASK_0000FF00, MASK_NOT_USED, 2, CALL_RR, RC, 0, RC, 0, N, 0, asm_rv32_opcode_cor }, + { MP_QSTR_c_slli, MASK_FFFFFFFE, MASK_0000001F, MASK_NOT_USED, 2, CALL_RI, R, 0, IU, 0, N, 0, asm_rv32_opcode_cslli }, + { MP_QSTR_c_srai, MASK_0000FF00, MASK_0000001F, MASK_NOT_USED, 2, CALL_RI, RC, 0, IU, 0, N, 0, asm_rv32_opcode_csrai }, + { MP_QSTR_c_srli, MASK_0000FF00, MASK_0000001F, MASK_NOT_USED, 2, CALL_RI, RC, 0, IU, 0, N, 0, asm_rv32_opcode_csrli }, + { MP_QSTR_c_sub, MASK_0000FF00, MASK_0000FF00, MASK_NOT_USED, 2, CALL_RR, RC, 0, RC, 0, N, 0, asm_rv32_opcode_csub }, + { MP_QSTR_c_sw, MASK_0000FF00, MASK_0000007C, MASK_0000FF00, 3, CALL_RIR, RC, 0, I, 0, RC, 0, asm_rv32_opcode_csw }, + { MP_QSTR_c_swsp, MASK_FFFFFFFF, MASK_000000FC, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 0, N, 0, asm_rv32_opcode_cswsp }, + { MP_QSTR_c_xor, MASK_0000FF00, MASK_0000FF00, MASK_NOT_USED, 2, CALL_RR, RC, 0, RC, 0, N, 0, asm_rv32_opcode_cxor }, + { MP_QSTR_div, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_div }, + { MP_QSTR_divu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_divu }, + { MP_QSTR_ebreak, MASK_NOT_USED, MASK_NOT_USED, MASK_NOT_USED, 0, CALL_N, N, 0, N, 0, N, 0, asm_rv32_opcode_ebreak }, + { MP_QSTR_ecall, MASK_NOT_USED, MASK_NOT_USED, MASK_NOT_USED, 0, CALL_N, N, 0, N, 0, N, 0, asm_rv32_opcode_ecall }, + { MP_QSTR_jal, MASK_FFFFFFFF, MASK_001FFFFE, MASK_NOT_USED, 2, CALL_RL, R, 0, L, 0, N, 0, asm_rv32_opcode_jal }, + { MP_QSTR_jalr, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, asm_rv32_opcode_jalr }, + { MP_QSTR_la, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_NOT_USED, 2, CALL_RL, R, 0, L, 0, N, 0, opcode_la }, + { MP_QSTR_lb, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_lb }, + { MP_QSTR_lbu, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_lbu }, + { MP_QSTR_lh, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_lh }, + { MP_QSTR_lhu, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_lhu }, + { MP_QSTR_li, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 0, N, 0, opcode_li }, + { MP_QSTR_lui, MASK_FFFFFFFF, MASK_FFFFF000, MASK_NOT_USED, 2, CALL_RI, R, 0, I, 12, N, 0, asm_rv32_opcode_lui }, + { MP_QSTR_lw, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_lw }, + { MP_QSTR_mv, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_NOT_USED, 2, CALL_RR, R, 0, R, 0, N, 0, asm_rv32_opcode_cmv }, + { MP_QSTR_mul, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_mul }, + { MP_QSTR_mulh, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_mulh }, + { MP_QSTR_mulhsu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_mulhsu }, + { MP_QSTR_mulhu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_mulhu }, + { MP_QSTR_or_, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_or }, + { MP_QSTR_ori, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, asm_rv32_opcode_ori }, + { MP_QSTR_rem, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_rem }, + { MP_QSTR_remu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_remu }, + { MP_QSTR_sb, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_sb }, + { MP_QSTR_sh, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_sh }, + { MP_QSTR_sll, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_sll }, + { MP_QSTR_slli, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_0000001F, 3, CALL_RRI, R, 0, R, 0, IU, 0, asm_rv32_opcode_slli }, + { MP_QSTR_slt, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_slt }, + { MP_QSTR_slti, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, asm_rv32_opcode_slti }, + { MP_QSTR_sltiu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, asm_rv32_opcode_sltiu }, + { MP_QSTR_sltu, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_sltu }, + { MP_QSTR_sra, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_sra }, + { MP_QSTR_srai, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_0000001F, 3, CALL_RRI, R, 0, R, 0, IU, 0, asm_rv32_opcode_srai }, + { MP_QSTR_srl, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_srl }, + { MP_QSTR_srli, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_0000001F, 3, CALL_RRI, R, 0, R, 0, IU, 0, asm_rv32_opcode_srli }, + { MP_QSTR_sub, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_sub }, + { MP_QSTR_sw, MASK_FFFFFFFF, MASK_00000FFF, MASK_FFFFFFFF, 3, CALL_RIR, R, 0, I, 0, R, 0, asm_rv32_opcode_sw }, + { MP_QSTR_xor, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_FFFFFFFF, 3, CALL_RRR, R, 0, R, 0, R, 0, asm_rv32_opcode_xor }, + { MP_QSTR_xori, MASK_FFFFFFFF, MASK_FFFFFFFF, MASK_00000FFF, 3, CALL_RRI, R, 0, R, 0, I, 0, asm_rv32_opcode_xori }, }; #undef RC @@ -384,6 +430,10 @@ static bool validate_integer(mp_uint_t value, mp_uint_t mask, mp_uint_t flags) { return true; } +#define ET_WRONG_ARGUMENT_KIND MP_ERROR_TEXT("opcode '%q' argument %d: expecting %q") +#define ET_WRONG_ARGUMENTS_COUNT MP_ERROR_TEXT("opcode '%q': expecting %d arguments") +#define ET_OUT_OF_RANGE MP_ERROR_TEXT("opcode '%q' argument %d: out of range") + static bool validate_argument(emit_inline_asm_t *emit, qstr opcode_qstr, const opcode_t *opcode, mp_parse_node_t node, mp_uint_t node_index) { assert((node_index < 3) && "Invalid argument node number."); @@ -396,19 +446,19 @@ static bool validate_argument(emit_inline_asm_t *emit, qstr opcode_qstr, case 0: kind = opcode->argument1_kind; shift = opcode->argument1_shift; - mask = opcode->argument1_mask; + mask = OPCODE_MASKS[opcode->argument1_mask]; break; case 1: kind = opcode->argument2_kind; shift = opcode->argument2_shift; - mask = opcode->argument2_mask; + mask = OPCODE_MASKS[opcode->argument2_mask]; break; case 2: kind = opcode->argument3_kind; shift = opcode->argument3_shift; - mask = opcode->argument3_mask; + mask = OPCODE_MASKS[opcode->argument3_mask]; break; default: @@ -417,6 +467,7 @@ static bool validate_argument(emit_inline_asm_t *emit, qstr opcode_qstr, switch (kind & 0x03) { case N: + assert(mask == OPCODE_MASKS[MASK_NOT_USED] && "Invalid mask index for missing operand."); return true; case R: { @@ -424,15 +475,14 @@ static bool validate_argument(emit_inline_asm_t *emit, qstr opcode_qstr, if (!parse_register_node(node, ®ister_index, false)) { emit_inline_rv32_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, - MP_ERROR_TEXT("opcode '%q' argument %d must be a register"), - opcode_qstr, node_index + 1)); + ET_WRONG_ARGUMENT_KIND, opcode_qstr, node_index + 1, MP_QSTR_register)); return false; } if ((mask & (1U << register_index)) == 0) { emit_inline_rv32_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, - MP_ERROR_TEXT("opcode '%q' argument %d is an invalid register"), + MP_ERROR_TEXT("opcode '%q' argument %d: unknown register"), opcode_qstr, node_index + 1)); return false; } @@ -446,8 +496,7 @@ static bool validate_argument(emit_inline_asm_t *emit, qstr opcode_qstr, if (!mp_parse_node_get_int_maybe(node, &object)) { emit_inline_rv32_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, - MP_ERROR_TEXT("opcode '%q' argument %d must be an integer"), - opcode_qstr, node_index + 1)); + ET_WRONG_ARGUMENT_KIND, opcode_qstr, node_index + 1, MP_QSTR_integer)); return false; } @@ -474,8 +523,7 @@ static bool validate_argument(emit_inline_asm_t *emit, qstr opcode_qstr, if (!MP_PARSE_NODE_IS_ID(node)) { emit_inline_rv32_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, - MP_ERROR_TEXT("opcode '%q' argument %d must be a label"), - opcode_qstr, node_index + 1)); + ET_WRONG_ARGUMENT_KIND, opcode_qstr, node_index + 1, MP_QSTR_label)); return false; } @@ -484,7 +532,7 @@ static bool validate_argument(emit_inline_asm_t *emit, qstr opcode_qstr, if (label_index >= emit->max_num_labels && emit->pass == MP_PASS_EMIT) { emit_inline_rv32_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, - MP_ERROR_TEXT("opcode '%q' argument %d label '%q' is undefined"), + MP_ERROR_TEXT("opcode '%q' argument %d: undefined label '%q'"), opcode_qstr, node_index + 1, qstring)); return false; } @@ -512,15 +560,13 @@ static bool validate_argument(emit_inline_asm_t *emit, qstr opcode_qstr, out_of_range: emit_inline_rv32_error_exc(emit, - mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, - MP_ERROR_TEXT("opcode '%q' argument %d is out of range"), - opcode_qstr, node_index + 1)); + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, ET_OUT_OF_RANGE, opcode_qstr, node_index + 1)); return false; zero_immediate: emit_inline_rv32_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, - MP_ERROR_TEXT("opcode '%q' argument %d must not be zero"), + MP_ERROR_TEXT("opcode '%q' argument %d: must not be zero"), opcode_qstr, node_index + 1)); return false; } @@ -556,18 +602,14 @@ static bool parse_register_offset_node(emit_inline_asm_t *emit, qstr opcode_qstr mp_obj_t object; if (!mp_parse_node_get_int_maybe(node_struct->nodes[0], &object)) { emit_inline_rv32_error_exc(emit, - mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, - MP_ERROR_TEXT("opcode '%q' argument %d must be an integer"), - opcode_qstr, 2)); + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, ET_WRONG_ARGUMENT_KIND, opcode_qstr, 2, MP_QSTR_integer)); return false; } mp_uint_t value = mp_obj_get_int_truncated(object); value = (~value + 1) & (mp_uint_t)-1; - if (!validate_integer(value << opcode_data->argument2_shift, opcode_data->argument2_mask, opcode_data->argument2_kind)) { + if (!validate_integer(value << opcode_data->argument2_shift, OPCODE_MASKS[opcode_data->argument2_mask], opcode_data->argument2_kind)) { emit_inline_rv32_error_exc(emit, - mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, - MP_ERROR_TEXT("opcode '%q' argument %d is out of range"), - opcode_qstr, 2)); + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, ET_OUT_OF_RANGE, opcode_qstr, 2)); return false; } } else { @@ -587,8 +629,7 @@ static bool parse_register_offset_node(emit_inline_asm_t *emit, qstr opcode_qstr invalid_structure: emit_inline_rv32_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, - MP_ERROR_TEXT("opcode '%q' argument %d must be an integer offset to a register"), - opcode_qstr, node_index + 1)); + ET_WRONG_ARGUMENT_KIND, opcode_qstr, node_index + 1, MP_QSTR_offset)); return false; } @@ -724,11 +765,9 @@ static bool handle_load_store_opcode_with_offset(emit_inline_asm_t *emit, qstr o if (negative) { immediate = (~immediate + 1) & (mp_uint_t)-1; } - if (!is_in_signed_mask(opcode_data->argument2_mask, immediate)) { + if (!is_in_signed_mask(OPCODE_MASKS[opcode_data->argument2_mask], immediate)) { emit_inline_rv32_error_exc(emit, - mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, - MP_ERROR_TEXT("opcode '%q' argument %d is out of range"), - opcode, 2)); + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, ET_OUT_OF_RANGE, opcode, 2)); return false; } @@ -748,49 +787,40 @@ static void emit_inline_rv32_opcode(emit_inline_asm_t *emit, qstr opcode, mp_uin if (!opcode_data) { emit_inline_rv32_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, - MP_ERROR_TEXT("unsupported or unknown RV32 instruction '%q'."), opcode)); + MP_ERROR_TEXT("unknown RV32 instruction '%q'"), opcode)); return; } - size_t required_arguments = 0; - if (opcode_data->argument1_kind != N) { - required_arguments++; - } - if (opcode_data->argument2_kind != N) { - required_arguments++; - } - if (opcode_data->argument3_kind != N) { - required_arguments++; - } - + assert((opcode_data->argument1_mask < MP_ARRAY_SIZE(OPCODE_MASKS)) && "Argument #1 opcode mask index out of bounds."); + assert((opcode_data->argument2_mask < MP_ARRAY_SIZE(OPCODE_MASKS)) && "Argument #2 opcode mask index out of bounds."); + assert((opcode_data->argument3_mask < MP_ARRAY_SIZE(OPCODE_MASKS)) && "Argument #3 opcode mask index out of bounds."); + assert((opcode_data->calling_convention < CALL_COUNT) && "Calling convention index out of bounds."); if (opcode_data->calling_convention != CALL_RIR) { - if (required_arguments != arguments_count) { + if (opcode_data->arguments_count != arguments_count) { emit_inline_rv32_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, - MP_ERROR_TEXT("RV32 instruction '%q' requires %d arguments."), opcode, required_arguments)); + ET_WRONG_ARGUMENTS_COUNT, opcode, opcode_data->arguments_count)); return; } - if (required_arguments >= 1 && !validate_argument(emit, opcode, opcode_data, argument_nodes[0], 0)) { + if (opcode_data->arguments_count >= 1 && !validate_argument(emit, opcode, opcode_data, argument_nodes[0], 0)) { return; } - if (required_arguments >= 2 && !validate_argument(emit, opcode, opcode_data, argument_nodes[1], 1)) { + if (opcode_data->arguments_count >= 2 && !validate_argument(emit, opcode, opcode_data, argument_nodes[1], 1)) { return; } - if (required_arguments >= 3 && !validate_argument(emit, opcode, opcode_data, argument_nodes[2], 2)) { + if (opcode_data->arguments_count >= 3 && !validate_argument(emit, opcode, opcode_data, argument_nodes[2], 2)) { return; } handle_opcode(emit, opcode, opcode_data, argument_nodes); return; } - assert(required_arguments == 3 && "Invalid arguments count for calling convention."); assert((opcode_data->argument2_kind & U) == 0 && "Offset must not be unsigned."); assert((opcode_data->argument2_kind & Z) == 0 && "Offset can be zero."); if (arguments_count != 2) { emit_inline_rv32_error_exc(emit, - mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, - MP_ERROR_TEXT("RV32 instruction '%q' requires %d arguments."), opcode, 2)); + mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, ET_WRONG_ARGUMENTS_COUNT, opcode, 2)); return; } From 50fab08e6b861adf65905d6adacd74201c87ddb9 Mon Sep 17 00:00:00 2001 From: Alessandro Gatti Date: Tue, 28 Jan 2025 14:58:29 +0100 Subject: [PATCH 102/110] py/emitinlinextensa: Simplify register name lookup. This commit changes the Xtensa inline assembly parser to use a slightly simpler (and probably a tiny bit more efficient) way to look up register names when decoding instruction parameters. Signed-off-by: Alessandro Gatti --- py/emitinlinextensa.c | 49 +++++++++---------------------------------- 1 file changed, 10 insertions(+), 39 deletions(-) diff --git a/py/emitinlinextensa.c b/py/emitinlinextensa.c index 57056d597aab7..fed259cfc6b20 100644 --- a/py/emitinlinextensa.c +++ b/py/emitinlinextensa.c @@ -115,50 +115,21 @@ static bool emit_inline_xtensa_label(emit_inline_asm_t *emit, mp_uint_t label_nu return true; } -typedef struct _reg_name_t { byte reg; - byte name[3]; -} reg_name_t; -static const reg_name_t reg_name_table[] = { - {0, "a0\0"}, - {1, "a1\0"}, - {2, "a2\0"}, - {3, "a3\0"}, - {4, "a4\0"}, - {5, "a5\0"}, - {6, "a6\0"}, - {7, "a7\0"}, - {8, "a8\0"}, - {9, "a9\0"}, - {10, "a10"}, - {11, "a11"}, - {12, "a12"}, - {13, "a13"}, - {14, "a14"}, - {15, "a15"}, +static const qstr_short_t REGISTERS[16] = { + MP_QSTR_a0, MP_QSTR_a1, MP_QSTR_a2, MP_QSTR_a3, MP_QSTR_a4, MP_QSTR_a5, MP_QSTR_a6, MP_QSTR_a7, + MP_QSTR_a8, MP_QSTR_a9, MP_QSTR_a10, MP_QSTR_a11, MP_QSTR_a12, MP_QSTR_a13, MP_QSTR_a14, MP_QSTR_a15 }; -// return empty string in case of error, so we can attempt to parse the string -// without a special check if it was in fact a string -static const char *get_arg_str(mp_parse_node_t pn) { - if (MP_PARSE_NODE_IS_ID(pn)) { - qstr qst = MP_PARSE_NODE_LEAF_ARG(pn); - return qstr_str(qst); - } else { - return ""; - } -} - static mp_uint_t get_arg_reg(emit_inline_asm_t *emit, const char *op, mp_parse_node_t pn) { - const char *reg_str = get_arg_str(pn); - for (mp_uint_t i = 0; i < MP_ARRAY_SIZE(reg_name_table); i++) { - const reg_name_t *r = ®_name_table[i]; - if (reg_str[0] == r->name[0] - && reg_str[1] == r->name[1] - && reg_str[2] == r->name[2] - && (reg_str[2] == '\0' || reg_str[3] == '\0')) { - return r->reg; + if (MP_PARSE_NODE_IS_ID(pn)) { + qstr node_qstr = MP_PARSE_NODE_LEAF_ARG(pn); + for (size_t i = 0; i < MP_ARRAY_SIZE(REGISTERS); i++) { + if (node_qstr == REGISTERS[i]) { + return i; + } } } + emit_inline_xtensa_error_exc(emit, mp_obj_new_exception_msg_varg(&mp_type_SyntaxError, MP_ERROR_TEXT("'%s' expects a register"), op)); From 5fdd249c55d44ed0513e931c9b7364dd8b3454d9 Mon Sep 17 00:00:00 2001 From: Yoctopuce dev Date: Thu, 30 Jan 2025 10:23:06 +0100 Subject: [PATCH 103/110] py/parsenum: Reduce code footprint of mp_parse_num_float. The mantissa parsing code uses a floating point variable to accumulate digits. Using an `mp_float_uint_t` variable instead and casting to `mp_float_t` at the very end reduces code size. In some cases, it also improves the rounding behaviour as extra digits are taken into account by the int-to-float conversion code. An extra test case handles the special case where mantissa overflow occurs while processing deferred trailing zeros. Signed-off-by: Yoctopuce dev --- py/parsenum.c | 25 ++++++++++++++----------- tests/float/float_parse.py | 3 +++ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/py/parsenum.c b/py/parsenum.c index 27d6641198730..3281eb4b8511a 100644 --- a/py/parsenum.c +++ b/py/parsenum.c @@ -179,39 +179,40 @@ typedef enum { } parse_dec_in_t; #if MICROPY_PY_BUILTINS_FLOAT -// DEC_VAL_MAX only needs to be rough and is used to retain precision while not overflowing +// MANTISSA_MAX is used to retain precision while not overflowing mantissa // SMALL_NORMAL_VAL is the smallest power of 10 that is still a normal float // EXACT_POWER_OF_10 is the largest value of x so that 10^x can be stored exactly in a float // Note: EXACT_POWER_OF_10 is at least floor(log_5(2^mantissa_length)). Indeed, 10^n = 2^n * 5^n // so we only have to store the 5^n part in the mantissa (the 2^n part will go into the float's // exponent). #if MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_FLOAT -#define DEC_VAL_MAX 1e20F +#define MANTISSA_MAX 0x19999998U #define SMALL_NORMAL_VAL (1e-37F) #define SMALL_NORMAL_EXP (-37) #define EXACT_POWER_OF_10 (9) #elif MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_DOUBLE -#define DEC_VAL_MAX 1e200 +#define MANTISSA_MAX 0x1999999999999998ULL #define SMALL_NORMAL_VAL (1e-307) #define SMALL_NORMAL_EXP (-307) #define EXACT_POWER_OF_10 (22) #endif // Break out inner digit accumulation routine to ease trailing zero deferral. -static void accept_digit(mp_float_t *p_dec_val, int dig, int *p_exp_extra, int in) { +static mp_float_uint_t accept_digit(mp_float_uint_t p_mantissa, unsigned int dig, int *p_exp_extra, int in) { // Core routine to ingest an additional digit. - if (*p_dec_val < DEC_VAL_MAX) { + if (p_mantissa < MANTISSA_MAX) { // dec_val won't overflow so keep accumulating - *p_dec_val = 10 * *p_dec_val + dig; if (in == PARSE_DEC_IN_FRAC) { --(*p_exp_extra); } + return 10u * p_mantissa + dig; } else { // dec_val might overflow and we anyway can't represent more digits // of precision, so ignore the digit and just adjust the exponent if (in == PARSE_DEC_IN_INTG) { ++(*p_exp_extra); } + return p_mantissa; } } #endif // MICROPY_PY_BUILTINS_FLOAT @@ -273,6 +274,7 @@ mp_obj_t mp_parse_num_float(const char *str, size_t len, bool allow_imag, mp_lex // string should be a decimal number parse_dec_in_t in = PARSE_DEC_IN_INTG; bool exp_neg = false; + mp_float_uint_t mantissa = 0; int exp_val = 0; int exp_extra = 0; int trailing_zeros_intg = 0, trailing_zeros_frac = 0; @@ -288,9 +290,9 @@ mp_obj_t mp_parse_num_float(const char *str, size_t len, bool allow_imag, mp_lex exp_val = 10 * exp_val + dig; } } else { - if (dig == 0 || dec_val >= DEC_VAL_MAX) { + if (dig == 0 || mantissa >= MANTISSA_MAX) { // Defer treatment of zeros in fractional part. If nothing comes afterwards, ignore them. - // Also, once we reach DEC_VAL_MAX, treat every additional digit as a trailing zero. + // Also, once we reach MANTISSA_MAX, treat every additional digit as a trailing zero. if (in == PARSE_DEC_IN_INTG) { ++trailing_zeros_intg; } else { @@ -299,14 +301,14 @@ mp_obj_t mp_parse_num_float(const char *str, size_t len, bool allow_imag, mp_lex } else { // Time to un-defer any trailing zeros. Intg zeros first. while (trailing_zeros_intg) { - accept_digit(&dec_val, 0, &exp_extra, PARSE_DEC_IN_INTG); + mantissa = accept_digit(mantissa, 0, &exp_extra, PARSE_DEC_IN_INTG); --trailing_zeros_intg; } while (trailing_zeros_frac) { - accept_digit(&dec_val, 0, &exp_extra, PARSE_DEC_IN_FRAC); + mantissa = accept_digit(mantissa, 0, &exp_extra, PARSE_DEC_IN_FRAC); --trailing_zeros_frac; } - accept_digit(&dec_val, dig, &exp_extra, in); + mantissa = accept_digit(mantissa, dig, &exp_extra, in); } } } else if (in == PARSE_DEC_IN_INTG && dig == '.') { @@ -340,6 +342,7 @@ mp_obj_t mp_parse_num_float(const char *str, size_t len, bool allow_imag, mp_lex // apply the exponent, making sure it's not a subnormal value exp_val += exp_extra + trailing_zeros_intg; + dec_val = (mp_float_t)mantissa; if (exp_val < SMALL_NORMAL_EXP) { exp_val -= SMALL_NORMAL_EXP; dec_val *= SMALL_NORMAL_VAL; diff --git a/tests/float/float_parse.py b/tests/float/float_parse.py index de27c33e7be57..6131da0a63ab9 100644 --- a/tests/float/float_parse.py +++ b/tests/float/float_parse.py @@ -31,6 +31,9 @@ print(float("1e18446744073709551621")) print(float("1e-18446744073709551621")) +# mantissa overflow while processing deferred trailing zeros +print(float("10000000000000000000001")) + # check small decimals are as close to their true value as possible for n in range(1, 10): print(float("0.%u" % n) == n / 10) From 86526e9c2b5fcde2ecdee3817cc002920078b07b Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Wed, 26 Feb 2025 14:37:55 +1100 Subject: [PATCH 104/110] tools/mpremote: Optimise readline support in mount. This significantly speeds up readline on files opened directly from an mpremote mount. Signed-off-by: Andrew Leech --- tools/mpremote/mpremote/transport_serial.py | 41 ++++++++++++++------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/tools/mpremote/mpremote/transport_serial.py b/tools/mpremote/mpremote/transport_serial.py index d3148f9ace318..6aed0bb496b93 100644 --- a/tools/mpremote/mpremote/transport_serial.py +++ b/tools/mpremote/mpremote/transport_serial.py @@ -404,12 +404,13 @@ def umount_local(self): "CMD_OPEN": 4, "CMD_CLOSE": 5, "CMD_READ": 6, - "CMD_WRITE": 7, - "CMD_SEEK": 8, - "CMD_REMOVE": 9, - "CMD_RENAME": 10, - "CMD_MKDIR": 11, - "CMD_RMDIR": 12, + "CMD_READLINE": 7, + "CMD_WRITE": 8, + "CMD_SEEK": 9, + "CMD_REMOVE": 10, + "CMD_RENAME": 11, + "CMD_MKDIR": 12, + "CMD_RMDIR": 13, } fs_hook_code = """\ @@ -592,12 +593,16 @@ def readinto(self, buf): return n def readline(self): - l = '' - while 1: - c = self.read(1) - l += c - if c == '\\n' or c == '': - return l + c = self.cmd + c.begin(CMD_READLINE) + c.wr_s8(self.fd) + data = c.rd_bytes(None) + c.end() + if self.is_text: + data = str(data, 'utf8') + else: + data = bytes(data) + return data def readlines(self): ls = [] @@ -746,8 +751,7 @@ def __mount(): """ # Apply basic compression on hook code. -for key, value in fs_hook_cmds.items(): - fs_hook_code = re.sub(key, str(value), fs_hook_code) +fs_hook_code = re.sub(r"CMD_[A-Z_]+", lambda m: str(fs_hook_cmds[m.group(0)]), fs_hook_code) fs_hook_code = re.sub(" *#.*$", "", fs_hook_code, flags=re.MULTILINE) fs_hook_code = re.sub("\n\n+", "\n", fs_hook_code) fs_hook_code = re.sub(" ", " ", fs_hook_code) @@ -887,6 +891,14 @@ def do_read(self): self.wr_bytes(buf) # self.log_cmd(f"read {fd} {n} -> {len(buf)}") + def do_readline(self): + fd = self.rd_s8() + buf = self.data_files[fd][0].readline() + if self.data_files[fd][1]: + buf = bytes(buf, "utf8") + self.wr_bytes(buf) + # self.log_cmd(f"readline {fd} -> {len(buf)}") + def do_seek(self): fd = self.rd_s8() n = self.rd_s32() @@ -960,6 +972,7 @@ def do_rmdir(self): fs_hook_cmds["CMD_OPEN"]: do_open, fs_hook_cmds["CMD_CLOSE"]: do_close, fs_hook_cmds["CMD_READ"]: do_read, + fs_hook_cmds["CMD_READLINE"]: do_readline, fs_hook_cmds["CMD_WRITE"]: do_write, fs_hook_cmds["CMD_SEEK"]: do_seek, fs_hook_cmds["CMD_REMOVE"]: do_remove, From 71c7c03e411121b0193590bb683961e83c787ab1 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 26 Feb 2025 17:12:45 +1100 Subject: [PATCH 105/110] tools/mpremote/tests: Add test for RemoteFile.readline. Signed-off-by: Damien George --- tools/mpremote/tests/test_mount.sh | 4 ++++ tools/mpremote/tests/test_mount.sh.exp | 3 +++ 2 files changed, 7 insertions(+) diff --git a/tools/mpremote/tests/test_mount.sh b/tools/mpremote/tests/test_mount.sh index 9eab0b0d6e8c5..724ae1417fccc 100755 --- a/tools/mpremote/tests/test_mount.sh +++ b/tools/mpremote/tests/test_mount.sh @@ -26,3 +26,7 @@ $MPREMOTE mount ${TMP} exec "import mount_package; mount_package.x(); mount_pack echo ----- $MPREMOTE mount ${TMP} exec "open('test.txt', 'w').write('hello world\n')" cat "${TMP}/test.txt" + +# Test RemoteFile.readline and RemoteFile.readlines methods. +echo ----- +$MPREMOTE mount ${TMP} exec "print(open('test.txt').readlines())" diff --git a/tools/mpremote/tests/test_mount.sh.exp b/tools/mpremote/tests/test_mount.sh.exp index 560f5e4f10a9b..616cb75bcf5f1 100644 --- a/tools/mpremote/tests/test_mount.sh.exp +++ b/tools/mpremote/tests/test_mount.sh.exp @@ -5,3 +5,6 @@ Local directory ${TMP} is mounted at /remote ----- Local directory ${TMP} is mounted at /remote hello world +----- +['hello world\n'] +Local directory ${TMP} is mounted at /remote From 13b02376af4e9d4b76bf053935b51ddc7c87df03 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 19 Feb 2025 12:44:19 +1100 Subject: [PATCH 106/110] lib/pico-sdk: Update to version 2.1.1. Release notes: https://github.com/raspberrypi/pico-sdk/releases/tag/2.1.1 Signed-off-by: Damien George --- lib/pico-sdk | 2 +- ports/rp2/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pico-sdk b/lib/pico-sdk index 95ea6acad1311..bddd20f928ce7 160000 --- a/lib/pico-sdk +++ b/lib/pico-sdk @@ -1 +1 @@ -Subproject commit 95ea6acad131124694cda1c162c52cd30e0aece0 +Subproject commit bddd20f928ce76142793bef434d4f75f4af6e433 diff --git a/ports/rp2/CMakeLists.txt b/ports/rp2/CMakeLists.txt index 8a4092529b1f2..7002ad8769ea9 100644 --- a/ports/rp2/CMakeLists.txt +++ b/ports/rp2/CMakeLists.txt @@ -276,7 +276,7 @@ if(PICO_RP2040) elseif(PICO_RP2350 AND PICO_ARM) target_sources(pico_float_micropython INTERFACE ${PICO_SDK_PATH}/src/rp2_common/pico_float/float_aeabi_dcp.S - ${PICO_SDK_PATH}/src/rp2_common/pico_float/float_conv_m33.S + ${PICO_SDK_PATH}/src/rp2_common/pico_float/float_common_m33.S ) endif() From 9939b6c6b906cec702e825a7b58f5647c70c178e Mon Sep 17 00:00:00 2001 From: machdyne Date: Fri, 20 Dec 2024 23:39:02 +0100 Subject: [PATCH 107/110] rp2/boards/MACHDYNE_WERKZEUG: Add support for Machdyne Werkzeug. Signed-off-by: machdyne --- ports/rp2/boards/MACHDYNE_WERKZEUG/board.json | 19 ++++++++++++ .../MACHDYNE_WERKZEUG/mpconfigboard.cmake | 3 ++ .../boards/MACHDYNE_WERKZEUG/mpconfigboard.h | 3 ++ ports/rp2/boards/MACHDYNE_WERKZEUG/pins.csv | 30 +++++++++++++++++++ 4 files changed, 55 insertions(+) create mode 100644 ports/rp2/boards/MACHDYNE_WERKZEUG/board.json create mode 100644 ports/rp2/boards/MACHDYNE_WERKZEUG/mpconfigboard.cmake create mode 100644 ports/rp2/boards/MACHDYNE_WERKZEUG/mpconfigboard.h create mode 100644 ports/rp2/boards/MACHDYNE_WERKZEUG/pins.csv diff --git a/ports/rp2/boards/MACHDYNE_WERKZEUG/board.json b/ports/rp2/boards/MACHDYNE_WERKZEUG/board.json new file mode 100644 index 0000000000000..c657f18b21de6 --- /dev/null +++ b/ports/rp2/boards/MACHDYNE_WERKZEUG/board.json @@ -0,0 +1,19 @@ +{ + "deploy": [ + "../deploy.md" + ], + "docs": "", + "features": [ + "Dual-core", + "External Flash", + "USB-C" + ], + "images": [ + "werkzeug.jpg" + ], + "mcu": "rp2040", + "product": "Werkzeug", + "thumbnail": "", + "url": "https://machdyne.com/product/werkzeug-multi-tool/", + "vendor": "Machdyne" +} diff --git a/ports/rp2/boards/MACHDYNE_WERKZEUG/mpconfigboard.cmake b/ports/rp2/boards/MACHDYNE_WERKZEUG/mpconfigboard.cmake new file mode 100644 index 0000000000000..281c4e6f1b45f --- /dev/null +++ b/ports/rp2/boards/MACHDYNE_WERKZEUG/mpconfigboard.cmake @@ -0,0 +1,3 @@ +# cmake file for Machdyne Werkzeug +set(PICO_BOARD "machdyne_werkzeug") +set(PICO_PLATFORM "rp2040") diff --git a/ports/rp2/boards/MACHDYNE_WERKZEUG/mpconfigboard.h b/ports/rp2/boards/MACHDYNE_WERKZEUG/mpconfigboard.h new file mode 100644 index 0000000000000..476ea23b9c86b --- /dev/null +++ b/ports/rp2/boards/MACHDYNE_WERKZEUG/mpconfigboard.h @@ -0,0 +1,3 @@ +// Board and hardware specific configuration +#define MICROPY_HW_BOARD_NAME "Machdyne Werkzeug" +#define MICROPY_HW_FLASH_STORAGE_BYTES (384 * 1024) diff --git a/ports/rp2/boards/MACHDYNE_WERKZEUG/pins.csv b/ports/rp2/boards/MACHDYNE_WERKZEUG/pins.csv new file mode 100644 index 0000000000000..c3c578132069c --- /dev/null +++ b/ports/rp2/boards/MACHDYNE_WERKZEUG/pins.csv @@ -0,0 +1,30 @@ +GP0,GPIO0 +GP1,GPIO1 +GP2,GPIO2 +GP3,GPIO3 +GP4,GPIO4 +GP5,GPIO5 +GP6,GPIO6 +GP7,GPIO7 +GP8,GPIO8 +GP9,GPIO9 +GP10,GPIO10 +GP11,GPIO11 +PMOD10,GPIO12 +PMOD4,GPIO13 +PMOD9,GPIO14 +PMOD3,GPIO15 +PMOD8,GPIO16 +PMOD2,GPIO17 +PMOD7,GPIO18 +PMOD1,GPIO19 +LED_GREEN,GPIO20 +LED_RED,GPIO21 +USBA_POWER,GPIO22 +USBA_DN,GPIO23 +USBA_DP,GPIO24 +USBA_DP_PU,GPIO25 +GP26,GPIO26 +GP27,GPIO27 +GP28,GPIO28 +GP29,GPIO29 From cad62c20f2971b1248f55f27ab9f6d9bf8ccdafa Mon Sep 17 00:00:00 2001 From: Dryw Wade Date: Tue, 18 Feb 2025 14:57:53 -0700 Subject: [PATCH 108/110] rp2/boards/SPARKFUN_XRP_CONTROLLER_BETA: Add SparkFun XRP Controller. Signed-off-by: Dryw Wade --- .../SPARKFUN_XRP_CONTROLLER_BETA/board.json | 23 +++++++ .../SPARKFUN_XRP_CONTROLLER_BETA/manifest.py | 6 ++ .../mpconfigboard.cmake | 14 ++++ .../mpconfigboard.h | 28 ++++++++ .../SPARKFUN_XRP_CONTROLLER_BETA/pins.csv | 65 +++++++++++++++++++ 5 files changed, 136 insertions(+) create mode 100644 ports/rp2/boards/SPARKFUN_XRP_CONTROLLER_BETA/board.json create mode 100644 ports/rp2/boards/SPARKFUN_XRP_CONTROLLER_BETA/manifest.py create mode 100644 ports/rp2/boards/SPARKFUN_XRP_CONTROLLER_BETA/mpconfigboard.cmake create mode 100644 ports/rp2/boards/SPARKFUN_XRP_CONTROLLER_BETA/mpconfigboard.h create mode 100644 ports/rp2/boards/SPARKFUN_XRP_CONTROLLER_BETA/pins.csv diff --git a/ports/rp2/boards/SPARKFUN_XRP_CONTROLLER_BETA/board.json b/ports/rp2/boards/SPARKFUN_XRP_CONTROLLER_BETA/board.json new file mode 100644 index 0000000000000..cece574326503 --- /dev/null +++ b/ports/rp2/boards/SPARKFUN_XRP_CONTROLLER_BETA/board.json @@ -0,0 +1,23 @@ +{ + "deploy": [ + "../deploy.md" + ], + "docs": "", + "features": [ + "BLE", + "Dual-core", + "External Flash", + "IMU", + "JST-SH", + "USB", + "WiFi" + ], + "images": [ + "22727-_01.jpg" + ], + "mcu": "rp2040", + "product": "XRP Controller (Beta)", + "thumbnail": "", + "url": "https://www.sparkfun.com/sparkfun-experiential-robotics-platform-xrp-controller.html", + "vendor": "Sparkfun" +} diff --git a/ports/rp2/boards/SPARKFUN_XRP_CONTROLLER_BETA/manifest.py b/ports/rp2/boards/SPARKFUN_XRP_CONTROLLER_BETA/manifest.py new file mode 100644 index 0000000000000..4e38f09cdee4f --- /dev/null +++ b/ports/rp2/boards/SPARKFUN_XRP_CONTROLLER_BETA/manifest.py @@ -0,0 +1,6 @@ +include("$(PORT_DIR)/boards/manifest.py") + +require("bundle-networking") + +# Bluetooth +require("aioble") diff --git a/ports/rp2/boards/SPARKFUN_XRP_CONTROLLER_BETA/mpconfigboard.cmake b/ports/rp2/boards/SPARKFUN_XRP_CONTROLLER_BETA/mpconfigboard.cmake new file mode 100644 index 0000000000000..6c06efefb6479 --- /dev/null +++ b/ports/rp2/boards/SPARKFUN_XRP_CONTROLLER_BETA/mpconfigboard.cmake @@ -0,0 +1,14 @@ +# cmake file for SparkFun XRP Controller (Beta) + +set(PICO_BOARD "pico_w") + +set(MICROPY_PY_LWIP ON) +set(MICROPY_PY_NETWORK_CYW43 ON) + +# Bluetooth +set(MICROPY_PY_BLUETOOTH ON) +set(MICROPY_BLUETOOTH_BTSTACK ON) +set(MICROPY_PY_BLUETOOTH_CYW43 ON) + +# Board specific version of the frozen manifest +set(MICROPY_FROZEN_MANIFEST ${MICROPY_BOARD_DIR}/manifest.py) diff --git a/ports/rp2/boards/SPARKFUN_XRP_CONTROLLER_BETA/mpconfigboard.h b/ports/rp2/boards/SPARKFUN_XRP_CONTROLLER_BETA/mpconfigboard.h new file mode 100644 index 0000000000000..56071e1873341 --- /dev/null +++ b/ports/rp2/boards/SPARKFUN_XRP_CONTROLLER_BETA/mpconfigboard.h @@ -0,0 +1,28 @@ +// Board and hardware specific configuration +#define MICROPY_HW_BOARD_NAME "SparkFun XRP Controller (Beta)" + +// todo: We need something to check our binary size +#define MICROPY_HW_FLASH_STORAGE_BYTES (848 * 1024) + +// Enable networking. +#define MICROPY_PY_NETWORK 1 +#define MICROPY_PY_NETWORK_HOSTNAME_DEFAULT "XRP" + +// CYW43 driver configuration. +#define CYW43_USE_SPI (1) +#define CYW43_LWIP (1) +#define CYW43_GPIO (1) +#define CYW43_SPI_PIO (1) + +// For debugging mbedtls - also set +// Debug level (0-4) 1=warning, 2=info, 3=debug, 4=verbose +// #define MODUSSL_MBEDTLS_DEBUG_LEVEL 1 + +#define MICROPY_HW_PIN_EXT_COUNT CYW43_WL_GPIO_COUNT + +// If this returns true for a pin then its irq will not be disabled on a soft reboot +int mp_hal_is_pin_reserved(int n); +#define MICROPY_HW_PIN_RESERVED(i) mp_hal_is_pin_reserved(i) + +#define MICROPY_HW_I2C1_SDA (18) +#define MICROPY_HW_I2C1_SCL (19) diff --git a/ports/rp2/boards/SPARKFUN_XRP_CONTROLLER_BETA/pins.csv b/ports/rp2/boards/SPARKFUN_XRP_CONTROLLER_BETA/pins.csv new file mode 100644 index 0000000000000..dd047063b5c02 --- /dev/null +++ b/ports/rp2/boards/SPARKFUN_XRP_CONTROLLER_BETA/pins.csv @@ -0,0 +1,65 @@ +# XRP default pin names +MOTOR_L_IN_1,GPIO6 +MOTOR_L_IN_2,GPIO7 +MOTOR_R_IN_1,GPIO14 +MOTOR_R_IN_2,GPIO15 +MOTOR_3_IN_1,GPIO2 +MOTOR_3_IN_2,GPIO3 +MOTOR_4_IN_1,GPIO10 +MOTOR_4_IN_2,GPIO11 +MOTOR_L_ENCODER_A,GPIO4 +MOTOR_L_ENCODER_B,GPIO5 +MOTOR_R_ENCODER_A,GPIO12 +MOTOR_R_ENCODER_B,GPIO13 +MOTOR_3_ENCODER_A,GPIO0 +MOTOR_3_ENCODER_B,GPIO1 +MOTOR_4_ENCODER_A,GPIO8 +MOTOR_4_ENCODER_B,GPIO9 +SERVO_1,GPIO16 +SERVO_2,GPIO17 +I2C_SDA_1,GPIO18 +I2C_SCL_1,GPIO19 +DISTANCE_TRIGGER,GPIO20 +DISTANCE_ECHO,GPIO21 +LINE_L,GPIO26 +LINE_R,GPIO27 +BOARD_VIN_MEASURE,GPIO28 +BOARD_USER_BUTTON,GPIO22 +BOARD_LED,EXT_GPIO0 + +# XRP alternate pin names +ML_IN_1,GPIO6 +ML_IN_2,GPIO7 +MR_IN_1,GPIO14 +MR_IN_2,GPIO15 +M3_IN_1,GPIO2 +M3_IN_2,GPIO3 +M4_IN_1,GPIO10 +M4_IN_2,GPIO11 +ML_ENC_A,GPIO4 +ML_ENC_B,GPIO5 +MR_ENC_A,GPIO12 +MR_ENC_B,GPIO13 +M3_ENC_A,GPIO0 +M3_ENC_B,GPIO1 +M4_ENC_A,GPIO8 +M4_ENC_B,GPIO9 +S1,GPIO16 +S2,GPIO17 +SDA_1,GPIO18 +SCL_1,GPIO19 +RANGE_TRIGGER,GPIO20 +RANGE_ECHO,GPIO21 +REFLECTANCE_L,GPIO26 +REFLECTANCE_R,GPIO27 +BRD_VIN,GPIO28 +BRD_USR_BTN,GPIO22 +BRD_LED,EXT_GPIO0 + +# LED default names +LED,EXT_GPIO0 + +# Radio GPIO pins +WL_GPIO0,EXT_GPIO0 +WL_GPIO1,EXT_GPIO1 +WL_GPIO2,EXT_GPIO2 From bb4ec886f8994ad7b2db6f80c7b46e6eedbc40e7 Mon Sep 17 00:00:00 2001 From: Malcolm McKellips Date: Fri, 24 Jan 2025 09:36:02 -0700 Subject: [PATCH 109/110] rp2/machine_i2c: Make I2C bus ID arg optional with default. This commit gives the option to not pass an I2C Bus ID when creating a machine I2C object. If the ID is not provided, the default bus ID (which is `PICO_DEFAULT_I2C`) is used. This allows users to simply declare an I2C object with `machine.I2C()` without passing any arguments, thus creating an object with the default I2C ID, SCL, and SDA. Signed-off-by: Malcolm McKellips --- ports/rp2/machine_i2c.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ports/rp2/machine_i2c.c b/ports/rp2/machine_i2c.c index 28032e8085c24..e97a852b24305 100644 --- a/ports/rp2/machine_i2c.c +++ b/ports/rp2/machine_i2c.c @@ -97,7 +97,11 @@ static void machine_i2c_print(const mp_print_t *print, mp_obj_t self_in, mp_prin mp_obj_t machine_i2c_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *all_args) { enum { ARG_id, ARG_freq, ARG_scl, ARG_sda, ARG_timeout }; static const mp_arg_t allowed_args[] = { - { MP_QSTR_id, MP_ARG_REQUIRED | MP_ARG_OBJ }, + #ifdef PICO_DEFAULT_I2C + { MP_QSTR_id, MP_ARG_INT, {.u_int = PICO_DEFAULT_I2C} }, + #else + { MP_QSTR_id, MP_ARG_INT, {.u_int = -1} }, + #endif { MP_QSTR_freq, MP_ARG_INT, {.u_int = DEFAULT_I2C_FREQ} }, { MP_QSTR_scl, MICROPY_I2C_PINS_ARG_OPTS | MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = MP_ROM_NONE} }, { MP_QSTR_sda, MICROPY_I2C_PINS_ARG_OPTS | MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = MP_ROM_NONE} }, @@ -108,8 +112,9 @@ mp_obj_t machine_i2c_make_new(const mp_obj_type_t *type, size_t n_args, size_t n mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; mp_arg_parse_all_kw_array(n_args, n_kw, all_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args); - // Get I2C bus. - int i2c_id = mp_obj_get_int(args[ARG_id].u_obj); + int i2c_id = args[ARG_id].u_int; + + // Check if the I2C bus is valid if (i2c_id < 0 || i2c_id >= MP_ARRAY_SIZE(machine_i2c_obj)) { mp_raise_msg_varg(&mp_type_ValueError, MP_ERROR_TEXT("I2C(%d) doesn't exist"), i2c_id); } From c143eb50248b3a3f521775f09f5073675095e48a Mon Sep 17 00:00:00 2001 From: Malcolm McKellips Date: Fri, 24 Jan 2025 13:05:07 -0700 Subject: [PATCH 110/110] esp32/machine_i2c: Make I2C bus ID arg optional with default. Similar to the previous commit, this allows constructing an I2C instance without specifying an ID. The default ID is I2C_NUM_0. Signed-off-by: Malcolm McKellips --- ports/esp32/machine_i2c.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ports/esp32/machine_i2c.c b/ports/esp32/machine_i2c.c index 732a62f47f193..12b86e4aa0294 100644 --- a/ports/esp32/machine_i2c.c +++ b/ports/esp32/machine_i2c.c @@ -146,12 +146,15 @@ static void machine_hw_i2c_print(const mp_print_t *print, mp_obj_t self_in, mp_p } mp_obj_t machine_hw_i2c_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *all_args) { - MP_MACHINE_I2C_CHECK_FOR_LEGACY_SOFTI2C_CONSTRUCTION(n_args, n_kw, all_args); + // Create a SoftI2C instance if no id is specified (or is -1) but other arguments are given + if (n_args != 0) { + MP_MACHINE_I2C_CHECK_FOR_LEGACY_SOFTI2C_CONSTRUCTION(n_args, n_kw, all_args); + } // Parse args enum { ARG_id, ARG_scl, ARG_sda, ARG_freq, ARG_timeout }; static const mp_arg_t allowed_args[] = { - { MP_QSTR_id, MP_ARG_REQUIRED | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL} }, + { MP_QSTR_id, MP_ARG_INT, {.u_int = I2C_NUM_0} }, { MP_QSTR_scl, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL} }, { MP_QSTR_sda, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL} }, { MP_QSTR_freq, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 400000} }, @@ -161,7 +164,9 @@ mp_obj_t machine_hw_i2c_make_new(const mp_obj_type_t *type, size_t n_args, size_ mp_arg_parse_all_kw_array(n_args, n_kw, all_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args); // Get I2C bus - mp_int_t i2c_id = mp_obj_get_int(args[ARG_id].u_obj); + mp_int_t i2c_id = args[ARG_id].u_int; + + // Check if the I2C bus is valid if (!(I2C_NUM_0 <= i2c_id && i2c_id < I2C_NUM_MAX)) { mp_raise_msg_varg(&mp_type_ValueError, MP_ERROR_TEXT("I2C(%d) doesn't exist"), i2c_id); }