Compress all translated strings with Huffman coding.

This saves code space in builds which use link-time optimization.
The optimization drops the untranslated strings and replaces them
with a compressed_string_t struct. It can then be decompressed to
a c string.

Builds without LTO work as well but include both untranslated
strings and compressed strings.

This work could be expanded to include QSTRs and loaded strings if
a compress method is added to C. Its tracked in micropython#531.

Author: tannewt

.gitmodules

@@ -80,3 +80,6 @@
 	path = lib/tinyusb
 	url = https://github.com/hathach/tinyusb.git
 	branch = develop
+[submodule "tools/huffman"]
+	path = tools/huffman
+	url = https://github.com/tannewt/huffman.git

extmod/machine_mem.c

@@ -42,7 +42,7 @@
 STATIC uintptr_t machine_mem_get_addr(mp_obj_t addr_o, uint align) {
     uintptr_t addr = mp_obj_int_get_truncated(addr_o);
     if ((addr & (align - 1)) != 0) {
-        nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_ValueError, "address %08x is not aligned to %d bytes", addr, align));
+        nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_ValueError, translate("address %08x is not aligned to %d bytes"), addr, align));
     }
     return addr;
 }

extmod/modframebuf.c

@@ -296,7 +296,7 @@ STATIC mp_obj_t framebuf_make_new(const mp_obj_type_t *type, size_t n_args, size
         case FRAMEBUF_GS8:
             break;
         default:
-            mp_raise_ValueError("invalid format");
+            mp_raise_ValueError(translate("invalid format"));
     }
 
     return MP_OBJ_FROM_PTR(o);

extmod/modubinascii.c

@@ -35,7 +35,7 @@
 static void check_not_unicode(const mp_obj_t arg) {
 #if MICROPY_CPYTHON_COMPAT
     if (MP_OBJ_IS_STR(arg)) {
-        mp_raise_TypeError("a bytes-like object is required");
+        mp_raise_TypeError(translate("a bytes-like object is required"));
     }
 #endif
 }
@@ -87,7 +87,7 @@ mp_obj_t mod_binascii_unhexlify(mp_obj_t data) {
     mp_get_buffer_raise(data, &bufinfo, MP_BUFFER_READ);
 
     if ((bufinfo.len & 1) != 0) {
-        mp_raise_ValueError("odd-length string");
+        mp_raise_ValueError(translate("odd-length string"));
     }
     vstr_t vstr;
     vstr_init_len(&vstr, bufinfo.len / 2);
@@ -98,7 +98,7 @@ mp_obj_t mod_binascii_unhexlify(mp_obj_t data) {
         if (unichar_isxdigit(hex_ch)) {
             hex_byte += unichar_xdigit_value(hex_ch);
         } else {
-            mp_raise_ValueError("non-hex digit found");
+            mp_raise_ValueError(translate("non-hex digit found"));
         }
         if (i & 1) {
             hex_byte <<= 4;
@@ -166,7 +166,7 @@ mp_obj_t mod_binascii_a2b_base64(mp_obj_t data) {
     }
 
     if (nbits) {
-        mp_raise_ValueError("incorrect padding");
+        mp_raise_ValueError(translate("incorrect padding"));
     }
 
     return mp_obj_new_str_from_vstr(&mp_type_bytes, &vstr);

extmod/modure.c

@@ -158,7 +158,7 @@ STATIC mp_obj_t re_split(size_t n_args, const mp_obj_t *args) {
         mp_obj_t s = mp_obj_new_str_of_type(str_type, (const byte*)subj.begin, caps[0] - subj.begin);
         mp_obj_list_append(retval, s);
         if (self->re.sub > 0) {
-            mp_raise_NotImplementedError("Splitting with sub-captures");
+            mp_raise_NotImplementedError(translate("Splitting with sub-captures"));
         }
         subj.begin = caps[1];
         if (maxsplit > 0 && --maxsplit == 0) {
@@ -204,7 +204,7 @@ STATIC mp_obj_t mod_re_compile(size_t n_args, const mp_obj_t *args) {
     int error = re1_5_compilecode(&o->re, re_str);
     if (error != 0) {
 error:
-        mp_raise_ValueError("Error in regex");
+        mp_raise_ValueError(translate("Error in regex"));
     }
     if (flags & FLAG_DEBUG) {
         re1_5_dumpcode(&o->re);

main.c

@@ -128,13 +128,22 @@ const char* first_existing_file_in_list(const char ** filenames) {
     return NULL;
 }
 
+void write_compressed(const compressed_string_t* compressed) {
+    char decompressed[compressed->length];
+    decompress(compressed, decompressed);
+    serial_write(decompressed);
+}
+
 bool maybe_run_list(const char ** filenames, pyexec_result_t* exec_result) {
     const char* filename = first_existing_file_in_list(filenames);
     if (filename == NULL) {
         return false;
     }
     mp_hal_stdout_tx_str(filename);
-    mp_hal_stdout_tx_str(translate(" output:\n"));
+    const compressed_string_t* compressed = translate(" output:\n");
+    char decompressed[compressed->length];
+    decompress(compressed, decompressed);
+    mp_hal_stdout_tx_str(decompressed);
     pyexec_file(filename, exec_result);
     return true;
 }
@@ -145,11 +154,11 @@ bool run_code_py(safe_mode_t safe_mode) {
     if (serial_connected_at_start) {
         serial_write("\n");
         if (autoreload_is_enabled()) {
-            serial_write(translate("Auto-reload is on. Simply save files over USB to run them or enter REPL to disable.\n"));
+            write_compressed(translate("Auto-reload is on. Simply save files over USB to run them or enter REPL to disable.\n"));
         } else if (safe_mode != NO_SAFE_MODE) {
-            serial_write(translate("Running in safe mode! Auto-reload is off.\n"));
+            write_compressed(translate("Running in safe mode! Auto-reload is off.\n"));
         } else if (!autoreload_is_enabled()) {
-            serial_write(translate("Auto-reload is off.\n"));
+            write_compressed(translate("Auto-reload is off.\n"));
         }
     }
     #endif
@@ -163,7 +172,7 @@ bool run_code_py(safe_mode_t safe_mode) {
     bool found_main = false;
 
     if (safe_mode != NO_SAFE_MODE) {
-        serial_write(translate("Running in safe mode! Not running saved code.\n"));
+        write_compressed(translate("Running in safe mode! Not running saved code.\n"));
     } else {
         new_status_color(MAIN_RUNNING);
 
@@ -179,7 +188,7 @@ bool run_code_py(safe_mode_t safe_mode) {
         if (!found_main){
             found_main = maybe_run_list(double_extension_filenames, &result);
             if (found_main) {
-                serial_write(translate("WARNING: Your code filename has two extensions\n"));
+                write_compressed(translate("WARNING: Your code filename has two extensions\n"));
             }
         }
         stop_mp();
@@ -218,37 +227,37 @@ bool run_code_py(safe_mode_t safe_mode) {
 
             if (!serial_connected_at_start) {
                 if (autoreload_is_enabled()) {
-                    serial_write(translate("Auto-reload is on. Simply save files over USB to run them or enter REPL to disable.\n"));
+                    write_compressed(translate("Auto-reload is on. Simply save files over USB to run them or enter REPL to disable.\n"));
                 } else {
-                    serial_write(translate("Auto-reload is off.\n"));
+                    write_compressed(translate("Auto-reload is off.\n"));
                 }
             }
             // Output a user safe mode string if its set.
             #ifdef BOARD_USER_SAFE_MODE
             if (safe_mode == USER_SAFE_MODE) {
                 serial_write("\n");
-                serial_write(translate("You requested starting safe mode by "));
+                write_compressed(translate("You requested starting safe mode by "));
                 serial_write(BOARD_USER_SAFE_MODE_ACTION);
                 serial_write("\n");
-                serial_write(translate("To exit, please reset the board without "));
+                write_compressed(translate("To exit, please reset the board without "));
                 serial_write(BOARD_USER_SAFE_MODE_ACTION);
                 serial_write("\n");
             } else
             #endif
             if (safe_mode != NO_SAFE_MODE) {
                 serial_write("\n");
-                serial_write(translate("You are running in safe mode which means something really bad happened.\n"));
+                write_compressed(translate("You are running in safe mode which means something really bad happened.\n"));
                 if (safe_mode == HARD_CRASH) {
-                    serial_write(translate("Looks like our core CircuitPython code crashed hard. Whoops!\n"));
-                    serial_write(translate("Please file an issue here with the contents of your CIRCUITPY drive:\n"));
+                    write_compressed(translate("Looks like our core CircuitPython code crashed hard. Whoops!\n"));
+                    write_compressed(translate("Please file an issue here with the contents of your CIRCUITPY drive:\n"));
                     serial_write("https://github.com/adafruit/circuitpython/issues\n");
                 } else if (safe_mode == BROWNOUT) {
-                    serial_write(translate("The microcontroller's power dipped. Please make sure your power supply provides\n"));
-                    serial_write(translate("enough power for the whole circuit and press reset (after ejecting CIRCUITPY).\n"));
+                    write_compressed(translate("The microcontroller's power dipped. Please make sure your power supply provides\n"));
+                    write_compressed(translate("enough power for the whole circuit and press reset (after ejecting CIRCUITPY).\n"));
                 }
             }
             serial_write("\n");
-            serial_write(translate("Press any key to enter the REPL. Use CTRL-D to reload."));
+            write_compressed(translate("Press any key to enter the REPL. Use CTRL-D to reload."));
         }
         if (serial_connected_before_animation && !serial_connected()) {
             serial_connected_at_start = false;
@@ -403,7 +412,7 @@ int __attribute__((used)) main(void) {
         }
         if (exit_code == PYEXEC_FORCED_EXIT) {
             if (!first_run) {
-                serial_write(translate("soft reboot\n"));
+                write_compressed(translate("soft reboot\n"));
             }
             first_run = false;
             skip_repl = run_code_py(safe_mode);

ports/atmel-samd/audio_dma.c

@@ -33,6 +33,7 @@
 #include "shared-bindings/audioio/WaveFile.h"
 
 #include "py/mpstate.h"
+#include "py/runtime.h"
 
 static audio_dma_t* audio_dma_state[AUDIO_DMA_CHANNEL_COUNT];
 
@@ -279,6 +280,10 @@ audio_dma_result audio_dma_setup_playback(audio_dma_t* dma,
         // We're likely double buffering so set up the block interrupts.
         turn_on_event_system();
         dma->event_channel = find_sync_event_channel();
+
+        if (dma->event_channel >= EVSYS_SYNCH_NUM) {
+            mp_raise_RuntimeError(translate("All sync event channels in use"));
+        }
         init_event_channel_interrupt(dma->event_channel, CORE_GCLK, EVSYS_ID_GEN_DMAC_CH_0 + dma_channel);
 
         // We keep the audio_dma_t for internal use and the sample as a root pointer because it

ports/atmel-samd/bindings/samd/Clock.c

@@ -132,9 +132,9 @@ STATIC mp_obj_t samd_clock_set_calibration(mp_obj_t self_in, mp_obj_t calibratio
     samd_clock_obj_t *self = MP_OBJ_TO_PTR(self_in);
     int ret = clock_set_calibration(self->type, self->index, mp_obj_get_int(calibration));
     if (ret == -2)
-        mp_raise_AttributeError("calibration is read only");
+        mp_raise_AttributeError(translate("calibration is read only"));
     if (ret == -1)
-        mp_raise_ValueError("calibration is out of range");
+        mp_raise_ValueError(translate("calibration is out of range"));
     return mp_const_none;
 }
 

ports/atmel-samd/common-hal/audiobusio/PDMIn.c

@@ -357,6 +357,9 @@ uint32_t common_hal_audiobusio_pdmin_record_to_buffer(audiobusio_pdmin_obj_t* se
         uint16_t* output_buffer, uint32_t output_buffer_length) {
     uint8_t dma_channel = find_free_audio_dma_channel();
     uint8_t event_channel = find_sync_event_channel();
+    if (event_channel >= EVSYS_SYNCH_NUM) {
+        mp_raise_RuntimeError(translate("All sync event channels in use"));
+    }
 
     // We allocate two buffers on the stack to use for double buffering.
     const uint8_t samples_per_buffer = SAMPLES_PER_BUFFER;

ports/atmel-samd/common-hal/audioio/AudioOut.c

@@ -211,6 +211,10 @@ void common_hal_audioio_audioout_construct(audioio_audioout_obj_t* self,
     // Find a free event channel. We start at the highest channels because we only need and async
     // path.
     uint8_t channel = find_async_event_channel();
+    if (channel >= EVSYS_CHANNELS) {
+        mp_raise_RuntimeError(translate("All event channels in use"));
+    }
+
     #ifdef SAMD51
     connect_event_user_to_channel(EVSYS_ID_USER_DAC_START_1, channel);
     #define EVSYS_ID_USER_DAC_START EVSYS_ID_USER_DAC_START_0

ports/atmel-samd/common-hal/busio/SPI.c

@@ -129,7 +129,7 @@ void common_hal_busio_spi_construct(busio_spi_obj_t *self,
     }
     #endif
     if (sercom == NULL) {
-        mp_raise_ValueError("Invalid pins");
+        mp_raise_ValueError(translate("Invalid pins"));
     }
 
     // Set up SPI clocks on SERCOM.

ports/atmel-samd/peripherals

@@ -156,7 +156,7 @@ STATIC void compile_error_set_line(compiler_t *comp, mp_parse_node_t pn) {
     }
 }
 
-STATIC void compile_syntax_error(compiler_t *comp, mp_parse_node_t pn, const char *msg) {
+STATIC void compile_syntax_error(compiler_t *comp, mp_parse_node_t pn, const compressed_string_t *msg) {
     // only register the error if there has been no other error
     if (comp->compile_error == MP_OBJ_NULL) {
         comp->compile_error = mp_obj_new_exception_msg(&mp_type_SyntaxError, msg);
@@ -949,7 +949,7 @@ STATIC void compile_del_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) {
 
 STATIC void compile_break_cont_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) {
     uint16_t label;
-    const char *error_msg;
+    const compressed_string_t *error_msg;
     if (MP_PARSE_NODE_STRUCT_KIND(pns) == PN_break_stmt) {
         label = comp->break_label;
         error_msg = translate("'break' outside loop");