From 11adc90f12d9d55f3868d03fe97d8deb5ed72256 Mon Sep 17 00:00:00 2001 From: "Memfault Inc." Date: Mon, 15 Jul 2024 17:43:13 -0400 Subject: [PATCH] memfaultd 1.13.0 (Build 2255289) --- .gitignore | 1 + Cargo.lock | 2759 +++++++++++++++++ Cargo.toml | 15 + Cross.toml | 13 + License.txt | 31 + README.md | 37 + VERSION | 3 + libmemfaultc/.clang-format | 10 + libmemfaultc/.gitignore | 2 + libmemfaultc/src/crash.c | 15 + libmemfaultc/src/swupdate.c | 215 ++ libmemfaultc/src/systemd.c | 200 ++ memfaultc-sys/Cargo.toml | 19 + memfaultc-sys/build.rs | 78 + memfaultc-sys/src/coredump.rs | 6 + memfaultc-sys/src/lib.rs | 18 + memfaultc-sys/src/swupdate.rs | 22 + memfaultc-sys/src/systemd.rs | 35 + memfaultc-sys/src/systemd_mock.rs | 119 + memfaultd.init | 150 + memfaultd.service | 15 + memfaultd/Cargo.toml | 104 + memfaultd/DEVELOPMENT.md | 60 + memfaultd/build.rs | 66 + memfaultd/builtin.conf | 64 + memfaultd/memfaultd.conf | 13 + memfaultd/src/bin/memfault-core-handler.rs | 11 + memfaultd/src/bin/memfaultctl.rs | 11 + memfaultd/src/bin/memfaultd.rs | 20 + memfaultd/src/bin/mfw.rs | 11 + memfaultd/src/cli/cargs.rs | 46 + .../src/cli/memfault_core_handler/arch.rs | 72 + .../src/cli/memfault_core_handler/auxv.rs | 114 + .../core_elf_memfault_note.rs | 221 ++ .../memfault_core_handler/core_elf_note.rs | 638 ++++ .../cli/memfault_core_handler/core_reader.rs | 193 ++ .../memfault_core_handler/core_transformer.rs | 474 +++ .../cli/memfault_core_handler/core_writer.rs | 392 +++ .../cli/memfault_core_handler/find_dynamic.rs | 392 +++ .../memfault_core_handler/find_elf_headers.rs | 168 + .../cli/memfault_core_handler/find_stack.rs | 133 + .../fixtures/elf-core-runtime-ld-paths.elf | Bin 0 -> 319488 bytes .../fixtures/sample_note.bin | Bin 0 -> 9612 bytes .../cli/memfault_core_handler/log_wrapper.rs | 123 + .../cli/memfault_core_handler/memory_range.rs | 127 + .../src/cli/memfault_core_handler/mod.rs | 306 ++ .../src/cli/memfault_core_handler/procfs.rs | 79 + .../src/cli/memfault_core_handler/r_debug.rs | 97 + ...ault_note__test__serialize_debug_data.snap | 36 + ...t__serialize_metadata_as_map@app_logs.snap | 165 + ...lize_metadata_as_map@kernel_selection.snap | 119 + ...st__serialize_metadata_as_map@threads.snap | 129 + ..._test__iterate_elf_notes_with_fixture.snap | 756 +++++ ...mer__test__transform@kernel_selection.snap | 387 +++ ...nsformer__test__transform@threads_32k.snap | 244 ++ ...ansform@threads_32k_no_filter_support.snap | 387 +++ ...mic__test__find_dynamic_linker_ranges.snap | 58 + ...andler__log_wrapper__test__log_saving.snap | 10 + .../cli/memfault_core_handler/test_utils.rs | 231 ++ memfaultd/src/cli/memfault_watch/buffer.rs | 120 + memfaultd/src/cli/memfault_watch/mod.rs | 218 ++ .../cli/memfaultctl/add_battery_reading.rs | 19 + memfaultd/src/cli/memfaultctl/config_file.rs | 176 ++ memfaultd/src/cli/memfaultctl/coredump.rs | 87 + memfaultd/src/cli/memfaultctl/export.rs | 54 + memfaultd/src/cli/memfaultctl/mod.rs | 322 ++ memfaultd/src/cli/memfaultctl/report_sync.rs | 25 + memfaultd/src/cli/memfaultctl/session.rs | 54 + memfaultd/src/cli/memfaultctl/sync.rs | 18 + .../src/cli/memfaultctl/write_attributes.rs | 82 + memfaultd/src/cli/memfaultd.rs | 154 + memfaultd/src/cli/memfaultd_client.rs | 206 ++ memfaultd/src/cli/mod.rs | 70 + memfaultd/src/cli/show_settings.rs | 192 ++ ...ultd__cli__show_settings__tests__test.snap | 26 + memfaultd/src/cli/version.rs | 11 + memfaultd/src/collectd/collectd_handler.rs | 216 ++ .../collectd/fixtures/sample-with-null.json | 158 + memfaultd/src/collectd/fixtures/sample1.json | 146 + .../fixtures/statsd-counter-first-seen.json | 50 + .../src/collectd/fixtures/statsd-counter.json | 50 + memfaultd/src/collectd/mod.rs | 7 + memfaultd/src/collectd/payload.rs | 210 ++ ...en_builtin_system_metrics_are_enabled.snap | 8 + ..._collectd_handler__tests__handle_push.snap | 7 + ...ores_data_when_data_collection_is_off.snap | 5 + ...ctd__payload__tests__sample-with-null.snap | 42 + ...td__collectd__payload__tests__sample1.snap | 168 + ...oad__tests__statsd-counter-first-seen.snap | 24 + ...lectd__payload__tests__statsd-counter.snap | 42 + memfaultd/src/config/config_file.rs | 521 ++++ memfaultd/src/config/device_config.rs | 93 + memfaultd/src/config/device_info.rs | 545 ++++ memfaultd/src/config/mod.rs | 457 +++ ...nfig__config_file__test__empty_object.snap | 73 + ...ig__config_file__test__metrics_config.snap | 79 + ...e_bool_to_runtime_config@empty_object.snap | 5 + ..._write_bool_to_runtime_config@no_file.snap | 5 + ...rite_bool_to_runtime_config@other_key.snap | 5 + ...file__test__with_connectivity_monitor.snap | 83 + ...ith_coredump_capture_strategy_threads.snap | 73 + ...file__test__with_log_to_metrics_rules.snap | 90 + ..._config_file__test__with_partial_logs.snap | 73 + ...fig__config_file__test__with_sessions.snap | 81 + ...e__test__without_coredump_compression.snap | 73 + .../src/config/test-config/empty_object.json | 1 + .../config/test-config/metrics_config.json | 12 + .../with_connectivity_monitor.json | 13 + ...ith_coredump_capture_strategy_threads.json | 8 + .../config/test-config/with_invalid_path.json | 3 + .../test-config/with_invalid_swt_swv.json | 4 + .../with_log_to_metrics_rules.json | 21 + .../config/test-config/with_partial_logs.json | 5 + .../src/config/test-config/with_sessions.json | 8 + .../with_sessions_invalid_metric_name.json | 11 + .../with_sessions_invalid_session_name.json | 8 + .../without_coredump_compression.json | 5 + memfaultd/src/config/utils.rs | 48 + memfaultd/src/coredump/mod.rs | 30 + memfaultd/src/fluent_bit/decode_time.rs | 233 ++ memfaultd/src/fluent_bit/mod.rs | 380 +++ memfaultd/src/http_server/handler.rs | 94 + memfaultd/src/http_server/mod.rs | 19 + memfaultd/src/http_server/request_bodies.rs | 28 + memfaultd/src/http_server/server.rs | 73 + ...rver__response__tests__response_error.snap | 5 + ...__response__tests__response_not_found.snap | 5 + ..._server__response__tests__response_ok.snap | 5 + memfaultd/src/http_server/utils.rs | 15 + memfaultd/src/lib.rs | 33 + memfaultd/src/logs/completed_log.rs | 15 + memfaultd/src/logs/fluent_bit_adapter.rs | 104 + memfaultd/src/logs/headroom.rs | 465 +++ memfaultd/src/logs/journald_parser.rs | 393 +++ memfaultd/src/logs/journald_provider.rs | 173 ++ memfaultd/src/logs/log_collector.rs | 755 +++++ memfaultd/src/logs/log_entry.rs | 87 + memfaultd/src/logs/log_file.rs | 267 ++ memfaultd/src/logs/log_to_metrics.rs | 204 ++ memfaultd/src/logs/mod.rs | 21 + memfaultd/src/logs/recovery.rs | 282 ++ ...it_adapter__tests__fluent_bit_adapter.snap | 10 + ..._parser__test__from_raw_journal_entry.snap | 11 + ...nald_parser__test__journal_happy_path.snap | 11 + ...__journald_provider__test__happy_path.snap | 11 + ...faultd__logs__log_entry__tests__empty.snap | 8 + ..._entry__tests__extra_attribute_filter.snap | 11 + ...td__logs__log_entry__tests__extra_key.snap | 10 + ...gs__log_entry__tests__multi_key_match.snap | 13 + ..._logs__log_entry__tests__only_message.snap | 10 + memfaultd/src/mar/chunks.rs | 14 + memfaultd/src/mar/chunks/chunk.rs | 81 + memfaultd/src/mar/chunks/chunk_header.rs | 59 + memfaultd/src/mar/chunks/chunk_message.rs | 45 + memfaultd/src/mar/chunks/chunk_wrapper.rs | 43 + memfaultd/src/mar/chunks/crc_padded_stream.rs | 64 + memfaultd/src/mar/clean.rs | 1072 +++++++ memfaultd/src/mar/export.rs | 421 +++ memfaultd/src/mar/export_format.rs | 78 + memfaultd/src/mar/manifest.rs | 463 +++ memfaultd/src/mar/mar_entry.rs | 152 + memfaultd/src/mar/mar_entry_builder.rs | 272 ++ memfaultd/src/mar/mod.rs | 25 + ...ltd__mar__manifest__tests__attributes.snap | 46 + ...__mar__manifest__tests__coredump-gzip.snap | 30 + ...__mar__manifest__tests__coredump-none.snap | 29 + ...__mar__manifest__tests__device_config.snap | 29 + ...d__mar__manifest__tests__elf_coredump.snap | 30 + ...ultd__mar__manifest__tests__heartbeat.snap | 52 + ...ifest__tests__heartbeat_with_duration.snap | 53 + ...aultd__mar__manifest__tests__log-none.snap | 39 + ...aultd__mar__manifest__tests__log-zlib.snap | 40 + ...__mar__manifest__tests__metric_report.snap | 56 + ...mfaultd__mar__manifest__tests__reboot.snap | 29 + ...tests__serialization_of_custom_reboot.snap | 32 + ...alization_of_custom_unexpected_reboot.snap | 32 + ...s__serialization_of_device_attributes.snap | 46 + ...ests__serialization_of_device_configc.snap | 29 + ...sts__serialization_of_linux_heartbeat.snap | 34 + ...ifest__tests__serialization_of_reboot.snap | 29 + .../src/mar/test-manifests/attributes.json | 38 + .../src/mar/test-manifests/device_config.json | 21 + .../src/mar/test-manifests/elf_coredump.json | 22 + .../src/mar/test-manifests/heartbeat.json | 48 + .../heartbeat_with_duration.json | 49 + memfaultd/src/mar/test-manifests/log.json | 25 + .../src/mar/test-manifests/metric_report.json | 52 + memfaultd/src/mar/test-manifests/reboot.json | 21 + memfaultd/src/mar/test_utils.rs | 243 ++ memfaultd/src/mar/upload.rs | 514 +++ memfaultd/src/memfaultd.rs | 534 ++++ .../src/metrics/battery/battery_monitor.rs | 365 +++ .../battery/battery_reading_handler.rs | 188 ++ memfaultd/src/metrics/battery/mod.rs | 9 + ...ler__tests__charging_then_discharging.snap | 9 + ...y_reading_handler__tests__handle_push.snap | 9 + ...ores_data_when_data_collection_is_off.snap | 5 + ...ndler__tests__non_integer_percentages.snap | 9 + ...ler__tests__nonconsecutive_discharges.snap | 9 + .../connectivity/connectivity_monitor.rs | 217 ++ memfaultd/src/metrics/connectivity/mod.rs | 8 + .../connectivity/report_sync_event_handler.rs | 150 + ...ty_monitor__tests__fully_disconnected.snap | 8 + ...sts__half_connected_half_disconnected.snap | 8 + ...ivity_monitor__tests__while_connected.snap | 8 + ...r__tests__handle_multiple_sync_events.snap | 8 + ...t_handler__tests__handle_sync_failure.snap | 7 + ...t_handler__tests__handle_sync_success.snap | 7 + memfaultd/src/metrics/core_metrics.rs | 20 + memfaultd/src/metrics/crashfree_interval.rs | 413 +++ memfaultd/src/metrics/metric_reading.rs | 255 ++ memfaultd/src/metrics/metric_report.rs | 337 ++ .../src/metrics/metric_report_manager.rs | 464 +++ memfaultd/src/metrics/metric_string_key.rs | 115 + memfaultd/src/metrics/metric_value.rs | 24 + memfaultd/src/metrics/mod.rs | 55 + .../src/metrics/periodic_metric_report.rs | 131 + .../src/metrics/session_event_handler.rs | 447 +++ memfaultd/src/metrics/session_name.rs | 94 + ...tests__aggregate_core_metrics_session.snap | 15 + ...ric_report__tests__heartbeat_report_1.snap | 9 + ...ric_report__tests__heartbeat_report_2.snap | 7 + ...ric_report__tests__heartbeat_report_3.snap | 7 + ...ric_report__tests__heartbeat_report_4.snap | 7 + ...ric_report__tests__heartbeat_report_5.snap | 7 + ...etric_report__tests__session_report_1.snap | 8 + ...etric_report__tests__session_report_2.snap | 7 + ...etric_report__tests__session_report_3.snap | 7 + ...etric_report__tests__session_report_4.snap | 7 + ...etric_report__tests__session_report_5.snap | 8 + ...eport_manager__tests__daily-heartbeat.snap | 15 + ...tric_report_manager__tests__heartbeat.snap | 15 + ...and_sessions_report_1.daily_heartbeat.snap | 16 + ...tbeat_and_sessions_report_1.heartbeat.snap | 16 + ...ons_report_1.test-session-all-metrics.snap | 19 + ...ns_report_1.test-session-some-metrics.snap | 18 + ...and_sessions_report_2.daily_heartbeat.snap | 14 + ...tbeat_and_sessions_report_2.heartbeat.snap | 14 + ...ons_report_2.test-session-all-metrics.snap | 17 + ...ns_report_2.test-session-some-metrics.snap | 17 + ...and_sessions_report_3.daily_heartbeat.snap | 14 + ...tbeat_and_sessions_report_3.heartbeat.snap | 14 + ...ons_report_3.test-session-all-metrics.snap | 17 + ...ns_report_3.test-session-some-metrics.snap | 17 + ...and_sessions_report_4.daily_heartbeat.snap | 15 + ...tbeat_and_sessions_report_4.heartbeat.snap | 15 + ...ons_report_4.test-session-all-metrics.snap | 18 + ...ns_report_4.test-session-some-metrics.snap | 17 + ...and_sessions_report_5.daily_heartbeat.snap | 15 + ...tbeat_and_sessions_report_5.heartbeat.snap | 15 + ...ons_report_5.test-session-all-metrics.snap | 18 + ...ns_report_5.test-session-some-metrics.snap | 18 + ...rt_manager__tests__heartbeat_report_1.snap | 16 + ...rt_manager__tests__heartbeat_report_2.snap | 14 + ...rt_manager__tests__heartbeat_report_3.snap | 14 + ...rt_manager__tests__heartbeat_report_4.snap | 14 + ...rt_manager__tests__heartbeat_report_5.snap | 14 + ...t_manager__tests__start_session_twice.snap | 18 + ...c_report_manager__tests__test-session.snap | 18 + ...vent_handler__tests__end_with_metrics.snap | 18 + ...event_handler__tests__start_then_stop.snap | 19 + ...sts__start_twice_without_stop_session.snap | 9 + ...nt_handler__tests__start_with_metrics.snap | 10 + ...er__tests__start_without_stop_session.snap | 9 + memfaultd/src/metrics/statsd_server/mod.rs | 126 + ...erver__test__test_counter_aggregation.snap | 7 + ...t__test_counter_and_gauge_aggregation.snap | 8 + ...ver__test__test_histogram_aggregation.snap | 9 + ...ics__statsd_server__test__test_simple.snap | 8 + memfaultd/src/metrics/system_metrics/cpu.rs | 304 ++ .../src/metrics/system_metrics/disk_space.rs | 341 ++ .../src/metrics/system_metrics/memory.rs | 291 ++ memfaultd/src/metrics/system_metrics/mod.rs | 180 ++ .../system_metrics/network_interfaces.rs | 489 +++ .../src/metrics/system_metrics/processes.rs | 526 ++++ ...s__cpu__test__basic_delta_a_b_metrics.snap | 69 + ...s__cpu__test__basic_delta_b_c_metrics.snap | 69 + ...pu__test__different_cores_a_c_metrics.snap | 69 + ...m_metrics__cpu__test__test_basic_line.snap | 16 + ...cpu__test__test_basic_line_with_extra.snap | 16 + ...lize_and_calc_disk_space_for_mounts-2.snap | 42 + ...ialize_and_calc_disk_space_for_mounts.snap | 14 + ...st__unmonitored_disks_not_initialized.snap | 5 + ...ics__memory__test__get_memory_metrics.snap | 24 + ...t__get_memory_metrics_no_memavailable.snap | 24 + ...rfaces__test__basic_delta_a_b_metrics.snap | 78 + ...rfaces__test__basic_delta_b_c_metrics.snap | 78 + ...est__different_interfaces_a_c_metrics.snap | 78 + ...trics__network_interfaces__test__eth0.snap | 25 + ...aces__test__with_overflow_a_b_metrics.snap | 78 + ...aces__test__with_overflow_b_c_metrics.snap | 78 + ...rics__network_interfaces__test__wlan1.snap | 25 + ...es__tests__process_metrics_auto_false.snap | 132 + ...ses__tests__process_metrics_auto_true.snap | 69 + ...sses__tests__simple_cpu_delta_metrics.snap | 69 + .../src/metrics/system_metrics/thermal.rs | 138 + memfaultd/src/metrics/timeseries/mod.rs | 447 +++ memfaultd/src/network/client.rs | 238 ++ memfaultd/src/network/mod.rs | 64 + memfaultd/src/network/requests.rs | 158 + memfaultd/src/reboot/mod.rs | 465 +++ memfaultd/src/reboot/reason.rs | 110 + memfaultd/src/reboot/reason_codes.rs | 85 + memfaultd/src/retriable_error.rs | 65 + memfaultd/src/service_manager/default.rs | 20 + memfaultd/src/service_manager/mod.rs | 65 + memfaultd/src/service_manager/systemd.rs | 42 + memfaultd/src/swupdate/config.rs | 51 + memfaultd/src/swupdate/mod.rs | 6 + memfaultd/src/test_utils.rs | 131 + .../src/test_utils/test_connection_checker.rs | 41 + memfaultd/src/test_utils/test_instant.rs | 82 + memfaultd/src/util/can_connect.rs | 78 + memfaultd/src/util/circular_queue.rs | 121 + memfaultd/src/util/die.rs | 26 + memfaultd/src/util/disk_backed.rs | 221 ++ memfaultd/src/util/disk_size.rs | 176 ++ memfaultd/src/util/etc_os_release.rs | 107 + memfaultd/src/util/fs.rs | 52 + memfaultd/src/util/io.rs | 194 ++ memfaultd/src/util/ipc.rs | 11 + memfaultd/src/util/math.rs | 12 + memfaultd/src/util/mem.rs | 28 + memfaultd/src/util/mod.rs | 31 + memfaultd/src/util/output_arg.rs | 42 + memfaultd/src/util/path.rs | 66 + memfaultd/src/util/patterns.rs | 84 + memfaultd/src/util/persistent_rate_limiter.rs | 146 + memfaultd/src/util/pid_file.rs | 59 + memfaultd/src/util/rate_limiter.rs | 154 + .../util/serialization/datetime_to_rfc3339.rs | 24 + .../util/serialization/float_to_datetime.rs | 30 + .../util/serialization/float_to_duration.rs | 23 + .../src/util/serialization/kib_to_usize.rs | 45 + .../serialization/milliseconds_to_duration.rs | 21 + memfaultd/src/util/serialization/mod.rs | 12 + .../serialization/number_to_compression.rs | 25 + .../optional_milliseconds_to_duration.rs | 28 + .../util/serialization/seconds_to_duration.rs | 21 + .../src/util/serialization/sorted_map.rs | 15 + memfaultd/src/util/string.rs | 94 + memfaultd/src/util/system.rs | 94 + memfaultd/src/util/task.rs | 263 ++ memfaultd/src/util/tcp_server.rs | 86 + memfaultd/src/util/time_measure.rs | 27 + memfaultd/src/util/zip.rs | 487 +++ 346 files changed, 36815 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 Cross.toml create mode 100644 License.txt create mode 100644 README.md create mode 100644 VERSION create mode 100644 libmemfaultc/.clang-format create mode 100644 libmemfaultc/.gitignore create mode 100644 libmemfaultc/src/crash.c create mode 100644 libmemfaultc/src/swupdate.c create mode 100644 libmemfaultc/src/systemd.c create mode 100644 memfaultc-sys/Cargo.toml create mode 100644 memfaultc-sys/build.rs create mode 100644 memfaultc-sys/src/coredump.rs create mode 100644 memfaultc-sys/src/lib.rs create mode 100644 memfaultc-sys/src/swupdate.rs create mode 100644 memfaultc-sys/src/systemd.rs create mode 100644 memfaultc-sys/src/systemd_mock.rs create mode 100644 memfaultd.init create mode 100644 memfaultd.service create mode 100644 memfaultd/Cargo.toml create mode 100644 memfaultd/DEVELOPMENT.md create mode 100644 memfaultd/build.rs create mode 100644 memfaultd/builtin.conf create mode 100644 memfaultd/memfaultd.conf create mode 100644 memfaultd/src/bin/memfault-core-handler.rs create mode 100644 memfaultd/src/bin/memfaultctl.rs create mode 100644 memfaultd/src/bin/memfaultd.rs create mode 100644 memfaultd/src/bin/mfw.rs create mode 100644 memfaultd/src/cli/cargs.rs create mode 100644 memfaultd/src/cli/memfault_core_handler/arch.rs create mode 100644 memfaultd/src/cli/memfault_core_handler/auxv.rs create mode 100644 memfaultd/src/cli/memfault_core_handler/core_elf_memfault_note.rs create mode 100644 memfaultd/src/cli/memfault_core_handler/core_elf_note.rs create mode 100644 memfaultd/src/cli/memfault_core_handler/core_reader.rs create mode 100644 memfaultd/src/cli/memfault_core_handler/core_transformer.rs create mode 100644 memfaultd/src/cli/memfault_core_handler/core_writer.rs create mode 100644 memfaultd/src/cli/memfault_core_handler/find_dynamic.rs create mode 100644 memfaultd/src/cli/memfault_core_handler/find_elf_headers.rs create mode 100644 memfaultd/src/cli/memfault_core_handler/find_stack.rs create mode 100644 memfaultd/src/cli/memfault_core_handler/fixtures/elf-core-runtime-ld-paths.elf create mode 100644 memfaultd/src/cli/memfault_core_handler/fixtures/sample_note.bin create mode 100644 memfaultd/src/cli/memfault_core_handler/log_wrapper.rs create mode 100644 memfaultd/src/cli/memfault_core_handler/memory_range.rs create mode 100644 memfaultd/src/cli/memfault_core_handler/mod.rs create mode 100644 memfaultd/src/cli/memfault_core_handler/procfs.rs create mode 100644 memfaultd/src/cli/memfault_core_handler/r_debug.rs create mode 100644 memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_debug_data.snap create mode 100644 memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_metadata_as_map@app_logs.snap create mode 100644 memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_metadata_as_map@kernel_selection.snap create mode 100644 memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_metadata_as_map@threads.snap create mode 100644 memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_note__test__iterate_elf_notes_with_fixture.snap create mode 100644 memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_transformer__test__transform@kernel_selection.snap create mode 100644 memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_transformer__test__transform@threads_32k.snap create mode 100644 memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_transformer__test__transform@threads_32k_no_filter_support.snap create mode 100644 memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__find_dynamic__test__find_dynamic_linker_ranges.snap create mode 100644 memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__log_wrapper__test__log_saving.snap create mode 100644 memfaultd/src/cli/memfault_core_handler/test_utils.rs create mode 100644 memfaultd/src/cli/memfault_watch/buffer.rs create mode 100644 memfaultd/src/cli/memfault_watch/mod.rs create mode 100644 memfaultd/src/cli/memfaultctl/add_battery_reading.rs create mode 100644 memfaultd/src/cli/memfaultctl/config_file.rs create mode 100644 memfaultd/src/cli/memfaultctl/coredump.rs create mode 100644 memfaultd/src/cli/memfaultctl/export.rs create mode 100644 memfaultd/src/cli/memfaultctl/mod.rs create mode 100644 memfaultd/src/cli/memfaultctl/report_sync.rs create mode 100644 memfaultd/src/cli/memfaultctl/session.rs create mode 100644 memfaultd/src/cli/memfaultctl/sync.rs create mode 100644 memfaultd/src/cli/memfaultctl/write_attributes.rs create mode 100644 memfaultd/src/cli/memfaultd.rs create mode 100644 memfaultd/src/cli/memfaultd_client.rs create mode 100644 memfaultd/src/cli/mod.rs create mode 100644 memfaultd/src/cli/show_settings.rs create mode 100644 memfaultd/src/cli/snapshots/memfaultd__cli__show_settings__tests__test.snap create mode 100644 memfaultd/src/cli/version.rs create mode 100644 memfaultd/src/collectd/collectd_handler.rs create mode 100644 memfaultd/src/collectd/fixtures/sample-with-null.json create mode 100644 memfaultd/src/collectd/fixtures/sample1.json create mode 100644 memfaultd/src/collectd/fixtures/statsd-counter-first-seen.json create mode 100644 memfaultd/src/collectd/fixtures/statsd-counter.json create mode 100644 memfaultd/src/collectd/mod.rs create mode 100644 memfaultd/src/collectd/payload.rs create mode 100644 memfaultd/src/collectd/snapshots/memfaultd__collectd__collectd_handler__tests__drops_cpu_metrics_when_builtin_system_metrics_are_enabled.snap create mode 100644 memfaultd/src/collectd/snapshots/memfaultd__collectd__collectd_handler__tests__handle_push.snap create mode 100644 memfaultd/src/collectd/snapshots/memfaultd__collectd__collectd_handler__tests__ignores_data_when_data_collection_is_off.snap create mode 100644 memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__sample-with-null.snap create mode 100644 memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__sample1.snap create mode 100644 memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__statsd-counter-first-seen.snap create mode 100644 memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__statsd-counter.snap create mode 100644 memfaultd/src/config/config_file.rs create mode 100644 memfaultd/src/config/device_config.rs create mode 100644 memfaultd/src/config/device_info.rs create mode 100644 memfaultd/src/config/mod.rs create mode 100644 memfaultd/src/config/snapshots/memfaultd__config__config_file__test__empty_object.snap create mode 100644 memfaultd/src/config/snapshots/memfaultd__config__config_file__test__metrics_config.snap create mode 100644 memfaultd/src/config/snapshots/memfaultd__config__config_file__test__set_and_write_bool_to_runtime_config@empty_object.snap create mode 100644 memfaultd/src/config/snapshots/memfaultd__config__config_file__test__set_and_write_bool_to_runtime_config@no_file.snap create mode 100644 memfaultd/src/config/snapshots/memfaultd__config__config_file__test__set_and_write_bool_to_runtime_config@other_key.snap create mode 100644 memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_connectivity_monitor.snap create mode 100644 memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_coredump_capture_strategy_threads.snap create mode 100644 memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_log_to_metrics_rules.snap create mode 100644 memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_partial_logs.snap create mode 100644 memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_sessions.snap create mode 100644 memfaultd/src/config/snapshots/memfaultd__config__config_file__test__without_coredump_compression.snap create mode 100644 memfaultd/src/config/test-config/empty_object.json create mode 100644 memfaultd/src/config/test-config/metrics_config.json create mode 100644 memfaultd/src/config/test-config/with_connectivity_monitor.json create mode 100644 memfaultd/src/config/test-config/with_coredump_capture_strategy_threads.json create mode 100644 memfaultd/src/config/test-config/with_invalid_path.json create mode 100644 memfaultd/src/config/test-config/with_invalid_swt_swv.json create mode 100644 memfaultd/src/config/test-config/with_log_to_metrics_rules.json create mode 100644 memfaultd/src/config/test-config/with_partial_logs.json create mode 100644 memfaultd/src/config/test-config/with_sessions.json create mode 100644 memfaultd/src/config/test-config/with_sessions_invalid_metric_name.json create mode 100644 memfaultd/src/config/test-config/with_sessions_invalid_session_name.json create mode 100644 memfaultd/src/config/test-config/without_coredump_compression.json create mode 100644 memfaultd/src/config/utils.rs create mode 100644 memfaultd/src/coredump/mod.rs create mode 100644 memfaultd/src/fluent_bit/decode_time.rs create mode 100644 memfaultd/src/fluent_bit/mod.rs create mode 100644 memfaultd/src/http_server/handler.rs create mode 100644 memfaultd/src/http_server/mod.rs create mode 100644 memfaultd/src/http_server/request_bodies.rs create mode 100644 memfaultd/src/http_server/server.rs create mode 100644 memfaultd/src/http_server/snapshots/memfaultd__http_server__response__tests__response_error.snap create mode 100644 memfaultd/src/http_server/snapshots/memfaultd__http_server__response__tests__response_not_found.snap create mode 100644 memfaultd/src/http_server/snapshots/memfaultd__http_server__response__tests__response_ok.snap create mode 100644 memfaultd/src/http_server/utils.rs create mode 100644 memfaultd/src/lib.rs create mode 100644 memfaultd/src/logs/completed_log.rs create mode 100644 memfaultd/src/logs/fluent_bit_adapter.rs create mode 100644 memfaultd/src/logs/headroom.rs create mode 100644 memfaultd/src/logs/journald_parser.rs create mode 100644 memfaultd/src/logs/journald_provider.rs create mode 100644 memfaultd/src/logs/log_collector.rs create mode 100644 memfaultd/src/logs/log_entry.rs create mode 100644 memfaultd/src/logs/log_file.rs create mode 100644 memfaultd/src/logs/log_to_metrics.rs create mode 100644 memfaultd/src/logs/mod.rs create mode 100644 memfaultd/src/logs/recovery.rs create mode 100644 memfaultd/src/logs/snapshots/memfaultd__logs__fluent_bit_adapter__tests__fluent_bit_adapter.snap create mode 100644 memfaultd/src/logs/snapshots/memfaultd__logs__journald_parser__test__from_raw_journal_entry.snap create mode 100644 memfaultd/src/logs/snapshots/memfaultd__logs__journald_parser__test__journal_happy_path.snap create mode 100644 memfaultd/src/logs/snapshots/memfaultd__logs__journald_provider__test__happy_path.snap create mode 100644 memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__empty.snap create mode 100644 memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__extra_attribute_filter.snap create mode 100644 memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__extra_key.snap create mode 100644 memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__multi_key_match.snap create mode 100644 memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__only_message.snap create mode 100644 memfaultd/src/mar/chunks.rs create mode 100644 memfaultd/src/mar/chunks/chunk.rs create mode 100644 memfaultd/src/mar/chunks/chunk_header.rs create mode 100644 memfaultd/src/mar/chunks/chunk_message.rs create mode 100644 memfaultd/src/mar/chunks/chunk_wrapper.rs create mode 100644 memfaultd/src/mar/chunks/crc_padded_stream.rs create mode 100644 memfaultd/src/mar/clean.rs create mode 100644 memfaultd/src/mar/export.rs create mode 100644 memfaultd/src/mar/export_format.rs create mode 100644 memfaultd/src/mar/manifest.rs create mode 100644 memfaultd/src/mar/mar_entry.rs create mode 100644 memfaultd/src/mar/mar_entry_builder.rs create mode 100644 memfaultd/src/mar/mod.rs create mode 100644 memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__attributes.snap create mode 100644 memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__coredump-gzip.snap create mode 100644 memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__coredump-none.snap create mode 100644 memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__device_config.snap create mode 100644 memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__elf_coredump.snap create mode 100644 memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__heartbeat.snap create mode 100644 memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__heartbeat_with_duration.snap create mode 100644 memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__log-none.snap create mode 100644 memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__log-zlib.snap create mode 100644 memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__metric_report.snap create mode 100644 memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__reboot.snap create mode 100644 memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_custom_reboot.snap create mode 100644 memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_custom_unexpected_reboot.snap create mode 100644 memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_device_attributes.snap create mode 100644 memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_device_configc.snap create mode 100644 memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_linux_heartbeat.snap create mode 100644 memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_reboot.snap create mode 100644 memfaultd/src/mar/test-manifests/attributes.json create mode 100644 memfaultd/src/mar/test-manifests/device_config.json create mode 100644 memfaultd/src/mar/test-manifests/elf_coredump.json create mode 100644 memfaultd/src/mar/test-manifests/heartbeat.json create mode 100644 memfaultd/src/mar/test-manifests/heartbeat_with_duration.json create mode 100644 memfaultd/src/mar/test-manifests/log.json create mode 100644 memfaultd/src/mar/test-manifests/metric_report.json create mode 100644 memfaultd/src/mar/test-manifests/reboot.json create mode 100644 memfaultd/src/mar/test_utils.rs create mode 100644 memfaultd/src/mar/upload.rs create mode 100644 memfaultd/src/memfaultd.rs create mode 100644 memfaultd/src/metrics/battery/battery_monitor.rs create mode 100644 memfaultd/src/metrics/battery/battery_reading_handler.rs create mode 100644 memfaultd/src/metrics/battery/mod.rs create mode 100644 memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__charging_then_discharging.snap create mode 100644 memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__handle_push.snap create mode 100644 memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__ignores_data_when_data_collection_is_off.snap create mode 100644 memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__non_integer_percentages.snap create mode 100644 memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__nonconsecutive_discharges.snap create mode 100644 memfaultd/src/metrics/connectivity/connectivity_monitor.rs create mode 100644 memfaultd/src/metrics/connectivity/mod.rs create mode 100644 memfaultd/src/metrics/connectivity/report_sync_event_handler.rs create mode 100644 memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__connectivity_monitor__tests__fully_disconnected.snap create mode 100644 memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__connectivity_monitor__tests__half_connected_half_disconnected.snap create mode 100644 memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__connectivity_monitor__tests__while_connected.snap create mode 100644 memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__report_sync_event_handler__tests__handle_multiple_sync_events.snap create mode 100644 memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__report_sync_event_handler__tests__handle_sync_failure.snap create mode 100644 memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__report_sync_event_handler__tests__handle_sync_success.snap create mode 100644 memfaultd/src/metrics/core_metrics.rs create mode 100644 memfaultd/src/metrics/crashfree_interval.rs create mode 100644 memfaultd/src/metrics/metric_reading.rs create mode 100644 memfaultd/src/metrics/metric_report.rs create mode 100644 memfaultd/src/metrics/metric_report_manager.rs create mode 100644 memfaultd/src/metrics/metric_string_key.rs create mode 100644 memfaultd/src/metrics/metric_value.rs create mode 100644 memfaultd/src/metrics/mod.rs create mode 100644 memfaultd/src/metrics/periodic_metric_report.rs create mode 100644 memfaultd/src/metrics/session_event_handler.rs create mode 100644 memfaultd/src/metrics/session_name.rs create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__aggregate_core_metrics_session.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_1.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_2.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_3.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_4.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_5.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_1.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_2.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_3.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_4.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_5.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__daily-heartbeat.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.daily_heartbeat.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.heartbeat.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.test-session-all-metrics.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.test-session-some-metrics.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.daily_heartbeat.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.heartbeat.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.test-session-all-metrics.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.test-session-some-metrics.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.daily_heartbeat.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.heartbeat.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.test-session-all-metrics.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.test-session-some-metrics.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.daily_heartbeat.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.heartbeat.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.test-session-all-metrics.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.test-session-some-metrics.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.daily_heartbeat.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.heartbeat.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.test-session-all-metrics.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.test-session-some-metrics.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_1.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_2.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_3.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_4.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_5.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__start_session_twice.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__test-session.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__end_with_metrics.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_then_stop.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_twice_without_stop_session.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_with_metrics.snap create mode 100644 memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_without_stop_session.snap create mode 100644 memfaultd/src/metrics/statsd_server/mod.rs create mode 100644 memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_counter_aggregation.snap create mode 100644 memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_counter_and_gauge_aggregation.snap create mode 100644 memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_histogram_aggregation.snap create mode 100644 memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_simple.snap create mode 100644 memfaultd/src/metrics/system_metrics/cpu.rs create mode 100644 memfaultd/src/metrics/system_metrics/disk_space.rs create mode 100644 memfaultd/src/metrics/system_metrics/memory.rs create mode 100644 memfaultd/src/metrics/system_metrics/mod.rs create mode 100644 memfaultd/src/metrics/system_metrics/network_interfaces.rs create mode 100644 memfaultd/src/metrics/system_metrics/processes.rs create mode 100644 memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__basic_delta_a_b_metrics.snap create mode 100644 memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__basic_delta_b_c_metrics.snap create mode 100644 memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__different_cores_a_c_metrics.snap create mode 100644 memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__test_basic_line.snap create mode 100644 memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__test_basic_line_with_extra.snap create mode 100644 memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__disk_space__test__initialize_and_calc_disk_space_for_mounts-2.snap create mode 100644 memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__disk_space__test__initialize_and_calc_disk_space_for_mounts.snap create mode 100644 memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__disk_space__test__unmonitored_disks_not_initialized.snap create mode 100644 memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__memory__test__get_memory_metrics.snap create mode 100644 memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__memory__test__get_memory_metrics_no_memavailable.snap create mode 100644 memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__basic_delta_a_b_metrics.snap create mode 100644 memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__basic_delta_b_c_metrics.snap create mode 100644 memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__different_interfaces_a_c_metrics.snap create mode 100644 memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__eth0.snap create mode 100644 memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__with_overflow_a_b_metrics.snap create mode 100644 memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__with_overflow_b_c_metrics.snap create mode 100644 memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__wlan1.snap create mode 100644 memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__processes__tests__process_metrics_auto_false.snap create mode 100644 memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__processes__tests__process_metrics_auto_true.snap create mode 100644 memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__processes__tests__simple_cpu_delta_metrics.snap create mode 100644 memfaultd/src/metrics/system_metrics/thermal.rs create mode 100644 memfaultd/src/metrics/timeseries/mod.rs create mode 100644 memfaultd/src/network/client.rs create mode 100644 memfaultd/src/network/mod.rs create mode 100644 memfaultd/src/network/requests.rs create mode 100644 memfaultd/src/reboot/mod.rs create mode 100644 memfaultd/src/reboot/reason.rs create mode 100644 memfaultd/src/reboot/reason_codes.rs create mode 100644 memfaultd/src/retriable_error.rs create mode 100644 memfaultd/src/service_manager/default.rs create mode 100644 memfaultd/src/service_manager/mod.rs create mode 100644 memfaultd/src/service_manager/systemd.rs create mode 100644 memfaultd/src/swupdate/config.rs create mode 100644 memfaultd/src/swupdate/mod.rs create mode 100644 memfaultd/src/test_utils.rs create mode 100644 memfaultd/src/test_utils/test_connection_checker.rs create mode 100644 memfaultd/src/test_utils/test_instant.rs create mode 100644 memfaultd/src/util/can_connect.rs create mode 100644 memfaultd/src/util/circular_queue.rs create mode 100644 memfaultd/src/util/die.rs create mode 100644 memfaultd/src/util/disk_backed.rs create mode 100644 memfaultd/src/util/disk_size.rs create mode 100644 memfaultd/src/util/etc_os_release.rs create mode 100644 memfaultd/src/util/fs.rs create mode 100644 memfaultd/src/util/io.rs create mode 100644 memfaultd/src/util/ipc.rs create mode 100644 memfaultd/src/util/math.rs create mode 100644 memfaultd/src/util/mem.rs create mode 100644 memfaultd/src/util/mod.rs create mode 100644 memfaultd/src/util/output_arg.rs create mode 100644 memfaultd/src/util/path.rs create mode 100644 memfaultd/src/util/patterns.rs create mode 100644 memfaultd/src/util/persistent_rate_limiter.rs create mode 100644 memfaultd/src/util/pid_file.rs create mode 100644 memfaultd/src/util/rate_limiter.rs create mode 100644 memfaultd/src/util/serialization/datetime_to_rfc3339.rs create mode 100644 memfaultd/src/util/serialization/float_to_datetime.rs create mode 100644 memfaultd/src/util/serialization/float_to_duration.rs create mode 100644 memfaultd/src/util/serialization/kib_to_usize.rs create mode 100644 memfaultd/src/util/serialization/milliseconds_to_duration.rs create mode 100644 memfaultd/src/util/serialization/mod.rs create mode 100644 memfaultd/src/util/serialization/number_to_compression.rs create mode 100644 memfaultd/src/util/serialization/optional_milliseconds_to_duration.rs create mode 100644 memfaultd/src/util/serialization/seconds_to_duration.rs create mode 100644 memfaultd/src/util/serialization/sorted_map.rs create mode 100644 memfaultd/src/util/string.rs create mode 100644 memfaultd/src/util/system.rs create mode 100644 memfaultd/src/util/task.rs create mode 100644 memfaultd/src/util/tcp_server.rs create mode 100644 memfaultd/src/util/time_measure.rs create mode 100644 memfaultd/src/util/zip.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..eb5a316 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +target diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..78af428 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2759 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "argh" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab257697eb9496bf75526f0217b5ed64636a9cfafa78b8365c71bd283fcef93e" +dependencies = [ + "argh_derive", + "argh_shared", +] + +[[package]] +name = "argh_derive" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b382dbd3288e053331f03399e1db106c9fb0d8562ad62cb04859ae926f324fa6" +dependencies = [ + "argh_shared", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "argh_shared" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cb94155d965e3d37ffbbe7cc5b82c3dd79dd33bd48e536f73d2cfb8d85506f" + +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cc" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32a725bc159af97c3e629873bb9f88fb8cf8a4867175f76dc987815ea07c83b" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-integer", + "num-traits", + "serde", + "time", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "chunked_transfer" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cca491388666e04d7248af3f60f0c40cfb0991c72205595d7c396e3510207d1a" + +[[package]] +name = "ciborium" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" + +[[package]] +name = "ciborium-ll" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "console" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "windows-sys 0.42.0", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "cpufeatures" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "cxx" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f61f1b6389c3fe1c316bf8a4dccc90a38208354b330925bce1f74a6c4756eb93" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cee708e8962df2aeb38f594aae5d827c022b6460ac71a7a3e2c3c2aae5a07b" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn 2.0.52", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7944172ae7e4068c533afbb984114a56c46e9ccddda550499caa222902c7f7bb" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "digest" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" + +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "eyre" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c2b6b5a29c02cdc822728b7d7b8ae1bab3e3b05d44522770ddd49722eeac7eb" +dependencies = [ + "indenter", + "once_cell", +] + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "flate2" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +dependencies = [ + "crc32fast", + "libz-sys", + "miniz_oxide", +] + +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fragile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-executor" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-timer" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "goblin" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27c1b4369c2cd341b5de549380158b105a04c331be5db9110eef7b6d2742134" +dependencies = [ + "log", + "plain", + "scroll", +] + +[[package]] +name = "governor" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c390a940a5d157878dd057c78680a33ce3415bcd05b4799509ea44210914b4d5" +dependencies = [ + "cfg-if", + "futures", + "futures-timer", + "no-std-compat", + "nonzero_ext", + "parking_lot", + "smallvec", +] + +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http", + "hyper", + "rustls", + "tokio", + "tokio-rustls", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indenter" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" + +[[package]] +name = "indexmap" +version = "2.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "insta" +version = "1.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a28d25139df397cbca21408bb742cf6837e04cdbebf1b07b760caf971d6a972" +dependencies = [ + "console", + "lazy_static", + "linked-hash-map", + "pest", + "pest_derive", + "serde", + "similar", + "yaml-rust", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi 0.3.9", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" + +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "kernlog" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0dd79b6ea06a97c93bc5587fba4c2ed6d723b939d35d5e3fb3c6870d12e6d17" +dependencies = [ + "libc", + "log", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.153" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "libz-sys" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "link-cplusplus" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" +dependencies = [ + "cc", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "linux-raw-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" + +[[package]] +name = "linux-raw-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" + +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" + +[[package]] +name = "memchr" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" + +[[package]] +name = "memfaultc-sys" +version = "0.1.0" +dependencies = [ + "cc", + "libc", + "pkg-config", + "walkdir", +] + +[[package]] +name = "memfaultd" +version = "0.1.0" +dependencies = [ + "argh", + "cfg-if", + "chrono", + "ciborium", + "crc", + "crc-catalog", + "eyre", + "flate2", + "fs_extra", + "goblin", + "governor", + "hex", + "insta", + "itertools", + "kernlog", + "libc", + "log", + "memfaultc-sys", + "mockall", + "nix", + "nom", + "once_cell", + "prctl", + "procfs", + "psm", + "rand", + "regex", + "reqwest", + "rmp-serde", + "rmpv", + "rstest", + "scroll", + "sealed_test", + "serde", + "serde_bytes", + "serde_json", + "serde_repr", + "shuteye", + "signal-hook", + "stderrlog", + "strum", + "strum_macros", + "take_mut", + "tempfile", + "thiserror", + "threadpool", + "tiny_http", + "urlencoding", + "uuid", + "zip", +] + +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.48.0", +] + +[[package]] +name = "mockall" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c84490118f2ee2d74570d114f3d0493cbf02790df303d2707606c3e14e07c96" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nix" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", + "memoffset", + "pin-utils", + "static_assertions", +] + +[[package]] +name = "no-std-compat" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nonzero_ext" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +dependencies = [ + "hermit-abi 0.2.6", + "libc", +] + +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "openssl" +version = "0.10.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +dependencies = [ + "bitflags 2.4.2", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-src" +version = "300.2.3+3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cff92b6f71555b61bb9315f7c64da3ca43d87531622120fea0195fc761b4843" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff" +dependencies = [ + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys 0.45.0", +] + +[[package]] +name = "paste" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pest" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e68e84bfb01f0507134eac1e9b410a12ba379d064eab48c50ba4ce329a527b70" +dependencies = [ + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b79d4c71c865a25a4322296122e3924d30bc8ee0834c8bfc8b95f7f054afbfb" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c435bf1076437b851ebc8edc3a18442796b30f1728ffea6262d59bbe28b077e" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.52", +] + +[[package]] +name = "pest_meta" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "745a452f8eb71e39ffd8ee32b3c5f51d03845f99786fa9b68db6ff509c505411" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "prctl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "059a34f111a9dee2ce1ac2826a68b24601c4298cfeb1a587c3cb493d5ab46f52" +dependencies = [ + "libc", + "nix", +] + +[[package]] +name = "predicates" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" +dependencies = [ + "difflib", + "float-cmp", + "itertools", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" + +[[package]] +name = "predicates-tree" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "proc-macro2" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "procfs" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "943ca7f9f29bab5844ecd8fdb3992c5969b6622bb9609b9502fef9b4310e3f1f" +dependencies = [ + "bitflags 1.3.2", + "byteorder", + "chrono", + "flate2", + "hex", + "lazy_static", + "rustix 0.36.16", +] + +[[package]] +name = "psm" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" +dependencies = [ + "cc", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "regex" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + +[[package]] +name = "reqwest" +version = "0.11.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78bf93c4af7a8bb7d879d51cebe797356ff10ae8516ace542b5182d9dcac10b2" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-rustls", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-rustls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", + "winreg", +] + +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rmp" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44519172358fd6d58656c86ab8e7fbc9e1490c3e8f14d35ed78ca0dd07403c9f" +dependencies = [ + "byteorder", + "num-traits", + "paste", +] + +[[package]] +name = "rmp-serde" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5b13be192e0220b8afb7222aa5813cb62cc269ebb5cac346ca6487681d2913e" +dependencies = [ + "byteorder", + "rmp", + "serde", +] + +[[package]] +name = "rmpv" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de8813b3a2f95c5138fe5925bfb8784175d88d6bff059ba8ce090aa891319754" +dependencies = [ + "num-traits", + "rmp", +] + +[[package]] +name = "rstest" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b07f2d176c472198ec1e6551dc7da28f1c089652f66a7b722676c2238ebc0edf" +dependencies = [ + "futures", + "futures-timer", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7229b505ae0706e64f37ffc54a9c163e11022a6636d58fe1f3f52018257ff9f7" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "rustc_version", + "syn 1.0.109", + "unicode-ident", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.36.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6da3636faa25820d8648e0e31c5d519bbb01f72fdf57131f0f5f7da5fed36eab" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys 0.1.4", + "windows-sys 0.45.0", +] + +[[package]] +name = "rustix" +version = "0.38.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +dependencies = [ + "bitflags 2.4.2", + "errno", + "libc", + "linux-raw-sys 0.4.13", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.21.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" + +[[package]] +name = "rusty-forkfork" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce85af4dfa2fb0c0143121ab5e424c71ea693867357c9159b8777b59984c218" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "ryu" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "scratch" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" + +[[package]] +name = "scroll" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04c565b551bafbef4157586fa379538366e4385d42082f255bfd96e4fe8519da" +dependencies = [ + "scroll_derive", +] + +[[package]] +name = "scroll_derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "sealed_test" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a1867f8f005bd7fb73c367e2e45dd628417906a2ca27597fe59cbf04279a222" +dependencies = [ + "fs_extra", + "rusty-forkfork", + "sealed_test_derive", + "tempfile", +] + +[[package]] +name = "sealed_test_derive" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77253fb2d4451418d07025826028bcb96ee42d3e58859689a70ce62908009db6" +dependencies = [ + "quote", + "syn 2.0.52", +] + +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" + +[[package]] +name = "serde" +version = "1.0.197" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "416bda436f9aab92e02c8e10d49a15ddd339cea90b6e340fe51ed97abb548294" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.197" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + +[[package]] +name = "serde_json" +version = "1.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_repr" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcec881020c684085e55a25f7fd888954d56609ef363479dc5a1305eb0d40cab" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shuteye" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ca27229f5eb21ea854a551c24b268c824949a01c41652b6d423c141539252cf" +dependencies = [ + "libc", +] + +[[package]] +name = "signal-hook" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "732768f1176d21d09e076c23a93123d40bba92d50c4058da34d45c8de8e682b9" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "similar" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf" + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "socket2" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "stderrlog" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69a26bbf6de627d389164afa9783739b56746c6c72c4ed16539f4ff54170327b" +dependencies = [ + "atty", + "chrono", + "log", + "termcolor", + "thread_local", +] + +[[package]] +name = "strum" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 1.0.109", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" + +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if", + "fastrand", + "rustix 0.38.31", + "windows-sys 0.52.0", +] + +[[package]] +name = "termcolor" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "termtree" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" + +[[package]] +name = "thiserror" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + +[[package]] +name = "thread_local" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "time" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + +[[package]] +name = "tiny_http" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82" +dependencies = [ + "ascii", + "chunked_transfer", + "httpdate", + "log", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" + +[[package]] +name = "ucd-trie" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" + +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "urlencoding" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" + +[[package]] +name = "uuid" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dad5567ad0cf5b760e5665964bec1b47dfd077ba8a2544b513f3556d3d239a2" +dependencies = [ + "getrandom", + "serde", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.52", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + +[[package]] +name = "web-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.4", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "zip" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0445d0fbc924bb93539b4316c11afb121ea39296f99a3c4c9edad09e3658cdef" +dependencies = [ + "byteorder", + "crc32fast", + "crossbeam-utils", + "flate2", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..3e6570b --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,15 @@ +[workspace] + +members = [ + "memfaultd", + "memfaultc-sys" +] +resolver = "2" + +[profile.release] +# We do not handle FFI unwinding so we need to keep panic abort for now. +panic="abort" +opt-level = "z" + +[profile.dev] +panic="abort" diff --git a/Cross.toml b/Cross.toml new file mode 100644 index 0000000..5229193 --- /dev/null +++ b/Cross.toml @@ -0,0 +1,13 @@ +[target.aarch64-unknown-linux-gnu] +pre-build = [ + "dpkg --add-architecture $CROSS_DEB_ARCH", + "apt-get update", + "apt-get install --assume-yes libsystemd-dev:$CROSS_DEB_ARCH libconfig-dev:$CROSS_DEB_ARCH" +] + +[target.x86_64-unknown-linux-gnu] +pre-build = [ + "dpkg --add-architecture $CROSS_DEB_ARCH", + "apt-get update", + "apt-get install --assume-yes libsystemd-dev:$CROSS_DEB_ARCH libconfig-dev:$CROSS_DEB_ARCH" +] diff --git a/License.txt b/License.txt new file mode 100644 index 0000000..c863fb1 --- /dev/null +++ b/License.txt @@ -0,0 +1,31 @@ +Copyright (c) 2019 - Present, Memfault +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code or in binary form must reproduce +the above copyright notice, this list of conditions and the following +disclaimer in the documentation and/or other materials provided with the +distribution. + +2. Neither the name of Memfault nor the names of its contributors may be +used to endorse or promote products derived from this software without +specific prior written permission. + +3. This software, with or without modification, must only be used with +the Memfault services and integrated with the Memfault server. + +4. Any software provided in binary form under this license must not be +reverse engineered, decompiled, modified and/or disassembled. + +THIS SOFTWARE IS PROVIDED BY MEMFAULT "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, +NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT +SHALL MEMFAULT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..0399447 --- /dev/null +++ b/README.md @@ -0,0 +1,37 @@ +# `memfaultd` + +`memfaultd` is a daemon that runs on your device and collects crash reports and +metrics. It is the core of the +[Memfault Linux SDK](https://github.com/memfault/memfault-linux-sdk/blob/kirkstone/README.md). + +## Overview + +`memfaultd` supports several features to help you maintain and debug your fleet +of devices: + +- **Crash reporting**: When your device crashes, `memfaultd` collects a crash + report from + [Linux Coredumps](https://man7.org/linux/man-pages/man5/core.5.html). For more + information, see the + [Coredumps documentation](https://docs.memfault.com/docs/linux/coredumps). + +- **Metrics**: `memfaultd` collects metrics from your device and uploads them to + Memfault. For more information, see the + [Metrics documentation](https://docs.memfault.com/docs/linux/metrics). + +- **Reboot reason tracking**: `memfaultd` detects various reboot reasons from + the system and reports them to the Memfault Dashboard. Users can also provide + a specific reboot reason before restarting the device. For more information, + see the + [Reboot Reason Tracking documentation](https://docs.memfault.com/docs/linux/reboot-reason-tracking). + +- **OTA Updates**: `memfaultd` supports [SWUpdate](https://swupdate.org/) out of + the box and is able to configure it to talk to our hawkBit DDI-compatible + endpoint. For more information, see the + [Linux OTA Management documentation](https://docs.memfault.com/docs/linux/ota). + +- **Logging**: `memfaultd` collects logs from your system. For more information, + see the [Logging documentation](https://docs.memfault.com/docs/linux/logging). + +And much more! [Register](https://app.memfault.com/register) and get started +today! diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..295e6ae --- /dev/null +++ b/VERSION @@ -0,0 +1,3 @@ +BUILD ID: 2255289 +GIT COMMIT: add51b077b +VERSION: 1.13.0 diff --git a/libmemfaultc/.clang-format b/libmemfaultc/.clang-format new file mode 100644 index 0000000..0f374d7 --- /dev/null +++ b/libmemfaultc/.clang-format @@ -0,0 +1,10 @@ +--- +BasedOnStyle: Google +--- +Language: Cpp + +AlwaysBreakBeforeMultilineStrings: false +ColumnLimit: '100' +ContinuationIndentWidth: 2 +IndentPPDirectives: BeforeHash +... diff --git a/libmemfaultc/.gitignore b/libmemfaultc/.gitignore new file mode 100644 index 0000000..65af97a --- /dev/null +++ b/libmemfaultc/.gitignore @@ -0,0 +1,2 @@ +/build +/.cache diff --git a/libmemfaultc/src/crash.c b/libmemfaultc/src/crash.c new file mode 100644 index 0000000..1afeb00 --- /dev/null +++ b/libmemfaultc/src/crash.c @@ -0,0 +1,15 @@ +//! @file +//! +//! Copyright (c) Memfault, Inc. +//! See License.txt for details +//! +//! @brief +//! + +#include + +void memfault_trigger_fp_exception(void) { + int divisor = 0; + // Triggers an illegal instruction on x86 - Floating Point Error on ARM + printf("%i", 42 / divisor); +} diff --git a/libmemfaultc/src/swupdate.c b/libmemfaultc/src/swupdate.c new file mode 100644 index 0000000..da13cc4 --- /dev/null +++ b/libmemfaultc/src/swupdate.c @@ -0,0 +1,215 @@ +//! @file +//! +//! Copyright (c) Memfault, Inc. +//! See License.txt for details +//! +//! @brief +//! Memfault SWUpdate config file generation + +#include +#include +#include +#include +#include + +#define DEFAULT_SURICATTA_TENANT "default" + +#define HAWKBIT_PATH "/api/v0/hawkbit" + +typedef struct { + char *base_url; + char *software_version; + char *software_type; + char *hardware_version; + char *device_id; + char *project_key; + + char *input_file; + char *output_file; +} sMemfaultdSwupdateConfig; + +/** + * @brief Add 'global' section to config + * + * @param config config object to build into + * @return true Successfully added global options to config + * @return false Failed to add + */ +static bool prv_swupdate_add_globals(config_t *config) { + if (!config_lookup(config, "globals")) { + if (!config_setting_add(config_root_setting(config), "globals", CONFIG_TYPE_GROUP)) { + fprintf(stderr, "swupdate:: Failed to add globals setting group\n"); + return false; + } + } + return true; +} + +/** + * @brief Add 'suricatta' section to config + * + * @param handle swupdate config handle + * @param config config object to build into + * @return true Successfully added suricatta options to config + * @return false Failed to add + */ +static bool prv_swupdate_add_suricatta(sMemfaultdSwupdateConfig *handle, config_t *config) { + config_setting_t *suricatta = config_lookup(config, "suricatta"); + if (!suricatta) { + if (!(suricatta = + config_setting_add(config_root_setting(config), "suricatta", CONFIG_TYPE_GROUP))) { + fprintf(stderr, "swupdate:: Failed to add suricatta group\n"); + return false; + } + } + + char *url = malloc(strlen(HAWKBIT_PATH) + strlen(handle->base_url) + 1); + strcpy(url, handle->base_url); + strcat(url, HAWKBIT_PATH); + + config_setting_t *element; + config_setting_remove(suricatta, "url"); + if (!(element = config_setting_add(suricatta, "url", CONFIG_TYPE_STRING)) || + !config_setting_set_string(element, url)) { + fprintf(stderr, "swupdate:: Failed to add suricatta:url\n"); + free(url); + return false; + } + + free(url); + + config_setting_remove(suricatta, "id"); + if (!(element = config_setting_add(suricatta, "id", CONFIG_TYPE_STRING)) || + !config_setting_set_string(element, handle->device_id)) { + fprintf(stderr, "swupdate:: Failed to add suricatta:id\n"); + return false; + } + + config_setting_remove(suricatta, "tenant"); + if (!(element = config_setting_add(suricatta, "tenant", CONFIG_TYPE_STRING)) || + !config_setting_set_string(element, DEFAULT_SURICATTA_TENANT)) { + fprintf(stderr, "swupdate:: Failed to add suricatta:tenant\n"); + return false; + } + + config_setting_remove(suricatta, "gatewaytoken"); + if (!(element = config_setting_add(suricatta, "gatewaytoken", CONFIG_TYPE_STRING)) || + !config_setting_set_string(element, handle->project_key)) { + fprintf(stderr, "swupdate:: Failed to add suricatta:id\n"); + return false; + } + + return true; +} + +/** + * @brief Add 'identify' section to config + * + * @param handle swupdate config handle + * @param config config object to build into + * @return true Successfully added identify options to config + * @return false Failed to add + */ +static bool prv_swupdate_add_identify(sMemfaultdSwupdateConfig *handle, config_t *config) { + config_setting_t *identify; + + config_setting_remove(config_root_setting(config), "identify"); + if (!(identify = config_setting_add(config_root_setting(config), "identify", CONFIG_TYPE_LIST))) { + fprintf(stderr, "swupdate:: Failed to add identify list\n"); + return false; + } + + config_setting_t *setting; + config_setting_t *element; + if (!(setting = config_setting_add(identify, NULL, CONFIG_TYPE_GROUP))) { + fprintf(stderr, "swupdate:: Failed to add identify current_version\n"); + return false; + } + if (!(element = config_setting_add(setting, "name", CONFIG_TYPE_STRING)) || + !config_setting_set_string(element, "memfault__current_version")) { + fprintf(stderr, "swupdate:: Failed to add identify current_version\n"); + return false; + } + if (!(element = config_setting_add(setting, "value", CONFIG_TYPE_STRING)) || + !config_setting_set_string(element, handle->software_version)) { + fprintf(stderr, "swupdate:: Failed to add identify current_version\n"); + return false; + } + + if (!(setting = config_setting_add(identify, NULL, CONFIG_TYPE_GROUP))) { + fprintf(stderr, "swupdate:: Failed to add identify hardware_version\n"); + return false; + } + if (!(element = config_setting_add(setting, "name", CONFIG_TYPE_STRING)) || + !config_setting_set_string(element, "memfault__hardware_version")) { + fprintf(stderr, "swupdate:: Failed to add identify hardware_version\n"); + return false; + } + if (!(element = config_setting_add(setting, "value", CONFIG_TYPE_STRING)) || + !config_setting_set_string(element, handle->hardware_version)) { + fprintf(stderr, "swupdate:: Failed to add identify hardware_version\n"); + return false; + } + + if (!(setting = config_setting_add(identify, NULL, CONFIG_TYPE_GROUP))) { + fprintf(stderr, "swupdate:: Failed to add identify software_type\n"); + return false; + } + if (!(element = config_setting_add(setting, "name", CONFIG_TYPE_STRING)) || + !config_setting_set_string(element, "memfault__software_type")) { + fprintf(stderr, "swupdate:: Failed to add identify software_type\n"); + return false; + } + if (!(element = config_setting_add(setting, "value", CONFIG_TYPE_STRING)) || + !config_setting_set_string(element, handle->software_type)) { + fprintf(stderr, "swupdate:: Failed to add identify software_type\n"); + return false; + } + + return true; +} + +/** + * @brief Generate new swupdate.cfg file from config + * + * @param handle swupdate config handle + * @return true Successfully generated new config + * @return false Failed to generate + */ +bool memfault_swupdate_generate_config(sMemfaultdSwupdateConfig *handle) { + config_t config; + + config_init(&config); + if (!config_read_file(&config, handle->input_file)) { + fprintf(stderr, + "swupdate:: Failed to read '%s', proceeding " + "with defaults\n", + handle->input_file); + } + + if (!prv_swupdate_add_globals(&config)) { + fprintf(stderr, "swupdate:: Failed to add global options to config\n"); + config_destroy(&config); + return false; + } + if (!prv_swupdate_add_suricatta(handle, &config)) { + fprintf(stderr, "swupdate:: Failed to add suricatta options to config\n"); + config_destroy(&config); + return false; + } + if (!prv_swupdate_add_identify(handle, &config)) { + fprintf(stderr, "swupdate:: Failed to add identify options to config\n"); + config_destroy(&config); + return false; + } + + if (!config_write_file(&config, handle->output_file)) { + fprintf(stderr, "swupdate:: Failed to write config file to '%s'\n", handle->output_file); + config_destroy(&config); + return false; + } + + config_destroy(&config); + + return true; +} diff --git a/libmemfaultc/src/systemd.c b/libmemfaultc/src/systemd.c new file mode 100644 index 0000000..6b322f2 --- /dev/null +++ b/libmemfaultc/src/systemd.c @@ -0,0 +1,200 @@ +//! @file +//! +//! Copyright (c) Memfault, Inc. +//! See License.txt for details +//! +//! @brief +//! memfaultd systemd helper + +#include +#include +#include +#include + +static const char *const systemd_service = "org.freedesktop.systemd1"; + +/** + * @param state output variable for current state (to be freed by caller). + */ +static bool prv_systemd_get_service_state(sd_bus *bus, const char *service_name, char **state) { + static const char *const unit_interface = "org.freedesktop.systemd1.Unit"; + + char *unit_path = NULL; + bool result = true; + + if (sd_bus_path_encode("/org/freedesktop/systemd1/unit", service_name, &unit_path) < 0) { + fprintf(stderr, "memfaultd:: Failed to generate SystemD unit path\n"); + + return false; + } + + sd_bus_error error = SD_BUS_ERROR_NULL; + if (sd_bus_get_property_string(bus, systemd_service, unit_path, unit_interface, "ActiveState", + &error, state) < 0) { + fprintf(stderr, "memfaultd:: Failed to get state of %s: %s\n", service_name, error.name); + sd_bus_error_free(&error); + result = false; + } + + if (unit_path) { + free(unit_path); + } + return result; +} + +/* + * List of SystemD commands: https://www.freedesktop.org/wiki/Software/systemd/dbus/ + */ + +static bool prv_systemd_restart_service(sd_bus *bus, const char *service_name) { + const char *manager_path = "/org/freedesktop/systemd1"; + const char *manager_interface = "org.freedesktop.systemd1.Manager"; + + sd_bus_error error = SD_BUS_ERROR_NULL; + sd_bus_message *msg = NULL; + + if (sd_bus_call_method(bus, systemd_service, manager_path, manager_interface, "RestartUnit", + &error, &msg, "ss", service_name, "replace") < 0) { + fprintf(stderr, "memfaultd:: Failed to restart %s: %s\n", service_name, error.name); + sd_bus_error_free(&error); + return false; + } + + sd_bus_message_unref(msg); + + return true; +} + +static bool prv_systemd_kill_service(sd_bus *bus, const char *service_name, int signal) { + const char *manager_path = "/org/freedesktop/systemd1"; + const char *manager_interface = "org.freedesktop.systemd1.Manager"; + + sd_bus_error error = SD_BUS_ERROR_NULL; + sd_bus_message *msg = NULL; + + /* + Refer to SystemD Bus documentation for arguments to the call: + KillUnit(in s name, + in s who, // "all": is the default (like systemctl kill service) + in i signal); + */ + if (sd_bus_call_method(bus, systemd_service, manager_path, manager_interface, "KillUnit", &error, + &msg, "ssi", service_name, "all", signal) < 0) { + fprintf(stderr, "memfaultd:: Failed to kill %s: %s\n", service_name, error.name); + sd_bus_error_free(&error); + return false; + } + + sd_bus_message_unref(msg); + + return true; +} + +/** + * @brief Restart service using systemd dbus API if already running + * + * @param src_module Friendly name of caller module + * @param service_name Service name to restart, e.g. collectd.service + * @return true Successfully restarted requested service + * @return false Failed to restart service + */ +bool memfaultd_restart_systemd_service_if_running(const char *service_name) { + bool result = true; + + // Initialize connection to SystemD + sd_bus *bus; + if (sd_bus_default_system(&bus) < 0) { + fprintf(stderr, "memfaultd:: Failed to find systemd system bus\n"); + return false; + } + + // Check if service is active before restarting it + char *state = NULL; + if (!prv_systemd_get_service_state(bus, service_name, &state)) { + result = false; + goto cleanup; + } + if (strcmp("active", state) != 0 && strcmp("activating", state) != 0) { + fprintf(stderr, "memfaultd:: %s is not active (%s). Not starting.\n", service_name, state); + result = true; + goto cleanup; + } + + // Restart the service + if (!prv_systemd_restart_service(bus, service_name)) { + result = false; + goto cleanup; + } + +cleanup: + if (state != NULL) { + free(state); + } + sd_bus_unref(bus); + return result; +} + +/** + * @brief Send a signal to a service if it's running. + * + * @param src_module Friendly name of caller module + * @param service_name Service name to "kill", e.g. collectd.service + * @param signal Signal to send, e.g. SIGUSR1 + * @return true Successfully sent signal to requested service + * @return false Failed to restart service + */ +bool memfaultd_kill_systemd_service(const char *service_name, int signal) { + bool result = true; + + // Initialize connection to SystemD + sd_bus *bus; + if (sd_bus_default_system(&bus) < 0) { + fprintf(stderr, "memfaultd:: Failed to find systemd system bus\n"); + return false; + } + + // Send signal to service + if (!prv_systemd_kill_service(bus, service_name, signal)) { + result = false; + goto cleanup; + } + +cleanup: + sd_bus_unref(bus); + return result; +} + +/** + * @brief Checks if the current systemd state matches the requested state + * + * @return NULL Failed to get current systemd state + * + * @note Caller must free returned string + */ +char *memfaultd_get_systemd_bus_state() { + sd_bus *bus = NULL; + sd_bus_error error = SD_BUS_ERROR_NULL; + char *cur_state = NULL; + + const char *service = "org.freedesktop.systemd1"; + const char *path = "/org/freedesktop/systemd1"; + const char *interface = "org.freedesktop.systemd1.Manager"; + const char *system_state = "SystemState"; + + const int bus_result = sd_bus_default_system(&bus); + if (bus_result < 0) { + fprintf(stderr, "reboot:: Failed to find systemd system bus: %s\n", strerror(-bus_result)); + goto cleanup; + } + + if (sd_bus_get_property_string(bus, service, path, interface, system_state, &error, &cur_state) < + 0) { + fprintf(stderr, "reboot:: Failed to get system state: %s\n", error.name); + goto cleanup; + } + +cleanup: + sd_bus_error_free(&error); + sd_bus_unref(bus); + return cur_state; +} diff --git a/memfaultc-sys/Cargo.toml b/memfaultc-sys/Cargo.toml new file mode 100644 index 0000000..d89ea2a --- /dev/null +++ b/memfaultc-sys/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "memfaultc-sys" +version = "0.1.0" +edition = "2021" +autobins = false + +[dependencies] +libc = "0.2.138" + +[build-dependencies] +cc = "1.0.95" +pkg-config = "0.3" +walkdir = "2" + +[features] +default = [] +coredump = [] +systemd = [] +swupdate = [] diff --git a/memfaultc-sys/build.rs b/memfaultc-sys/build.rs new file mode 100644 index 0000000..ea9abcf --- /dev/null +++ b/memfaultc-sys/build.rs @@ -0,0 +1,78 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use cc::Build; +use std::env; +use walkdir::WalkDir; + +static LIBMEMFAULTC: &str = "../libmemfaultc/"; + +fn main() { + // Cross-compile C flags set by the CC Crate can conflict with the + // flags set by Yocto. Tell CC-rs to not set default flags. + env::set_var("CRATE_CC_NO_DEFAULTS", "true"); + + let mut cc = Build::new(); + cc.flag("-fPIC"); + cc.include(format!("{}/include", LIBMEMFAULTC)); + + cc.file(format!("{}/src/crash.c", LIBMEMFAULTC)); + + // Build a list of the library that we want to link into the final binary. + let mut libs = vec![]; + + if env::var("CARGO_FEATURE_SYSTEMD").is_ok() { + // Systemd is not available on macOS. We silently accept the feature so + // that the rust code can be checked but we don't actually build the C + // code. + if cfg!(not(target_os = "macos")) { + cc.file(format!("{}/src/systemd.c", LIBMEMFAULTC)); + libs.push("libsystemd"); + } + } + + if env::var("CARGO_FEATURE_SWUPDATE").is_ok() { + cc.file(format!("{}/src/swupdate.c", LIBMEMFAULTC)); + libs.push("libconfig"); + } + + // Linting needs to run `cargo` (and thus execute this file) to verify the + // project but the C dependencies are not installed on this machine. This + // environment variable will stop the script here, before doing any actual build. + // There is no standard Cargo way to tell if we are being called as part of `cargo lint`. + // See: https://github.com/rust-lang/cargo/issues/4001 + if env::var("MEMFAULTD_SKIP_CMAKE").is_ok() { + return; + } + + // Find required C libraries and tell Cargo how to link them + let pkg_config = pkg_config::Config::new(); + libs.iter().for_each(|lib| { + // This line will print the required Cargo config if the library is found. + match pkg_config.probe(lib) { + Ok(lib) => { + cc.includes(lib.include_paths); + } + Err(e) => println!("WARNING - Library {} was not found: {}", lib, e), + } + }); + + // Build the libmemfaultc library and link tell Cargo to link it in the project + cc.compile("memfaultc"); + println!("cargo:rustc-link-lib=static=memfaultc"); + + // Tell cargo to rebuild the project when any of the C project files changes + WalkDir::new("../libmemfaultc/src") + .into_iter() + .filter_map(Result::ok) + .filter_map( + |e| match e.path().extension().and_then(std::ffi::OsStr::to_str) { + Some("c") => Some(e), + Some("h") => Some(e), + _ => None, + }, + ) + .for_each(|e| println!("cargo:rerun-if-changed={}", e.path().display())); + + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/memfaultc-sys/src/coredump.rs b/memfaultc-sys/src/coredump.rs new file mode 100644 index 0000000..748322e --- /dev/null +++ b/memfaultc-sys/src/coredump.rs @@ -0,0 +1,6 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +extern "C" { + pub fn memfault_trigger_fp_exception(); +} diff --git a/memfaultc-sys/src/lib.rs b/memfaultc-sys/src/lib.rs new file mode 100644 index 0000000..3724f73 --- /dev/null +++ b/memfaultc-sys/src/lib.rs @@ -0,0 +1,18 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +// main functions + +#[cfg(feature = "coredump")] +pub mod coredump; + +#[cfg(all(feature = "systemd", not(target_os = "macos")))] +pub mod systemd; +#[cfg(all(feature = "systemd", target_os = "macos"))] +pub mod systemd_mock; + +#[cfg(all(feature = "systemd", target_os = "macos"))] +pub use systemd_mock as systemd; + +#[cfg(feature = "swupdate")] +pub mod swupdate; diff --git a/memfaultc-sys/src/swupdate.rs b/memfaultc-sys/src/swupdate.rs new file mode 100644 index 0000000..2b90807 --- /dev/null +++ b/memfaultc-sys/src/swupdate.rs @@ -0,0 +1,22 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use libc::c_char; + +#[repr(C)] +pub struct MemfaultSwupdateCtx { + pub base_url: *const c_char, + + pub software_version: *const c_char, + pub software_type: *const c_char, + pub hardware_version: *const c_char, + pub device_id: *const c_char, + pub project_key: *const c_char, + + pub input_file: *const c_char, + pub output_file: *const c_char, +} + +extern "C" { + pub fn memfault_swupdate_generate_config(ctx: *const MemfaultSwupdateCtx) -> bool; +} diff --git a/memfaultc-sys/src/systemd.rs b/memfaultc-sys/src/systemd.rs new file mode 100644 index 0000000..4e48d2b --- /dev/null +++ b/memfaultc-sys/src/systemd.rs @@ -0,0 +1,35 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::ffi::c_int; + +use libc::{c_char, size_t}; + +#[allow(non_camel_case_types)] +pub enum sd_journal {} + +extern "C" { + pub fn memfaultd_restart_systemd_service_if_running(service_name: *const c_char) -> bool; + pub fn memfaultd_get_systemd_bus_state() -> *const c_char; + pub fn sd_journal_open(ret: *mut *mut sd_journal, flags: c_int) -> c_int; + pub fn sd_journal_seek_tail(j: *mut sd_journal) -> c_int; + pub fn sd_journal_previous(j: *mut sd_journal) -> c_int; + pub fn sd_journal_next(j: *mut sd_journal) -> c_int; + pub fn sd_journal_get_data( + j: *mut sd_journal, + field: *const c_char, + data: *mut *mut u8, + l: *mut size_t, + ) -> c_int; + pub fn sd_journal_enumerate_data( + j: *mut sd_journal, + data: *mut *mut u8, + l: *mut size_t, + ) -> c_int; + pub fn sd_journal_get_fd(j: *mut sd_journal) -> c_int; + pub fn sd_journal_process(j: *mut sd_journal) -> c_int; + pub fn sd_journal_get_realtime_usec(j: *mut sd_journal, ret: *mut u64) -> c_int; + pub fn sd_journal_get_cursor(j: *mut sd_journal, cursor: *mut *const c_char) -> c_int; + pub fn sd_journal_seek_cursor(j: *mut sd_journal, cursor: *const c_char) -> c_int; + pub fn sd_journal_add_match(j: *mut sd_journal, data: *const c_char, size: size_t) -> c_int; +} diff --git a/memfaultc-sys/src/systemd_mock.rs b/memfaultc-sys/src/systemd_mock.rs new file mode 100644 index 0000000..67180a5 --- /dev/null +++ b/memfaultc-sys/src/systemd_mock.rs @@ -0,0 +1,119 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::ptr::null; + +use libc::{c_char, c_int, size_t}; + +#[allow(non_camel_case_types)] +pub enum sd_journal {} + +/// Get the status of the systemd service manager. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn memfaultd_restart_systemd_service_if_running(service_name: *const c_char) -> bool { + eprintln!("memfaultd_restart_systemd_service_if_running is not implemented for this target (restarting service {})", std::ffi::CStr::from_ptr(service_name).to_string_lossy()); + true +} + +/// Get the status of the systemd service manager. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn memfaultd_get_systemd_bus_state() -> *const c_char { + null() +} + +/// Open the systemd journal. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_open(_ret: *mut *mut sd_journal, _flags: c_int) -> c_int { + eprintln!("sd_journal_open is not implemented for this target"); + -1 +} + +/// Seek to the end of the systemd journal. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_seek_tail(_j: *mut sd_journal) -> c_int { + eprintln!("sd_journal_seek_tail is not implemented for this target"); + -1 +} + +/// Get the previous entry in the systemd journal. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_previous(_j: *mut sd_journal) -> c_int { + eprintln!("sd_journal_previous is not implemented for this target"); + -1 +} + +/// Get the next entry in the systemd journal. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_next(_j: *mut sd_journal) -> c_int { + eprintln!("sd_journal_next is not implemented for this target"); + -1 +} + +/// Get the data of the systemd journal entry field. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_get_data( + _j: *mut sd_journal, + _field: *const c_char, + _data: *mut *mut u8, + _l: *mut size_t, +) -> c_int { + eprintln!("sd_journal_get_data is not implemented for this target"); + -1 +} + +/// Get all the field data of the systemd journal entry. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_enumerate_data( + _j: *mut sd_journal, + _data: *mut *mut u8, + _l: *mut size_t, +) -> c_int { + eprintln!("sd_journal_enumerate_data is not implemented for this target"); + -1 +} + +/// Get the file descriptor of the systemd journal. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_get_fd(_j: *mut sd_journal) -> c_int { + eprintln!("sd_journal_get_fd is not implemented for this target"); + -1 +} + +/// Signal that we've processed the journal entry. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_process(_j: *mut sd_journal) -> c_int { + eprintln!("sd_journal_process is not implemented for this target"); + -1 +} + +/// Get timestamp for the systemd journal entry. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_get_realtime_usec(_j: *mut sd_journal, _ret: *mut u64) -> c_int { + eprintln!("sd_journal_get_realtime_usec is not implemented for this target"); + -1 +} + +/// Get the cursor for the systemd journal. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_get_cursor(_j: *mut sd_journal, _cursor: *mut *const c_char) -> c_int { + eprintln!("sd_journal_get_cursor is not implemented for this target"); + -1 +} + +/// Seek to the cursor in the systemd journal. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_seek_cursor(_j: *mut sd_journal, _cursor: *const c_char) -> c_int { + eprintln!("sd_journal_seek_cursor is not implemented for this target"); + -1 +} + +/// Seek to the cursor in the systemd journal. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_add_match( + _j: *mut sd_journal, + _data: *const c_char, + _size: size_t, +) -> c_int { + eprintln!("sd_journal_add_match is not implemented for this target"); + -1 +} diff --git a/memfaultd.init b/memfaultd.init new file mode 100644 index 0000000..63e9ca8 --- /dev/null +++ b/memfaultd.init @@ -0,0 +1,150 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: memfaultd +# Required-Start: $local_fs +# Should-Start: +# Required-Stop: $local_fs +# Should-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: manages the memfaultd daemon +### END INIT INFO + +# PATH should only include /usr/* if it runs after the mountnfs.sh script +PATH=/sbin:/usr/sbin:/bin:/usr/bin + +DESC="memfaultd" +NAME="memfaultd" +DAEMON=/usr/bin/memfaultd +PIDFILE=/var/run/$NAME.pid +ARGS="-Z -q" + +. /etc/init.d/functions || exit 1 + +# Exit if the package is not installed +[ -x "$DAEMON" ] || exit 0 + +# Read configuration variable file if it is present +[ -r /etc/default/$NAME ] && . /etc/default/$NAME + +# +# Function that starts the daemon/service +# +do_start() { + local status pid + + status=0 + pid=`pidofproc $NAME` || status=$? + case $status in + 0) + echo "$DESC already running ($pid)." + exit 1 + ;; + *) + echo "Starting $DESC ..." + cd /home/root + + exec $DAEMON $ARGS + exit 0 + ;; + esac +} + +# +# Function that stops the daemon/service +# +do_stop() { + local pid status + + status=0 + pid=`pidofproc $NAME` || status=$? + case $status in + 0) + # Exit when fail to stop, the kill would complain when fail + kill -s SIGTERM $pid >/dev/null && rm -f $PIDFILE && \ + echo "Stopped $DESC ($pid)." || exit $? + + # Wait up to 10 seconds for the process to exit + for i in `seq 10`; do + if ! pidofproc $NAME > /dev/null; then + break + fi + sleep 1 + done + ;; + *) + echo "$DESC is not running; none killed." >&2 + ;; + esac + + return 0 +} + +# +# Function that sends a SIGHUP to the daemon/service +# +do_reload() { + local pid status + + status=0 + pid=`pidofproc $NAME` || status=$? + case $status in + 0) + echo "Reloading $DESC ..." + kill -s SIGHUP $pid || exit $? + ;; + *) + echo "$DESC is not running; none reloaded." >&2 + ;; + esac + exit $status +} + + +# +# Function that shows the daemon/service status +# +status_of_proc () { + local pid status + + status=0 + # pidof output null when no program is running, so no "2>/dev/null". + pid=`pidofproc $NAME` || status=$? + case $status in + 0) + echo "$DESC is running ($pid)." + exit 0 + ;; + *) + echo "$DESC is not running." >&2 + exit $status + ;; + esac +} + +case "$1" in +start) + do_start + ;; +stop) + do_stop || exit $? + ;; +status) + status_of_proc + ;; +restart) + # Always start the service regardless the status of do_stop + do_stop + do_start + ;; +try-restart|force-reload) + do_stop && do_start + ;; +reload) + do_reload + ;; +*) + echo "Usage: $0 {start|stop|status|restart|try-restart|force-reload}" >&2 + exit 3 + ;; +esac diff --git a/memfaultd.service b/memfaultd.service new file mode 100644 index 0000000..7c13c32 --- /dev/null +++ b/memfaultd.service @@ -0,0 +1,15 @@ +[Unit] +Description=memfaultd daemon +After=local-fs.target network.target dbus.service +Before=swupdate.service collectd.service + +[Service] +Type=forking +PIDFile=/run/memfaultd.pid +ExecStart=/usr/bin/memfaultd --daemonize +# Wait for the PID file to be populated before returning +ExecStartPost=/bin/sh -c "while [ $(cat /run/memfaultd.pid 2>/dev/null | wc -c) -eq 0 ]; do sleep 0.1; done" +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/memfaultd/Cargo.toml b/memfaultd/Cargo.toml new file mode 100644 index 0000000..474de6e --- /dev/null +++ b/memfaultd/Cargo.toml @@ -0,0 +1,104 @@ +[package] +name = "memfaultd" +version = "0.1.0" +edition = "2021" +autobins = false +rust-version = "1.65" + +[[bin]] +name = "memfaultd" +path = "src/bin/memfaultd.rs" + +[[bin]] +name= "memfaultctl" +path= "src/bin/memfaultctl.rs" + +[[bin]] +name= "memfault-core-handler" +path= "src/bin/memfault-core-handler.rs" + +[[bin]] +name= "mfw" +path= "src/bin/mfw.rs" + +[dependencies] +memfaultc-sys = { path= "../memfaultc-sys" } +argh = "0.1.10" +cfg-if = "1.0.0" +chrono = { version = "0.4.23", features = ["serde"]} +ciborium = { version = "0.2.1", optional = true} +eyre = "0.6.8" +goblin = { version = "0.7", optional = true, default-features = false, features = ["elf32", "elf64", "std"] } +libc = "0.2.138" +log = "0.4.17" +prctl = { version = "1.0.0", optional = true} +psm = { version = "0.1.21", optional = true } +reqwest = { version = "0.11", default-features = false, features = ["blocking", "json" ] } +rmp-serde = { version = "1.1.1", optional = true } +rmpv = { version = "1.0.0", optional = true } +scroll = { version = "0.11", optional = true } +serde = { version = "1.0.150", features = ["derive"] } +serde_bytes = "0.11.8" +serde_json = "1.0.89" +serde_repr = "0.1" +shuteye = "0.3.3" +signal-hook = "0.3.14" +stderrlog = "0.5.4" +strum = { version = "0.24", features = ["derive"] } +strum_macros = "0.24" +tempfile = "3.3.0" +thiserror = "1.0.38" +threadpool = { version = "1.8.1"} +urlencoding = "2.1.2" +uuid = { version = "1.3.0", features = ["v4", "serde"] } +once_cell = "1.17.0" +fs_extra = "1.3.0" +flate2 = { version = "1.0.28", default-features = false, features = ["zlib"] } +take_mut = "0.2.2" +itertools = "0.10.5" +governor = { version = "0.5.1", default-features = false, features = ["std"], optional = true} +nix = { version = "0.26.2", default-features = false, features = ["process", "signal", "poll", "fs"]} +kernlog = { version = "0.3.1", optional = true } +tiny_http = {version = "0.12.0" } +crc = "3.0.1" +crc-catalog = "2.2.0" +regex= { version = "1.10.2", optional = true} +nom = "7.1.3" +sealed_test = "1.1.0" + +[target.'cfg(target_os = "linux")'.dependencies] +procfs = { version = "0.15.1", optional = true } + +[dev-dependencies] +tempfile = "3.3.0" +mockall = "0.11.3" +rstest = "0.16.0" +goblin = { version = "0.7", default-features = false, features = ["elf32", "elf64", "std", "endian_fd"] } +insta = {version= "1.26.0", features= ["json", "redactions"]} +zip = { version = "0.6.3", default-features = false, features = ["deflate"] } +hex = "0.4.3" +rand = "0.8.5" +nom = "7.1.3" + +[features] +default = ["coredump", "collectd", "logging", "log-to-metrics", "systemd", "rust-tls" ] +coredump = [ + "memfaultc-sys/coredump", + "dep:prctl", + "dep:procfs", + "dep:psm", + "dep:kernlog", + "dep:ciborium", + "dep:goblin", + "dep:scroll" +] +collectd = [] +swupdate = ["memfaultc-sys/swupdate"] +logging = ["dep:governor", "dep:rmp-serde", "dep:rmpv"] +systemd = ["memfaultc-sys/systemd"] +rust-tls = ["reqwest/rustls-tls"] +openssl-tls = ["reqwest/native-tls"] +openssl-vendored-tls = ["reqwest/native-tls-vendored"] +log-to-metrics = ["dep:regex"] +experimental = ["mfw"] +mfw = [] diff --git a/memfaultd/DEVELOPMENT.md b/memfaultd/DEVELOPMENT.md new file mode 100644 index 0000000..0019195 --- /dev/null +++ b/memfaultd/DEVELOPMENT.md @@ -0,0 +1,60 @@ +# Development + +`memfaultd` build is controlled by Cargo. The `Cargo.toml` and `build.rs` +control the rust build process and compile the few C files. + +## Building outside Yocto + +### Dependencies + +#### Debian/Ubuntu + +```sh +apt install libsystemd-dev libconfig-dev +``` + +#### macOS + +```sh +brew install libconfig +``` + +(note: `libsystemd` is not available on macOS and the build system will not try +to link it) + +### Building + +```sh +cargo build +``` + +## Building with Yocto + +Use the `docker/run.sh` script to run a docker container with all the required +dependencies. Use the alias `b` to build the image. + +## Running tests + +### Unit tests + +Do this after running a build, inside the (cmake) build directory: + +```sh +cargo test +``` + +### Updating snapshots + +Install `insta` if necessary, and run the command: + +```bash +cargo install cargo-insta +cargo insta review +``` + +## IDE integration + +### Using VSCode to work on memfaultd + +VSCode rust plugin will not find the `Cargo.toml` file unless you open the +`meta-memfault/recipes-memfault/memfaultd/files/memfaultd/` directly. diff --git a/memfaultd/build.rs b/memfaultd/build.rs new file mode 100644 index 0000000..9f6ef59 --- /dev/null +++ b/memfaultd/build.rs @@ -0,0 +1,66 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::collections::HashMap; +use std::env; +use std::fs::write; +use std::path::{Path, PathBuf}; + +/// Generates $OUT_DIR/build_info.rs, based on the values in the VERSION file (generated by sdk_release.py). +fn generate_build_info_rs() { + let mut version_file_path = PathBuf::from(&env::var("CARGO_MANIFEST_DIR").unwrap()); + version_file_path.push("../VERSION"); + println!("cargo:rerun-if-changed={}", version_file_path.display()); + + let version_file_content = + std::fs::read_to_string(version_file_path).unwrap_or_else(|_| String::new()); + + let key_values_from_version_file: HashMap<&str, &str> = version_file_content + .lines() + .filter_map(|l| l.split_once(':')) + .map(|(k, v)| (k.trim(), v.trim())) + .collect(); + + struct VersionVarInfo { + key: &'static str, + var: &'static str, + default: &'static str, + } + + let keys_and_vars_defaults = [ + VersionVarInfo { + key: "VERSION", + var: "VERSION", + default: "dev", + }, + VersionVarInfo { + key: "GIT COMMIT", + var: "GIT_COMMIT", + default: "unknown", + }, + VersionVarInfo { + key: "BUILD ID", + var: "BUILD_ID", + default: "unknown", + }, + ]; + + let build_info_rs_src = keys_and_vars_defaults + .iter() + .fold(String::new(), |mut acc, info| { + let value = key_values_from_version_file + .get(info.key) + .unwrap_or(&info.default); + acc.push_str(&format!("pub const {}: &str = \"{}\";\n", info.var, value)); + acc + }); + + let out_dir = env::var_os("OUT_DIR").unwrap(); + let dest_path = Path::new(&out_dir).join("build_info.rs"); + write(dest_path, build_info_rs_src).unwrap(); +} + +fn main() { + generate_build_info_rs(); + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/memfaultd/builtin.conf b/memfaultd/builtin.conf new file mode 100644 index 0000000..40917c9 --- /dev/null +++ b/memfaultd/builtin.conf @@ -0,0 +1,64 @@ +/* +See https://mflt.io/memfaultd.conf for information on the available +parameters. +*/ +{ + "persist_dir": "/media/memfault", + "tmp_dir": null, + "tmp_dir_min_headroom_kib": 10240, + "tmp_dir_min_inodes": 100, + "tmp_dir_max_usage_kib": 102400, + "upload_interval_seconds": 3600, + "heartbeat_interval_seconds": 3600, + "enable_data_collection": false, + "enable_dev_mode": false, + "project_key": "", + "base_url": "https://device.memfault.com", + "swupdate": { + "input_file": "/etc/swupdate.cfg", + "output_file": "/tmp/swupdate.cfg" + }, + "reboot": { + "last_reboot_reason_file": "/media/last_reboot_reason" + }, + "coredump": { + "coredump_max_size_kib": 96000, + "compression": "gzip", + "rate_limit_count": 5, + "rate_limit_duration_seconds": 3600, + "capture_strategy": { + "type": "threads", + "max_thread_size_kib": 32 + }, + "log_lines": 100 + }, + "http_server": { + "bind_address": "127.0.0.1:8787" + }, + "fluent-bit": { + "extra_fluentd_attributes": [], + "bind_address": "127.0.0.1:5170", + "max_buffered_lines": 1000, + "max_connections": 4 + }, + "logs": { + "compression_level": 1, + "max_lines_per_minute": 500, + "rotate_size_kib": 10240, + "rotate_after_seconds": 3600, + "storage": "persist", + "source": "fluent-bit" + }, + "mar": { + "mar_file_max_size_kib": 10240, + "mar_entry_max_age_seconds": 604800 + }, + "battery_monitor": null, + "metrics": { + "enable_daily_heartbeats": false, + "system_metric_collection": { + "poll_interval_seconds": 10, + "enable": false + } + } +} diff --git a/memfaultd/memfaultd.conf b/memfaultd/memfaultd.conf new file mode 100644 index 0000000..f72e2b9 --- /dev/null +++ b/memfaultd/memfaultd.conf @@ -0,0 +1,13 @@ +/* +See builtin.conf (in the source files) for configuration options. +At minimum, your override of this file should provide these keys: +{ + "software_version": "", + "software_type": "", + "project_key": "", + "persist_dir": "" +} +*/ + +{ +} diff --git a/memfaultd/src/bin/memfault-core-handler.rs b/memfaultd/src/bin/memfault-core-handler.rs new file mode 100644 index 0000000..83cdded --- /dev/null +++ b/memfaultd/src/bin/memfault-core-handler.rs @@ -0,0 +1,11 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use memfaultd::cli; + +/// memfault-core-handlers is an alias to the main function in cli.rs +/// +/// For further details, see comments on memfaultd.rs +fn main() { + cli::main() +} diff --git a/memfaultd/src/bin/memfaultctl.rs b/memfaultd/src/bin/memfaultctl.rs new file mode 100644 index 0000000..7d62156 --- /dev/null +++ b/memfaultd/src/bin/memfaultctl.rs @@ -0,0 +1,11 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use memfaultd::cli; + +/// memfaultctl is an alias to the main function in cli.rs +/// +/// For further details, see comments on memfaultd.rs +fn main() { + cli::main() +} diff --git a/memfaultd/src/bin/memfaultd.rs b/memfaultd/src/bin/memfaultd.rs new file mode 100644 index 0000000..ab835cf --- /dev/null +++ b/memfaultd/src/bin/memfaultd.rs @@ -0,0 +1,20 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use memfaultd::cli; + +/// memfaultd is an alias to the main function in cli.rs +/// +/// In the target machine, memfaultd is the only binary that remains. memfaultctl and +/// memfault-core-handler are symlinked by build scripts. In the case of Yocto, this is +/// meta-memfault/recipes-memfault/memfaultd/memfaultd.bb +/// +/// The binary setup in this crate could opt for a single bin/memfaultd.rs entrypoint, however +/// setting up three different binaries makes development easier because it mimics what build +/// systems do in target devices. +/// +/// Alternative solutions are possible, for example adding a post-build script that does the +/// symlinking manually in the build directory. +fn main() { + cli::main() +} diff --git a/memfaultd/src/bin/mfw.rs b/memfaultd/src/bin/mfw.rs new file mode 100644 index 0000000..46868bb --- /dev/null +++ b/memfaultd/src/bin/mfw.rs @@ -0,0 +1,11 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use memfaultd::cli; + +/// mfw is an alias to the main function in cli.rs +/// +/// For further details, see comments on memfaultd.rs +fn main() { + cli::main() +} diff --git a/memfaultd/src/cli/cargs.rs b/memfaultd/src/cli/cargs.rs new file mode 100644 index 0000000..8e9e5dd --- /dev/null +++ b/memfaultd/src/cli/cargs.rs @@ -0,0 +1,46 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use libc::c_char; +use libc::c_int; +use std::ffi::CString; +use std::path::Path; + +pub struct CArgs { + argv: Vec, + argv_ptr: Vec<*const libc::c_char>, +} + +impl CArgs { + pub fn new(args: impl IntoIterator) -> Self { + let argv: Vec<_> = args + .into_iter() + .map(|arg| CString::new(arg.as_str()).unwrap()) + .collect(); + let argv_ptr: Vec<_> = argv + .iter() + .map(|arg| arg.as_ptr()) + .chain(std::iter::once(std::ptr::null())) + .collect(); + Self { argv, argv_ptr } + } + + /// Returns the number of arguments, ie the C language's `argc`. + pub fn argc(&self) -> c_int { + self.argv.len() as c_int + } + + /// Returns the C language's `argv` (`*const *const c_char`). + pub fn argv(&self) -> *const *const c_char { + self.argv_ptr.as_ptr() + } + + /// Returns the name of the command invoked by the user - removing any path information. + pub fn name(&self) -> &str { + Path::new(self.argv[0].to_str().unwrap()) + .file_name() + .unwrap() + .to_str() + .unwrap() + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/arch.rs b/memfaultd/src/cli/memfault_core_handler/arch.rs new file mode 100644 index 0000000..0102688 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/arch.rs @@ -0,0 +1,72 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::cli::memfault_core_handler::elf; +use cfg_if::cfg_if; + +cfg_if! { + if #[cfg(target_arch = "aarch64")] { + pub use libc::user_regs_struct as ElfGRegSet; + pub fn get_stack_pointer(regs: &ElfGRegSet) -> usize { + regs.sp as usize + } + pub use elf::header::EM_AARCH64 as ELF_TARGET_MACHINE; + pub use elf::header::ELFCLASS64 as ELF_TARGET_CLASS; + } else if #[cfg(target_arch = "x86_64")] { + pub use libc::user_regs_struct as ElfGRegSet; + pub fn get_stack_pointer(regs: &ElfGRegSet) -> usize { + regs.rsp as usize + } + pub use elf::header::EM_X86_64 as ELF_TARGET_MACHINE; + pub use elf::header::ELFCLASS64 as ELF_TARGET_CLASS; + } else if #[cfg(target_arch = "arm")] { + pub use libc::user_regs as ElfGRegSet; + pub fn get_stack_pointer(regs: &ElfGRegSet) -> usize { + regs.arm_sp as usize + } + pub use elf::header::EM_ARM as ELF_TARGET_MACHINE; + pub use elf::header::ELFCLASS32 as ELF_TARGET_CLASS; + } else if #[cfg(target_arch = "x86")] { + pub use libc::user_regs_struct as ElfGRegSet; + pub fn get_stack_pointer(regs: &ElfGRegSet) -> usize { + regs.esp as usize + } + pub use elf::header::EM_386 as ELF_TARGET_MACHINE; + pub use elf::header::ELFCLASS32 as ELF_TARGET_CLASS; + } + else { + // Provide dummy symbols for unsupported architectures. This is preferable to + // a compile time error, as we want to be able to compile memfaultd for all + // architectures, but we don't need register access for all of them. Currently + // these registers are only used to filter out stack memory from coredumps. + pub struct ElfGRegSet; + pub fn get_stack_pointer(_regs: &ElfGRegSet) -> usize { + 0 + } + } +} + +// Function definitions for coredump thread filter support. If the target architecture +// is not supported, these functions will always return false. +cfg_if! { + if #[cfg(any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "x86", + target_arch = "x86_64" + ))] { + pub const fn coredump_thread_filter_supported() -> bool { + true + } + } else { + pub const fn coredump_thread_filter_supported() -> bool { + false + } + } +} + +#[cfg(target_endian = "little")] +pub use elf::header::ELFDATA2LSB as ELF_TARGET_ENDIANNESS; + +#[cfg(target_endian = "big")] +pub use elf::header::ELFDATA2MSB as ELF_TARGET_ENDIANNESS; diff --git a/memfaultd/src/cli/memfault_core_handler/auxv.rs b/memfaultd/src/cli/memfault_core_handler/auxv.rs new file mode 100644 index 0000000..6b24753 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/auxv.rs @@ -0,0 +1,114 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use scroll::Pread; +use std::fmt::Debug; + +#[cfg(target_pointer_width = "64")] +pub type AuxvUint = u64; + +#[cfg(target_pointer_width = "32")] +pub type AuxvUint = u32; + +#[derive(Eq, PartialEq)] +pub struct Auxv<'a> { + data: &'a [u8], +} + +#[derive(Eq, PartialEq, Debug)] +pub struct AuxvEntry { + pub key: AuxvUint, + pub value: AuxvUint, +} + +impl<'a> Auxv<'a> { + pub fn new(data: &'a [u8]) -> Self { + Self { data } + } + + pub fn iter(&'a self) -> AuxvIterator<'a> { + AuxvIterator::new(self) + } + + pub fn find_value(&self, key: AuxvUint) -> Option { + self.iter() + .find(|a| a.key == key as AuxvUint) + .map(|a| a.value) + } +} + +impl<'a> Debug for Auxv<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.iter().collect::>().fmt(f) + } +} + +pub struct AuxvIterator<'a> { + auxv: &'a Auxv<'a>, + offset: usize, +} + +impl<'a> AuxvIterator<'a> { + fn new(auxv: &'a Auxv<'a>) -> Self { + Self { auxv, offset: 0 } + } +} + +impl<'a> Iterator for AuxvIterator<'a> { + type Item = AuxvEntry; + + fn next(&mut self) -> Option { + let mut read = || self.auxv.data.gread::(&mut self.offset); + match (read(), read()) { + (Ok(key), Ok(value)) => Some(AuxvEntry { key, value }), + _ => None, + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use rstest::rstest; + use scroll::IOwrite; + use std::io::Cursor; + + #[rstest] + // Empty auxv: + #[case( + vec![], + vec![] + )] + // Happy path: + #[case( + vec![1, 2, 3, 4], + vec![AuxvEntry { key: 1, value: 2 }, AuxvEntry { key: 3, value: 4 }] + )] + // Partial entry at the end is ignored: + #[case( + vec![1, 2, 3], + vec![AuxvEntry { key: 1, value: 2 }] + )] + fn test_auxv_iterator(#[case] values: Vec, #[case] expected: Vec) { + let buffer = make_fixture(values); + let auxv = Auxv::new(buffer.as_slice()); + assert_eq!(auxv.iter().collect::>(), expected); + } + + #[test] + fn test_auxv_find_value() { + let buffer = make_fixture(vec![1, 2]); + let auxv = Auxv::new(buffer.as_slice()); + assert!(auxv.find_value(1).is_some()); + assert!(auxv.find_value(9).is_none()); + } + + fn make_fixture(values: Vec) -> Vec { + let mut cursor = Cursor::new(vec![]); + for value in values { + cursor.iowrite::(value).unwrap(); + } + + cursor.into_inner() + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/core_elf_memfault_note.rs b/memfaultd/src/cli/memfault_core_handler/core_elf_memfault_note.rs new file mode 100644 index 0000000..5f1eb0c --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/core_elf_memfault_note.rs @@ -0,0 +1,221 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Utilities and data types for writing Memfault-specific ELF notes to a core dump file. +//! +//! Currently we write two notes: +//! +//! 1. A note containing metadata about the core dump. This note is written by the +//! `CoreHandler` whenever it receives a core dump. It contains information about the device, +//! that will be used to associate the core dump with a device in the Memfault cloud. +//! 2. A note containing debug data about the core dump. Currently this note only contains +//! logs written during the coredump capture process. These logs are used by Memfault to debug +//! issues with coredump capture. +use std::time::SystemTime; + +use crate::config::CoredumpCaptureStrategy; +use crate::{build_info::VERSION, mar::LinuxLogsFormat}; + +use ciborium::{cbor, into_writer}; +use eyre::Result; +use serde::Serialize; + +use super::core_elf_note::build_elf_note; + +const NOTE_NAME: &str = "Memfault\0"; +const METADATA_NOTE_TYPE: u32 = 0x4154454d; +const DEBUG_DATA_NOTE_TYPE: u32 = 0x4154454e; +const MEMFAULT_CORE_ELF_METADATA_SCHEMA_VERSION_V1: u32 = 1; +const MEMFAULT_CORE_ELF_DEBUG_DATA_SCHEMA_VERSION_V1: u32 = 1; + +/// Map of keys used in the Memfault core ELF metadata note. +/// +/// Integer keys are used here instead of strings to reduce the size of the note. +enum MemfaultCoreElfMetadataKey { + SchemaVersion = 1, + LinuxSdkVersion = 2, + CapturedTime = 3, + DeviceSerial = 4, + HardwareVersion = 5, + SoftwareType = 6, + SoftwareVersion = 7, + CmdLine = 8, + CaptureStrategy = 9, + ApplicationLogs = 10, +} + +#[derive(Debug, Serialize)] +pub struct MemfaultMetadataLogs { + logs: Vec, + format: LinuxLogsFormat, +} + +impl MemfaultMetadataLogs { + pub fn new(logs: Vec, format: LinuxLogsFormat) -> Self { + Self { logs, format } + } +} + +/// Metadata about a core dump. +#[derive(Debug)] +pub struct CoredumpMetadata { + pub device_id: String, + pub hardware_version: String, + pub software_type: String, + pub software_version: String, + pub sdk_version: String, + pub captured_time_epoch_s: u64, + pub cmd_line: String, + pub capture_strategy: CoredumpCaptureStrategy, + pub app_logs: Option, +} + +impl CoredumpMetadata { + pub fn new(config: &crate::config::Config, cmd_line: String) -> Self { + Self { + device_id: config.device_info.device_id.clone(), + hardware_version: config.device_info.hardware_version.clone(), + software_type: config.software_type().to_string(), + software_version: config.software_version().to_string(), + sdk_version: VERSION.to_string(), + captured_time_epoch_s: SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(), + cmd_line, + capture_strategy: config.config_file.coredump.capture_strategy, + app_logs: None, + } + } +} + +/// Serialize a `CoredumpMetadata` struct as a CBOR map. +/// +/// This CBOR map uses integer keys instead of strings to reduce the size of the note. +pub fn serialize_metadata_as_map(metadata: &CoredumpMetadata) -> Result> { + let cbor_val = cbor!({ + MemfaultCoreElfMetadataKey::SchemaVersion as u32 => MEMFAULT_CORE_ELF_METADATA_SCHEMA_VERSION_V1, + MemfaultCoreElfMetadataKey::LinuxSdkVersion as u32 => metadata.sdk_version, + MemfaultCoreElfMetadataKey::CapturedTime as u32 => metadata.captured_time_epoch_s, + MemfaultCoreElfMetadataKey::DeviceSerial as u32 => metadata.device_id, + MemfaultCoreElfMetadataKey::HardwareVersion as u32 => metadata.hardware_version, + MemfaultCoreElfMetadataKey::SoftwareType as u32 => metadata.software_type, + MemfaultCoreElfMetadataKey::SoftwareVersion as u32 => metadata.software_version, + MemfaultCoreElfMetadataKey::CmdLine as u32 => metadata.cmd_line, + MemfaultCoreElfMetadataKey::CaptureStrategy as u32 => metadata.capture_strategy, + MemfaultCoreElfMetadataKey::ApplicationLogs as u32 => metadata.app_logs, + })?; + + let mut buffer = Vec::new(); + into_writer(&cbor_val, &mut buffer)?; + + Ok(buffer) +} + +/// Write a core ELF note containing metadata about a core dump. +/// +/// This note is written by the `CoreHandler` whenever it receives a core dump. It contains +/// information about the device, that will be used to associate the core dump with a device in the +/// Memfault cloud. +pub fn write_memfault_metadata_note(metadata: &CoredumpMetadata) -> Result> { + let description_buffer = serialize_metadata_as_map(metadata)?; + + build_elf_note(NOTE_NAME, &description_buffer, METADATA_NOTE_TYPE) +} + +/// A note containing a list of errors that occurred during coredump capture. +/// +/// This note is written by the `CoreHandlerLogWrapper` when it receives an error or warning log. +/// These logs will help us debug issues with coredump capture. +#[derive(Debug, Serialize)] +pub struct CoredumpDebugData { + pub schema_version: u32, + pub capture_logs: Vec, +} + +/// Write a core ELF note containing debug data about the coredump capture process. +/// +/// See `CoredumpDebugData` for more information. +pub fn write_memfault_debug_data_note(errors: Vec) -> Result> { + let coredump_capture_logs = CoredumpDebugData { + schema_version: MEMFAULT_CORE_ELF_DEBUG_DATA_SCHEMA_VERSION_V1, + capture_logs: errors, + }; + + let mut buffer = Vec::new(); + into_writer(&coredump_capture_logs, &mut buffer)?; + + build_elf_note(NOTE_NAME, &buffer, DEBUG_DATA_NOTE_TYPE) +} + +#[cfg(test)] +mod test { + use ciborium::{from_reader, Value}; + use rstest::rstest; + + use crate::test_utils::set_snapshot_suffix; + + use super::*; + + #[rstest] + #[case( + "kernel_selection", + CoredumpCaptureStrategy::KernelSelection, + 91, + false + )] + #[case("threads", CoredumpCaptureStrategy::Threads{ max_thread_size: 32 * 1024}, 104, false)] + #[case("app_logs", CoredumpCaptureStrategy::KernelSelection, 160, true)] + fn test_serialize_metadata_as_map( + #[case] test_name: &str, + #[case] capture_strategy: CoredumpCaptureStrategy, + #[case] expected_size: usize, + #[case] has_app_logs: bool, + ) { + let app_logs = has_app_logs.then(|| MemfaultMetadataLogs { + logs: vec![ + "Error 1".to_string(), + "Error 2".to_string(), + "Error 3".to_string(), + ], + format: LinuxLogsFormat::default(), + }); + let metadata = CoredumpMetadata { + device_id: "12345678".to_string(), + hardware_version: "evt".to_string(), + software_type: "main".to_string(), + software_version: "1.0.0".to_string(), + sdk_version: "SDK_VERSION".to_string(), + captured_time_epoch_s: 1234, + cmd_line: "binary -a -b -c".to_string(), + capture_strategy, + app_logs, + }; + + let map = serialize_metadata_as_map(&metadata).unwrap(); + let deser_map: Value = from_reader(map.as_slice()).unwrap(); + + set_snapshot_suffix!("{}", test_name); + insta::assert_debug_snapshot!(deser_map); + assert_eq!(map.len(), expected_size); + } + + #[test] + fn serialize_debug_data() { + let capture_logs = CoredumpDebugData { + schema_version: MEMFAULT_CORE_ELF_DEBUG_DATA_SCHEMA_VERSION_V1, + capture_logs: vec![ + "Error 1".to_string(), + "Error 2".to_string(), + "Error 3".to_string(), + ], + }; + + let mut capture_logs_buffer = Vec::new(); + into_writer(&capture_logs, &mut capture_logs_buffer).unwrap(); + + let deser_capture_logs: Value = from_reader(capture_logs_buffer.as_slice()).unwrap(); + + insta::assert_debug_snapshot!(deser_capture_logs); + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/core_elf_note.rs b/memfaultd/src/cli/memfault_core_handler/core_elf_note.rs new file mode 100644 index 0000000..2e5b10f --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/core_elf_note.rs @@ -0,0 +1,638 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::cmp::min; +use std::ffi::{CStr, OsStr}; +use std::mem::size_of; +use std::os::unix::ffi::OsStrExt; +use std::path::Path; + +use eyre::{eyre, Error, Result}; + +// The Nhdr64 struct is incorrect in goblin. We need to open a PR with them, +// but for now going to use the 32 bit version. +use crate::cli::memfault_core_handler::arch::ElfGRegSet; +use crate::cli::memfault_core_handler::auxv::Auxv; +use crate::cli::memfault_core_handler::ElfPtrSize; +use crate::util::math::align_up; +use goblin::elf::note::Nhdr32 as Nhdr; +use goblin::elf::note::{NT_FILE, NT_GNU_BUILD_ID, NT_PRSTATUS}; +use log::{error, warn}; +use scroll::{Pread, Pwrite}; + +/// Builds an ELF note from a given name, description, and note type. +pub fn build_elf_note(name: &str, description: &[u8], note_type: u32) -> Result> { + let name_bytes = name.as_bytes(); + let mut name_size = name_bytes.len(); + // NOTE: As per the spec, the terminating NUL byte should be included in the namesz, but 0 is + // used for an empty string: + if name_size > 0 { + name_size += 1; + } + let note_header = Nhdr { + n_namesz: name_size.try_into()?, + n_descsz: description.len().try_into()?, + n_type: note_type, + }; + let aligned_name_size = align_up(name_size, 4); + + let header_size = size_of::(); + let mut note_buffer = + vec![0u8; header_size + aligned_name_size + align_up(description.len(), 4)]; + note_buffer.pwrite(note_header, 0)?; + note_buffer[header_size..(header_size + name_bytes.len())].copy_from_slice(name_bytes); + + let desc_offset = header_size + aligned_name_size; + note_buffer[desc_offset..(desc_offset + description.len())].copy_from_slice(description); + + Ok(note_buffer) +} + +#[derive(Debug, PartialEq, Eq)] +/// Parsed ELF note. +/// +/// Contains the deserialized ELF note description for a given note type. +/// Unknown can be used in case the note type is not supported or if parsing failed. +pub enum ElfNote<'a> { + /// Parsed CORE::NT_PRSTATUS note. + ProcessStatus(&'a ProcessStatusNote), + // Parsed CORE::NT_FILE note + File(FileNote<'a>), + // Parsed GNU::NT_GNU_BUILD_ID note. + GnuBuildId(&'a [u8]), + Auxv(Auxv<'a>), + Unknown { + name: &'a [u8], + note_type: u32, + description: &'a [u8], + }, +} + +const NOTE_NAME_CORE: &[u8] = b"CORE"; +const NOTE_NAME_GNU: &[u8] = b"GNU"; + +const NT_AUXV: u32 = 6; + +impl<'a> ElfNote<'a> { + fn try_parse(name: &'a [u8], note_type: u32, description: &'a [u8]) -> Result> { + match (name, note_type) { + (NOTE_NAME_CORE, NT_PRSTATUS) => Ok(Some(Self::ProcessStatus(description.try_into()?))), + (NOTE_NAME_CORE, NT_FILE) => Ok(Some(Self::File(description.try_into()?))), + (NOTE_NAME_CORE, NT_AUXV) => Ok(Some(Self::Auxv(Auxv::new(description)))), + (NOTE_NAME_GNU, NT_GNU_BUILD_ID) => Ok(Some(Self::GnuBuildId(description))), + _ => Ok(None), + } + } + + fn parse(name: &'a [u8], note_type: u32, description: &'a [u8]) -> Self { + match Self::try_parse(name, note_type, description) { + Ok(Some(note)) => note, + r => { + if let Err(e) = r { + warn!( + "Failed to parse ELF note: name={} type={}: {}", + String::from_utf8_lossy(name), + note_type, + e + ); + } + Self::Unknown { + name, + note_type, + description, + } + } + } + } +} + +/// Iterator over ELF notes in a buffer. +/// +/// Only the current note is deserialized at a time. +/// This prevents us from having to make multiple copies of all notes. +pub struct ElfNoteIterator<'a> { + note_buffer: &'a [u8], + offset: usize, +} + +impl<'a> ElfNoteIterator<'a> { + fn new(note_buffer: &'a [u8]) -> Self { + Self { + note_buffer, + offset: 0, + } + } + + /// Try to create an ELF note from the given buffer at the given offset. + /// + /// If the note type is not supported, return `None`. + fn try_next_note(offset: &mut usize, note_buffer: &'a [u8]) -> Result> { + let note_header = note_buffer.gread::(offset)?; + let name_size = note_header.n_namesz as usize; + let aligned_name_size = align_up(name_size, 4); + let desc_size = note_header.n_descsz as usize; + let aligned_desc_size = align_up(desc_size, 4); + + if *offset + aligned_name_size + aligned_desc_size > note_buffer.len() { + return Err(eyre!("Note buffer shorter than expected")); + } + + let name = match name_size { + // NOTE: As per the spec, the terminating NUL byte is included in the namesz, + // but 0 is used for an empty string: + 0 => &[], + _ => ¬e_buffer[*offset..(*offset + name_size - 1)], + }; + let desc_offset = *offset + aligned_name_size; + let desc = ¬e_buffer[desc_offset..(desc_offset + desc_size)]; + + *offset += aligned_name_size + aligned_desc_size; + Ok(ElfNote::parse(name, note_header.n_type, desc)) + } +} + +impl<'a> Iterator for ElfNoteIterator<'a> { + type Item = ElfNote<'a>; + + fn next(&mut self) -> Option { + if self.offset >= self.note_buffer.len() { + None + } else { + match Self::try_next_note(&mut self.offset, self.note_buffer) { + Ok(note) => Some(note), + Err(e) => { + error!("{}", e); + None + } + } + } + } +} + +/// Helper function to iterate over ELF notes in a buffer. +pub fn iterate_elf_notes(note_buffer: &[u8]) -> ElfNoteIterator { + ElfNoteIterator::new(note_buffer) +} + +#[derive(Debug, PartialEq, Eq)] +#[repr(C)] +/// Time value for a process. +pub struct ProcessTimeVal { + pub tv_sec: usize, + pub tv_usec: usize, +} + +#[derive(Debug, PartialEq, Eq)] +#[repr(C)] +/// Deserialized process status note. +/// +/// This is the deserialized form of the NT_PRSTATUS note type. +/// Note that this structure is architecture-specific. +pub struct ProcessStatusNote { + pub si_signo: u32, + pub si_code: u32, + pub si_errno: u32, + pub pr_cursig: u16, + pub pad0: u16, + pub pr_sigpend: usize, + pub pr_sighold: usize, + pub pr_pid: u32, + pub pr_ppid: u32, + pub pr_pgrp: u32, + pub pr_sid: u32, + pub pr_utime: ProcessTimeVal, + pub pr_stime: ProcessTimeVal, + pub pr_cutime: ProcessTimeVal, + pub pr_cstime: ProcessTimeVal, + pub pr_reg: ElfGRegSet, + pub pr_fpvalid: u32, + // Padding only needed on 64 bit systems for proper alignment. We could + // just set alignment for this struct, but being explicit is better. + #[cfg(target_pointer_width = "64")] + pub pad1: u32, +} + +impl<'a> TryFrom<&'a [u8]> for &'a ProcessStatusNote { + type Error = Error; + + fn try_from(value: &'a [u8]) -> std::result::Result { + if value.len() != size_of::() { + return Err(eyre!( + "Invalid size for ProcessStatusNote: actual size: {} - expected size: {}", + value.len(), + size_of::(), + )); + } + + // SAFETY: ProcessStatusNote only contains scalar values, no pointers. + unsafe { (value.as_ptr() as *const ProcessStatusNote).as_ref() } + .ok_or(eyre!("Invalid pointer ProcessStatusNote")) + } +} + +/// An entry in a ElfNote::File::mapped_files vector. +#[derive(Debug, PartialEq, Eq)] +pub struct MappedFile<'a> { + pub path: Option<&'a Path>, + pub start_addr: usize, + pub end_addr: usize, + pub page_offset: usize, +} + +#[derive(Debug, PartialEq, Eq)] +/// Parsed CORE::NT_FILE note. +pub struct FileNote<'a> { + page_size: usize, + mapped_files: Vec>, + /// The input data was incomplete, so the mapped_files list is not complete. + incomplete: bool, +} + +impl<'a> FileNote<'a> { + const NT_FILE_ENTRY_SIZE: usize = size_of::() * 3; + + /// Parses a CORE::NT_FILE note's description data. + /// + /// Really tries hard to parse out as much as possible, even if the data is incomplete. + /// For example, if the string table is missing, it will still parse the header, but set the + /// path to None. + fn try_parse(data: &'a [u8]) -> Result { + // See linux/fs/binfmt_elf.c: + // https://github.com/torvalds/linux/blob/6465e260f48790807eef06b583b38ca9789b6072/fs/binfmt_elf.c#L1633-L1644 + // + // - long count -- how many files are mapped + // - long page_size -- units for file_ofs + // - array of [COUNT] elements of + // - long start + // - long end + // - long file_ofs + // - followed by COUNT filenames in ASCII: "FILE1" NUL "FILE2" NUL... + // + let mut offset = 0; + let count = data.gread::(&mut offset)? as usize; + let page_size = data.gread::(&mut offset)? as usize; + let mut mapped_files = Vec::with_capacity(count); + + let str_table_start = min(offset + count * Self::NT_FILE_ENTRY_SIZE, data.len()); + let str_table = &data[str_table_start..]; + let mut str_table_offset = 0; + + let mut incomplete = false; + + let mut get_next_path = || -> Option<&Path> { + match str_table.gread::<&CStr>(&mut str_table_offset) { + Ok(cstr) => Some(Path::new(OsStr::from_bytes(cstr.to_bytes()))), + Err(_) => { + incomplete = true; + None + } + } + }; + + let mut parse_entry = || -> Result { + let start_addr = data.gread::(&mut offset)? as usize; + let end_addr = data.gread::(&mut offset)? as usize; + let page_offset = data.gread::(&mut offset)? as usize; + Ok(MappedFile { + path: get_next_path(), + start_addr, + end_addr, + page_offset, + }) + }; + + for _ in 0..count { + match parse_entry() { + Ok(entry) => mapped_files.push(entry), + Err(_) => { + incomplete = true; + break; + } + } + } + + if incomplete { + // Log an error but keep the list with what we've gathered so far + warn!("Incomplete NT_FILE note."); + } + + Ok(Self { + page_size, + incomplete, + mapped_files, + }) + } + + // TODO: MFLT-11766 Use NT_FILE note and PT_LOADs in case /proc/pid/maps read failed + #[allow(dead_code)] + pub fn iter(&self) -> impl Iterator { + self.mapped_files.iter() + } +} + +impl<'a> TryFrom<&'a [u8]> for FileNote<'a> { + type Error = Error; + + fn try_from(value: &'a [u8]) -> std::result::Result { + Self::try_parse(value) + } +} + +#[cfg(test)] +mod test { + use hex::decode; + use insta::assert_debug_snapshot; + use rstest::rstest; + use scroll::IOwrite; + use std::fs::File; + use std::io::{Cursor, Read, Write}; + use std::path::PathBuf; + + use super::*; + + #[rstest] + // Header-only size in case there are no name and description: + #[case( + "", + 0, + "00000000\ + 00000000\ + 78563412" + )] + // Description data is padded to 4-byte alignment: + #[case( + "", + 1, + "00000000\ + 01000000\ + 78563412\ + FF000000" + )] + // Description data already 4-byte aligned: + #[case( + "", + 4, + "00000000\ + 04000000\ + 78563412\ + FFFFFFFF" + )] + // Name data and size includes NUL terminator and is padded to 4-byte alignment: + #[case( + "ABC", + 0, + "04000000\ + 00000000\ + 78563412\ + 41424300" + )] + // Both name and description: + #[case( + "A", + 1, + "02000000\ + 01000000\ + 78563412\ + 41000000\ + FF000000" + )] + fn test_build_elf_note( + #[case] name: &str, + #[case] description_size: usize, + #[case] expected_buffer_contents_hex: &str, + ) { + let note_type = 0x12345678; + let note_desc = [0xffu8; 40]; + let note = build_elf_note(name, ¬e_desc[..description_size], note_type).unwrap(); + + // compare the note buffer contents to the expected hex string: + let expected_buffer_contents = decode(expected_buffer_contents_hex).unwrap(); + assert_eq!(note, expected_buffer_contents); + } + + #[test] + fn test_elf_note_try_parse_build_id_note() { + let build_id = b"ABCD"; + assert_eq!( + ElfNote::try_parse(b"GNU", 3, build_id).unwrap().unwrap(), + ElfNote::GnuBuildId(build_id) + ); + } + + #[test] + fn test_iterate_elf_notes_with_fixture() { + let name = "sample_note"; + let input_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/cli/memfault_core_handler/fixtures") + .join(name) + .with_extension("bin"); + let mut input_file = File::open(input_path).unwrap(); + + let mut note = Vec::new(); + input_file.read_to_end(&mut note).unwrap(); + + let all_notes: Vec<_> = iterate_elf_notes(¬e).collect(); + assert_eq!(all_notes.len(), 7); + + let notes: Vec<_> = all_notes + .into_iter() + .filter(|n| match n { + ElfNote::ProcessStatus(_) => true, + ElfNote::File { .. } => true, + ElfNote::Auxv(_) => true, + ElfNote::GnuBuildId { .. } => false, // Fixture doesn't have a build ID note + ElfNote::Unknown { .. } => false, // Ignore + }) + .collect(); + + assert_debug_snapshot!(notes); + } + + #[test] + fn test_iterate_elf_notes_empty() { + let notes: Vec<_> = ElfNoteIterator::collect(iterate_elf_notes(&[])); + assert!(notes.is_empty()); + } + + #[rstest] + // Aligned desc: + #[case(b"TestNote", &[1, 2, 3, 4, 5, 6, 7, 8])] + // Non-aligned desc: + #[case(b"TestNote", &[1, 2, 3, 4, 5])] + // Empty desc: + #[case(b"TestNote", &[])] + // Aligned name: + #[case(b"TestNot", &[])] + // Empty name: + #[case(b"", &[1, 2, 3, 4, 5])] + fn test_iterate_elf_notes_basic_edge_cases( + #[case] name_value: &[u8], + #[case] note_desc: &[u8], + ) { + let note_type_value = 0x12345678; + let note_buffer = build_elf_note( + &String::from_utf8(name_value.into()).unwrap(), + note_desc, + note_type_value, + ) + .unwrap(); + let notes: Vec<_> = iterate_elf_notes(note_buffer.as_slice()).collect(); + assert_eq!(notes.len(), 1); + + match ¬es[0] { + ElfNote::Unknown { + name, + note_type, + description, + } => { + assert_eq!(*name, name_value); + assert_eq!(*note_type, note_type_value); + assert_eq!(description, ¬e_desc); + } + _ => panic!("Expected unknown note"), + }; + } + + #[test] + fn test_iterate_elf_notes_note_parsing_failure_handling() { + // 2 NT_PRSTATUS notes with a description that is 1 byte short, + // making ElfNote::try_parse fail and returning them as ElfNote::Unknown notes: + let note_desc = [0xffu8; size_of::() - 1]; + let note_buffer = [ + build_elf_note("CORE", ¬e_desc, NT_PRSTATUS).unwrap(), + build_elf_note("CORE", ¬e_desc, NT_PRSTATUS).unwrap(), + ] + .concat(); + + // Note: iteration continues, even if parsing fails for a note: + let notes: Vec<_> = iterate_elf_notes(note_buffer.as_slice()).collect(); + assert_eq!(notes.len(), 2); + for n in notes { + assert!(matches!(n, ElfNote::Unknown { .. })); + } + } + + #[rstest] + // Note header is too short: + #[case(size_of::() - 1)] + // Name is short: + #[case(size_of::() + 1)] + // Desc is short: + #[case(size_of::() + 8 /* "Hello" + padding */ + 1)] + fn test_iterate_elf_notes_note_data_short(#[case] note_size: usize) { + let note_type_value = 0x12345678; + let note_decs = [0xffu8; 40]; + let note_buffer = build_elf_note("Hello", ¬e_decs, note_type_value).unwrap(); + + // Note data is short -- iteration should stop immediately (no more data to read after): + let notes: Vec<_> = ElfNoteIterator::collect(iterate_elf_notes(¬e_buffer[..note_size])); + assert!(notes.is_empty()); + } + + const NT_FILE_HDR_SIZE: usize = size_of::() * 2; + + #[rstest] + // If we don't have the header, the parsing will fail: + #[case(0)] + #[case(NT_FILE_HDR_SIZE - 1)] + fn test_elf_note_try_parse_file_note_too_short(#[case] desc_size: usize) { + let file_note_desc = make_nt_file_note_data_fixture(); + + // Clip the description data to the given size: + let file_note_desc = &file_note_desc[..desc_size]; + + assert!(FileNote::try_parse(file_note_desc).is_err()); + } + + #[rstest] + // Header only -- no mapped_files: + #[case( + NT_FILE_HDR_SIZE, + FileNote { + mapped_files: vec![], + incomplete: true, + page_size: 0x1000, + } + )] + // Header + one entry -- one mapped_file, no paths (missing string table): + #[case( + NT_FILE_HDR_SIZE + FileNote::NT_FILE_ENTRY_SIZE, + FileNote { + mapped_files: vec![ + MappedFile { + path: None, + start_addr: 0, + end_addr: 1, + page_offset: 2, + }, + ], + incomplete: true, + page_size: 0x1000, + } + )] + // Header + one entry -- one mapped_file, no paths (missing string table): + #[case( + NT_FILE_HDR_SIZE + (2 * FileNote::NT_FILE_ENTRY_SIZE) + 16, + FileNote { + mapped_files: vec![ + MappedFile { + path: Some(Path::new("/path/to/file")), + start_addr: 0, + end_addr: 1, + page_offset: 2, + }, + MappedFile { + path: None, + start_addr: 0, + end_addr: 1, + page_offset: 2, + }, + ], + incomplete: true, + page_size: 0x1000, + } + )] + + fn test_elf_note_try_parse_file_note_incomplete_string_table( + #[case] desc_size: usize, + #[case] expected: FileNote, + ) { + let file_note_desc = make_nt_file_note_data_fixture(); + + // Clip the description data to the given size: + let file_note_desc = &file_note_desc[..desc_size]; + + let note = FileNote::try_parse(file_note_desc).unwrap(); + assert_eq!(note, expected); + } + + const FIXTURE_FILE_PATH: &[u8; 14] = b"/path/to/file\0"; + + fn make_nt_file_note_data_fixture() -> Vec { + let mut cursor = Cursor::new(vec![]); + let count = 2; + + // Header: + // Count: + cursor.iowrite::(count).unwrap(); + // Page size (0x1000): + cursor.iowrite::(0x1000).unwrap(); + + // Entries: + for _ in 0..count { + // Start, end, file offset: + for n in 0..3 { + cursor.iowrite::(n).unwrap(); + } + } + + // String table: + for _ in 0..count { + let _ = cursor.write(FIXTURE_FILE_PATH).unwrap(); + } + + let file_note_desc = cursor.into_inner(); + assert_eq!(file_note_desc.len(), 92); + file_note_desc + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/core_reader.rs b/memfaultd/src/cli/memfault_core_handler/core_reader.rs new file mode 100644 index 0000000..f7096b5 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/core_reader.rs @@ -0,0 +1,193 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{Read, Seek, SeekFrom}; + +use crate::cli::memfault_core_handler::arch::{ELF_TARGET_ENDIANNESS, ELF_TARGET_MACHINE}; +use crate::{cli::memfault_core_handler::elf, util::io::ForwardOnlySeeker}; + +use elf::header::{Header, EI_CLASS, EI_DATA, ELFCLASS, ELFMAG, EV_CURRENT, SELFMAG, SIZEOF_EHDR}; +use elf::program_header::{ProgramHeader, SIZEOF_PHDR}; +use eyre::{eyre, Result}; + +pub trait CoreReader { + /// Reads program headers from the input stream + fn read_program_headers(&mut self) -> Result>; + + /// Reads segment data from the input stream + fn read_segment_data(&mut self, program_header: &ProgramHeader) -> Result>; +} + +// Reads ELF headers and segments from a core file +pub struct CoreReaderImpl { + input_stream: ForwardOnlySeeker, + elf_header: Header, +} + +impl CoreReader for CoreReaderImpl { + fn read_program_headers(&mut self) -> Result> { + // Ignore unnecessary cast here as it is needed on 32-bit systems. + #[allow(clippy::unnecessary_cast)] + self.input_stream + .seek(SeekFrom::Start(self.elf_header.e_phoff as u64))?; + + let mut program_headers = + read_program_headers(&mut self.input_stream, self.elf_header.e_phnum as usize)?; + + // Sort, just in case the program headers are not sorted by offset. + // Otherwise the read_segment_data() calls later may fail. + program_headers.sort_by_key(|ph| ph.p_offset); + + Ok(program_headers) + } + + fn read_segment_data(&mut self, program_header: &ProgramHeader) -> Result> { + // Ignore unnecessary cast here as it is needed on 32-bit systems. + #[allow(clippy::unnecessary_cast)] + self.input_stream + .seek(SeekFrom::Start(program_header.p_offset as u64))?; + + let mut buf = vec![0; program_header.p_filesz as usize]; + self.input_stream.read_exact(&mut buf)?; + + Ok(buf) + } +} + +impl CoreReaderImpl { + /// Creates an instance of `CoreReader` from an input stream + pub fn new(input_stream: R) -> Result { + let mut input_stream = ForwardOnlySeeker::new(input_stream); + let mut header_buf = [0u8; SIZEOF_EHDR]; + input_stream.read_exact(&mut header_buf)?; + + let elf_header = *Header::from_bytes(&header_buf); + if !Self::verify_elf_header(&elf_header) { + return Err(eyre!("Invalid ELF header")); + } + + Ok(CoreReaderImpl { + input_stream, + elf_header, + }) + } + + fn verify_elf_header(header: &Header) -> bool { + &header.e_ident[0..SELFMAG] == ELFMAG + && header.e_ident[EI_CLASS] == ELFCLASS + && header.e_ident[EI_DATA] == ELF_TARGET_ENDIANNESS + && header.e_version == EV_CURRENT as u32 + && header.e_ehsize == SIZEOF_EHDR as u16 + && header.e_phentsize == SIZEOF_PHDR as u16 + && header.e_machine == ELF_TARGET_MACHINE + } + + pub fn elf_header(&self) -> Header { + self.elf_header + } +} + +/// Reads `count` ELF program headers from the provided input stream. +pub fn read_program_headers( + input_stream: &mut R, + count: usize, +) -> Result> { + let size = count * SIZEOF_PHDR; + let mut buffer = vec![0; size]; + input_stream.read_exact(&mut buffer)?; + Ok(ProgramHeader::from_bytes(&buffer, count)) +} + +#[cfg(test)] +mod test { + use std::fs::File; + use std::io::Cursor; + + use super::*; + + use crate::cli::memfault_core_handler::arch::ELF_TARGET_CLASS; + use crate::cli::memfault_core_handler::test_utils::build_test_header; + use elf::header::{ELFCLASSNONE, ELFDATANONE, EM_NONE}; + + use rstest::rstest; + use scroll::Pwrite; + + #[rstest] + #[case(0)] + #[case(1024)] // Test with padding between header and program headers + fn test_read_program_headers(#[case] ph_offset: usize) { + let mut elf_header = + build_test_header(ELF_TARGET_CLASS, ELF_TARGET_ENDIANNESS, ELF_TARGET_MACHINE); + elf_header.e_phnum = 2; + elf_header.e_phoff = (SIZEOF_EHDR + ph_offset).try_into().unwrap(); + let load_program_header = ProgramHeader { + p_type: elf::program_header::PT_LOAD, + p_vaddr: 0, + ..Default::default() + }; + let note_program_header = ProgramHeader { + p_type: elf::program_header::PT_NOTE, + p_offset: 0x1000, + ..Default::default() + }; + + // Build ELF input stream + let mut input_stream = vec![0; SIZEOF_EHDR + 2 * SIZEOF_PHDR + ph_offset]; + input_stream.pwrite(elf_header, 0).unwrap(); + input_stream + .pwrite(load_program_header, SIZEOF_EHDR + ph_offset) + .unwrap(); + input_stream + .pwrite(note_program_header, SIZEOF_EHDR + SIZEOF_PHDR + ph_offset) + .unwrap(); + + // Verify headers are read correctly + let mut reader = CoreReaderImpl::new(Cursor::new(input_stream)).unwrap(); + let program_headers = reader.read_program_headers().unwrap(); + assert_eq!(program_headers.len(), 2); + assert_eq!(program_headers[0], load_program_header); + assert_eq!(program_headers[1], note_program_header); + } + + #[rstest] + #[case(0, 1024)] + #[case(1024, 1024)] + fn test_read_segment_data(#[case] offset: usize, #[case] size: usize) { + const TEST_BYTE: u8 = 0x42; + + let elf_header = + build_test_header(ELF_TARGET_CLASS, ELF_TARGET_ENDIANNESS, ELF_TARGET_MACHINE); + let offset = offset + SIZEOF_EHDR; + let note_program_header = ProgramHeader { + p_type: elf::program_header::PT_NOTE, + p_offset: offset.try_into().unwrap(), + p_filesz: size.try_into().unwrap(), + ..Default::default() + }; + + let mut input_stream = vec![0u8; offset + size]; + input_stream.pwrite(elf_header, 0).unwrap(); + input_stream[offset..(offset + size)].fill(TEST_BYTE); + + let mut reader = CoreReaderImpl::new(Cursor::new(&input_stream)).unwrap(); + let segment_data = reader.read_segment_data(¬e_program_header).unwrap(); + + assert_eq!(segment_data, input_stream[offset..(offset + size)]); + } + + #[rstest] + // Mismatching class (32 vs 64 bit): + #[case(ELFCLASSNONE, ELF_TARGET_ENDIANNESS, ELF_TARGET_MACHINE)] + // Mismatching endianness: + #[case(ELF_TARGET_CLASS, ELFDATANONE, ELF_TARGET_MACHINE)] + // Mismatching machine: + #[case(ELF_TARGET_CLASS, ELF_TARGET_ENDIANNESS, EM_NONE)] + fn test_verify_elf_header_fails_for_mismatching_arch( + #[case] class: u8, + #[case] endianness: u8, + #[case] machine: u16, + ) { + let elf_header = build_test_header(class, endianness, machine); + assert!(!CoreReaderImpl::::verify_elf_header(&elf_header)); + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/core_transformer.rs b/memfaultd/src/cli/memfault_core_handler/core_transformer.rs new file mode 100644 index 0000000..906eca6 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/core_transformer.rs @@ -0,0 +1,474 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + io::{Read, Seek, SeekFrom}, + sync::mpsc::Receiver, +}; +use std::{mem::size_of, time::Duration}; + +use crate::cli::memfault_core_handler::core_elf_note::{iterate_elf_notes, ElfNote}; +use crate::cli::memfault_core_handler::core_reader::CoreReader; +use crate::cli::memfault_core_handler::core_writer::{CoreWriter, SegmentData}; +use crate::cli::memfault_core_handler::elf; +use crate::cli::memfault_core_handler::find_dynamic::find_dynamic_linker_ranges; +use crate::cli::memfault_core_handler::find_elf_headers::find_elf_headers_and_build_id_note_ranges; +use crate::cli::memfault_core_handler::find_stack::find_stack; +use crate::cli::memfault_core_handler::memory_range::{merge_memory_ranges, MemoryRange}; +use crate::cli::memfault_core_handler::procfs::ProcMaps; +use crate::cli::memfault_core_handler::ElfPtrSize; +use crate::config::CoredumpCaptureStrategy; +use crate::{ + cli::memfault_core_handler::core_elf_memfault_note::{ + write_memfault_metadata_note, CoredumpMetadata, + }, + mar::LinuxLogsFormat, +}; + +use elf::program_header::{ProgramHeader, PT_LOAD, PT_NOTE}; +use eyre::{eyre, Result}; +use libc::{AT_PHDR, AT_PHNUM}; +use log::{debug, warn}; +use procfs::process::MemoryMap; + +use super::{ + core_elf_memfault_note::{write_memfault_debug_data_note, MemfaultMetadataLogs}, + log_wrapper::CAPTURE_LOG_CHANNEL_SIZE, +}; + +#[derive(Debug)] +pub struct CoreTransformerOptions { + pub max_size: usize, + pub capture_strategy: CoredumpCaptureStrategy, + pub thread_filter_supported: bool, +} + +/// Reads segments from core elf stream and memory stream and builds a core new elf file. +pub struct CoreTransformer +where + R: CoreReader, + W: CoreWriter, + P: Read + Seek, + M: ProcMaps, +{ + core_reader: R, + core_writer: W, + proc_mem_stream: P, + metadata: CoredumpMetadata, + options: CoreTransformerOptions, + proc_maps: M, + log_fetcher: CoreTransformerLogFetcher, +} + +impl CoreTransformer +where + R: CoreReader, + W: CoreWriter, + P: Read + Seek, + M: ProcMaps, +{ + /// Creates an instance of `CoreTransformer` from an input stream and output stream + pub fn new( + core_reader: R, + core_writer: W, + proc_mem_stream: P, + options: CoreTransformerOptions, + metadata: CoredumpMetadata, + proc_maps: M, + log_fetcher: CoreTransformerLogFetcher, + ) -> Result { + Ok(Self { + core_reader, + core_writer, + proc_mem_stream, + metadata, + options, + proc_maps, + log_fetcher, + }) + } + + /// Reads segments from core elf stream and memory stream and builds a new elf file + /// + /// Reads all PT_LOAD and PT_NOTE program headers and their associated segments from the core. + /// The memory for each PT_LOAD segment will be fetched from `/proc//mem` and the + /// resulting elf file will be written to `output_stream`. + pub fn run_transformer(&mut self) -> Result<()> { + let program_headers = self.core_reader.read_program_headers()?; + let all_notes = self.read_all_note_segments(&program_headers); + + let segments_to_capture = match self.options.capture_strategy { + CoredumpCaptureStrategy::KernelSelection => { + self.kernel_selection_segments(&program_headers) + } + CoredumpCaptureStrategy::Threads { max_thread_size } => { + // Fallback to kernel selection if thread filtering is not supported. + // Support is dictated by whether we have a definition for the platform + // user regs. See arch.rs for more details. + if self.options.thread_filter_supported { + let mapped_ranges = self.proc_maps.get_process_maps()?; + self.threads_segments(&mapped_ranges, &all_notes, max_thread_size) + } else { + // Update metadata so capture strategy is reflected properly in the memfault note. + self.metadata.capture_strategy = CoredumpCaptureStrategy::KernelSelection; + self.kernel_selection_segments(&program_headers) + } + } + }; + + // Always copy over all note segments, regardless of the capturing strategy: + for (ph, data) in all_notes { + self.core_writer.add_segment(*ph, SegmentData::Buffer(data)); + } + + for ph in segments_to_capture { + self.core_writer.add_segment(ph, SegmentData::ProcessMemory); + } + + self.add_memfault_metadata_note()?; + self.add_memfault_debug_data_note()?; + self.check_output_size()?; + self.core_writer.write()?; + + Ok(()) + } + + /// Add note segments from the core elf to the output elf, verbatim. + fn read_all_note_segments<'a>( + &mut self, + program_headers: &'a [ProgramHeader], + ) -> Vec<(&'a ProgramHeader, Vec)> { + program_headers + .iter() + .filter_map(|ph| match ph.p_type { + PT_NOTE => match self.core_reader.read_segment_data(ph) { + Ok(data) => Some((ph, data)), + _ => None, + }, + _ => None, + }) + .collect::>() + } + + /// All load segments from the original/input core elf as provided by the kernel, verbatim. + fn kernel_selection_segments( + &mut self, + program_headers: &[ProgramHeader], + ) -> Vec { + program_headers + .iter() + .filter_map(|ph| match ph.p_type { + PT_LOAD => Some(*ph), + _ => None, + }) + .collect::>() + } + + /// Synthesizes segments for the stacks of all threads and adds them to the output, as well as + /// any mandatory segments (build id notes, r_debug, etc.). + fn threads_segments( + &mut self, + memory_maps: &[MemoryMap], + all_notes: &[(&ProgramHeader, Vec)], + max_thread_size: usize, + ) -> Vec { + let memory_maps_ranges: Vec = + memory_maps.iter().map(MemoryRange::from).collect(); + + let parsed_notes = all_notes + .iter() + .flat_map(|(_, data)| iterate_elf_notes(data)) + .collect::>(); + + let mut mem_ranges = Vec::new(); + let mut phdr_vaddr = None; + let mut phdr_num = None; + + for note in &parsed_notes { + match note { + ElfNote::ProcessStatus(s) => { + if let Some(stack) = find_stack(&s.pr_reg, &memory_maps_ranges, max_thread_size) + { + mem_ranges.push(stack); + } else { + warn!("Failed to collect stack for thread: {}", s.pr_pid); + } + } + ElfNote::Auxv(auxv) => { + phdr_num = auxv.find_value(AT_PHNUM); + phdr_vaddr = auxv.find_value(AT_PHDR); + } + _ => {} + } + } + + mem_ranges.extend( + memory_maps + .iter() + .filter(|mmap| mmap.offset == 0) + .flat_map( + |mmap| match self.elf_metadata_ranges_for_mapped_file(mmap.address.0) { + Ok(ranges) => ranges, + Err(e) => { + debug!( + "Failed to collect metadata for {:?} @ {:#x}: {}", + mmap.pathname, mmap.address.0, e + ); + vec![] + } + }, + ), + ); + + match (phdr_vaddr, phdr_num) { + (Some(phdr_vaddr), Some(phdr_num)) => { + if let Err(e) = find_dynamic_linker_ranges( + &mut self.proc_mem_stream, + phdr_vaddr, + phdr_num, + &memory_maps_ranges, + &mut mem_ranges, + ) { + warn!("Failed to collect dynamic linker ranges: {}", e); + } + } + _ => { + warn!("Missing AT_PHDR or AT_PHNUM auxv entry"); + } + }; + + // Merge overlapping memory ranges and turn them into PT_LOAD program headers. As a + // side-effect, this will also sort the program headers by vaddr. + let merged_ranges = merge_memory_ranges(mem_ranges); + merged_ranges.into_iter().map(ProgramHeader::from).collect() + } + + fn elf_metadata_ranges_for_mapped_file(&mut self, vaddr_base: u64) -> Result> { + // Ignore unnecessary cast here as it is needed on 32-bit systems. + #[allow(clippy::unnecessary_cast)] + self.proc_mem_stream + .seek(SeekFrom::Start(vaddr_base as u64))?; + find_elf_headers_and_build_id_note_ranges( + vaddr_base as ElfPtrSize, + &mut self.proc_mem_stream, + ) + } + + /// Check if the output file size exceeds the max size available + fn check_output_size(&self) -> Result<()> { + let output_size = self.core_writer.calc_output_size(); + if output_size > self.options.max_size { + return Err(eyre!( + "Core file size {} exceeds max size {}", + output_size, + self.options.max_size + )); + } + + Ok(()) + } + + fn add_memfault_metadata_note(&mut self) -> Result<()> { + let app_logs = self + .log_fetcher + .get_app_logs() + .map(|logs| MemfaultMetadataLogs::new(logs, LinuxLogsFormat::default())); + + self.metadata.app_logs = app_logs; + + let note_data = write_memfault_metadata_note(&self.metadata)?; + self.add_memfault_note(note_data) + } + + fn add_memfault_debug_data_note(&mut self) -> Result<()> { + let mut capture_logs = self.log_fetcher.get_capture_logs(); + if capture_logs.is_empty() { + return Ok(()); + } + + if capture_logs.len() == CAPTURE_LOG_CHANNEL_SIZE { + capture_logs.push("Log overflow, some logs may have been dropped".to_string()); + } + + let buffer = write_memfault_debug_data_note(capture_logs)?; + self.add_memfault_note(buffer) + } + + /// Add memfault note to the core elf + /// + /// Contains CBOR encoded information about the system capturing the coredump. See + /// [`CoredumpMetadata`] for more information. + fn add_memfault_note(&mut self, desc: Vec) -> Result<()> { + let program_header = ProgramHeader { + p_type: PT_NOTE, + p_filesz: desc.len().try_into()?, + ..Default::default() + }; + self.core_writer + .add_segment(program_header, SegmentData::Buffer(desc)); + + Ok(()) + } +} + +/// Convenience struct to hold the log receivers for the core transformer. +/// +/// This is used to receive logs from the `CoreHandlerLogWrapper` and the the circular buffer +/// capture logs. +#[derive(Debug)] +pub struct CoreTransformerLogFetcher { + capture_logs_rx: Receiver, + app_logs_rx: Receiver>>, +} + +impl CoreTransformerLogFetcher { + /// A timeout of 500s is used to prevent the coredump handler from blocking indefinitely. + /// This is to prevent a slow http request from blocking the coredump handler indefinitely. + const APPLICATION_LOGS_TIMEOUT: Duration = Duration::from_millis(500); + + pub fn new( + capture_logs_rx: Receiver, + app_logs_rx: Receiver>>, + ) -> Self { + Self { + capture_logs_rx, + app_logs_rx, + } + } + + /// Fetches all logs that were captured in the coredump handler during execution. + pub fn get_capture_logs(&self) -> Vec { + self.capture_logs_rx.try_iter().collect() + } + + /// Fetches all application logs that were in the circular buffer at time of crash. + /// + /// This method will block for a maximum of 500ms before returning. This is to prevent a + /// slow http request from blocking the coredump handler indefinitely. + pub fn get_app_logs(&self) -> Option> { + let logs = self + .app_logs_rx + .recv_timeout(Self::APPLICATION_LOGS_TIMEOUT); + + match logs { + Ok(logs) => logs, + Err(e) => { + debug!("Failed to fetch application logs within timeout: {}", e); + None + } + } + } +} + +impl From for ProgramHeader { + fn from(range: MemoryRange) -> Self { + ProgramHeader { + p_type: PT_LOAD, + p_vaddr: range.start, + p_filesz: range.size(), + p_memsz: range.size(), + p_align: size_of::() as ElfPtrSize, + ..Default::default() + } + } +} + +#[cfg(test)] +mod test { + use crate::cli::memfault_core_handler::test_utils::{ + FakeProcMaps, FakeProcMem, MockCoreWriter, + }; + use crate::test_utils::setup_logger; + use crate::{ + cli::memfault_core_handler::core_reader::CoreReaderImpl, test_utils::set_snapshot_suffix, + }; + use insta::assert_debug_snapshot; + use rstest::rstest; + use std::fs::File; + use std::path::PathBuf; + use std::sync::mpsc::sync_channel; + + use super::*; + + #[rstest] + #[case("kernel_selection", CoredumpCaptureStrategy::KernelSelection, true)] + #[case("threads_32k", CoredumpCaptureStrategy::Threads { max_thread_size: 32 * 1024 }, true)] + #[case( + "threads_32k_no_filter_support", + CoredumpCaptureStrategy::Threads { max_thread_size: 32 * 1024 }, + false + )] + fn test_transform( + #[case] test_case_name: &str, + #[case] capture_strategy: CoredumpCaptureStrategy, + #[case] thread_filter_supported: bool, + _setup_logger: (), + ) { + let input_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/cli/memfault_core_handler/fixtures/elf-core-runtime-ld-paths.elf"); + let input_stream = File::open(&input_path).unwrap(); + let proc_mem_stream = FakeProcMem::new_from_path(&input_path).unwrap(); + let proc_maps = FakeProcMaps::new_from_path(&input_path).unwrap(); + let opts = CoreTransformerOptions { + max_size: 1024 * 1024, + capture_strategy, + thread_filter_supported, + }; + let metadata = CoredumpMetadata { + device_id: "12345678".to_string(), + hardware_version: "evt".to_string(), + software_type: "main".to_string(), + software_version: "1.0.0".to_string(), + sdk_version: "SDK_VERSION".to_string(), + captured_time_epoch_s: 1234, + cmd_line: "binary -a -b -c".to_string(), + capture_strategy, + app_logs: None, + }; + + let (_capture_logs_tx, capture_logs_rx) = sync_channel(32); + let (app_logs_tx, app_logs_rx) = sync_channel(1); + app_logs_tx + .send(Some(vec![ + "Error 1".to_string(), + "Error 2".to_string(), + "Error 3".to_string(), + ])) + .unwrap(); + + let core_reader = CoreReaderImpl::new(input_stream).unwrap(); + let mut segments = vec![]; + let mock_core_writer = MockCoreWriter::new(&mut segments); + let log_rx = CoreTransformerLogFetcher { + capture_logs_rx, + app_logs_rx, + }; + let mut transformer = CoreTransformer::new( + core_reader, + mock_core_writer, + proc_mem_stream, + opts, + metadata, + proc_maps, + log_rx, + ) + .unwrap(); + + transformer.run_transformer().unwrap(); + + // Omit the actual data from the notes: + let segments = segments + .iter() + .map(|(ph, seg)| { + let seg = match seg { + SegmentData::ProcessMemory => SegmentData::ProcessMemory, + SegmentData::Buffer(_) => SegmentData::Buffer(vec![]), + }; + (ph, seg) + }) + .collect::>(); + + set_snapshot_suffix!("{}", test_case_name); + assert_debug_snapshot!(segments); + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/core_writer.rs b/memfaultd/src/cli/memfault_core_handler/core_writer.rs new file mode 100644 index 0000000..1b266d4 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/core_writer.rs @@ -0,0 +1,392 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{copy, repeat, Read, Seek, SeekFrom, Write}; + +use crate::cli::memfault_core_handler::elf; +use crate::util::io::StreamPosition; +use crate::util::math::align_up; + +use elf::header::{ + Header, EI_CLASS, EI_DATA, EI_VERSION, ELFMAG, ET_CORE, EV_CURRENT, SELFMAG, SIZEOF_EHDR, +}; +use elf::program_header::{ProgramHeader, SIZEOF_PHDR}; +use eyre::Result; +use scroll::Pwrite; + +const FILL_BYTE: u8 = 0xEF; + +pub trait CoreWriter { + // Adds a data segment to the writer. + fn add_segment(&mut self, program_header: ProgramHeader, data: SegmentData); + + /// Writes the core elf to the output stream. + fn write(&mut self) -> Result<()>; + + /// Calculate output coredump size. + /// + /// The max size is calculated as the size of the elf header, program header table, and all + /// segment data. We take the conservative route here and calculate the size of the ELF + /// uncompressed, even if compression is enabled. It's likely that the compressed file will + /// be smaller, but this at least gives us a worst case estimate. + fn calc_output_size(&self) -> usize; +} + +#[derive(Debug)] +pub enum SegmentData { + Buffer(Vec), + ProcessMemory, +} + +#[derive(Debug)] +pub struct Segment { + program_header: ProgramHeader, + data: SegmentData, +} + +/// Creates a new ELF core file from a set of program headers and associated segment data. +pub struct CoreWriterImpl +where + W: Write + StreamPosition, + R: Read + Seek, +{ + elf_header: Header, + data_segments: Vec, + process_memory: R, + output_stream: W, +} + +impl CoreWriter for CoreWriterImpl +where + W: Write + StreamPosition, + R: Read + Seek, +{ + fn add_segment(&mut self, program_header: ProgramHeader, data: SegmentData) { + self.data_segments.push(Segment { + program_header, + data, + }); + } + + fn write(&mut self) -> Result<()> { + self.write_elf_header()?; + self.write_data_segments()?; + Ok(()) + } + + fn calc_output_size(&self) -> usize { + let initial = (self.elf_header.e_phnum * self.elf_header.e_phentsize + + self.elf_header.e_ehsize) as usize; + self.data_segments.iter().fold(initial, |acc, s| { + align_up( + acc + s.program_header.p_filesz as usize, + s.program_header.p_align as usize, + ) + }) + } +} +impl CoreWriterImpl +where + W: Write + StreamPosition, + R: Read + Seek, +{ + /// Creates a new instance of `CoreWriter` + pub fn new(elf_header: Header, output_stream: W, process_memory: R) -> Self { + Self { + elf_header, + data_segments: Vec::new(), + process_memory, + output_stream, + } + } + + /// Write ELF header to output stream. + fn write_elf_header(&mut self) -> Result<()> { + let mut e_ident = [0u8; 16]; + e_ident[..SELFMAG].copy_from_slice(ELFMAG); + e_ident[EI_CLASS] = self.elf_header.e_ident[EI_CLASS]; + e_ident[EI_DATA] = self.elf_header.e_ident[EI_DATA]; + e_ident[EI_VERSION] = EV_CURRENT; + + let segment_count = self.data_segments.len(); + let (pheader_size, pheader_offset) = if segment_count > 0 { + (SIZEOF_PHDR as u16, SIZEOF_EHDR) + } else { + (0, 0) + }; + + let header = Header { + e_ident, + e_type: ET_CORE, + e_machine: self.elf_header.e_machine, + e_version: EV_CURRENT as u32, + e_ehsize: SIZEOF_EHDR as u16, + e_phentsize: pheader_size, + e_phnum: segment_count as u16, + e_phoff: pheader_offset.try_into()?, + ..Default::default() + }; + + let mut bytes: Vec = vec![0; self.elf_header.e_ehsize as usize]; + self.elf_header.e_phnum = self.data_segments.len().try_into()?; + bytes.pwrite(header, 0)?; + + Self::write_to_output(&mut self.output_stream, &bytes)?; + Ok(()) + } + + /// Write program header table and associated segment data + /// + /// The program header table is written first, followed by the segment data. The segment data + /// is written in the same order as the program headers, so that we don't have to seek through + /// the output data stream. + fn write_data_segments(&mut self) -> Result<()> { + // Write the program header table first. For each program header, calculate the offset + // for the associated segment data, calculate padding if necessary. We calculate the + // offset here so that we can later write all data segments sequentially without + // seeking. + let mut segment_data_offset = self.elf_header.e_phoff as usize + + self.elf_header.e_phentsize as usize * self.data_segments.len(); + for segment in &mut self.data_segments { + let padding = calc_padding( + segment_data_offset, + segment.program_header.p_align.try_into()?, + ); + segment.program_header.p_offset = (segment_data_offset + padding).try_into()?; + let mut bytes: Vec = vec![0; self.elf_header.e_phentsize as usize]; + segment_data_offset += segment.program_header.p_filesz as usize + padding; + + bytes.pwrite(segment.program_header, 0)?; + Self::write_to_output(&mut self.output_stream, &bytes)?; + } + + // Iterate through all segments and write the data to the output stream. Zeroed padding + // is written if the file offset is less than expected segment data offset. + for segment in &self.data_segments { + let cur_position = self.output_stream.stream_position(); + let padding = segment.program_header.p_offset as usize - cur_position; + Self::write_padding(&mut self.output_stream, padding)?; + + match &segment.data { + SegmentData::Buffer(data) => { + Self::write_to_output(&mut self.output_stream, data)?; + } + SegmentData::ProcessMemory => { + let header = &segment.program_header; + + if Self::read_process_memory( + header.p_vaddr as usize, + header.p_filesz as usize, + &mut self.process_memory, + &mut self.output_stream, + ) + .is_err() + { + let segment_end = (header.p_offset + header.p_filesz) as usize; + Self::fill_remaining_bytes(segment_end, &mut self.output_stream)?; + } + } + } + } + + Ok(()) + } + + /// Fill remaining bytes of segment when reading process memory fails. + fn fill_remaining_bytes(segment_end: usize, output_stream: &mut W) -> Result<()> { + let fill_size = segment_end.checked_sub(output_stream.stream_position()); + if let Some(fill_size) = fill_size { + let mut fill_stream = repeat(FILL_BYTE).take(fill_size as u64); + copy(&mut fill_stream, output_stream)?; + } + Ok(()) + } + + fn read_process_memory( + addr: usize, + len: usize, + process_memory: &mut R, + output_stream: &mut W, + ) -> Result { + process_memory.seek(SeekFrom::Start(addr as u64))?; + let mut mem_reader = process_memory.take(len as u64); + let bytes_read = copy(&mut mem_reader, output_stream)?; + + Ok(bytes_read) + } + + /// Write padding if necessary + fn write_padding(output_stream: &mut W, padding: usize) -> Result<()> { + if padding > 0 { + let mut padding_stream = repeat(0u8).take(padding as u64); + copy(&mut padding_stream, output_stream)?; + } + Ok(()) + } + + /// Write to output stream and increment cursor + fn write_to_output(output_stream: &mut W, bytes: &[u8]) -> Result<()> { + output_stream.write_all(bytes)?; + Ok(()) + } +} + +fn calc_padding(offset: usize, alignment: usize) -> usize { + if alignment <= 1 { + return 0; + } + + let next_addr = align_up(offset, alignment); + next_addr - offset +} + +#[cfg(test)] +mod test { + use std::io; + + use crate::cli::memfault_core_handler::arch::{ + ELF_TARGET_CLASS, ELF_TARGET_ENDIANNESS, ELF_TARGET_MACHINE, + }; + use crate::cli::memfault_core_handler::test_utils::build_test_header; + use crate::util::io::StreamPositionTracker; + + use super::*; + + use rstest::rstest; + use std::io::Cursor; + + const PROC_MEM_READ_CHUNK_SIZE: usize = 1024; + + #[rstest] + #[case(SegmentData::Buffer(vec![0xa5; 1024]), vec![])] + #[case(SegmentData::ProcessMemory, vec![0xaa; PROC_MEM_READ_CHUNK_SIZE])] + #[case(SegmentData::ProcessMemory, vec![0xaa; PROC_MEM_READ_CHUNK_SIZE + PROC_MEM_READ_CHUNK_SIZE / 4])] + fn test_added_segments(#[case] segment_data: SegmentData, #[case] mem_buffer: Vec) { + let mem_stream = Cursor::new(mem_buffer.clone()); + let mut output_buf = Vec::new(); + let mut core_writer = build_test_writer(mem_stream, &mut output_buf); + + let segment_buffer = match &segment_data { + SegmentData::Buffer(data) => data.clone(), + SegmentData::ProcessMemory => mem_buffer, + }; + + core_writer.add_segment( + ProgramHeader { + p_type: elf::program_header::PT_LOAD, + p_offset: 0, + p_filesz: segment_buffer.len().try_into().unwrap(), + p_align: 0, + ..Default::default() + }, + segment_data, + ); + + core_writer.write().expect("Failed to write core"); + + let elf_header_buf = output_buf[..SIZEOF_EHDR].try_into().unwrap(); + let elf_header = Header::from_bytes(elf_header_buf); + assert_eq!(elf_header.e_phnum, 1); + + // Build program header table and verify correct number of headers + let ph_table_sz = elf_header.e_phnum as usize * elf_header.e_phentsize as usize; + let ph_header_buf = &output_buf[SIZEOF_EHDR..(SIZEOF_EHDR + ph_table_sz)]; + let ph_headers = ProgramHeader::from_bytes(ph_header_buf, elf_header.e_phnum as usize); + assert_eq!(ph_headers.len(), 1); + + // Verify correct program header for added segment + assert_eq!(ph_headers[0].p_type, elf::program_header::PT_LOAD); + assert_eq!(ph_headers[0].p_filesz as usize, segment_buffer.len()); + + // Verify segment data starts after elf header and program header table + let segment_data_offset = ph_headers[0].p_offset as usize; + assert_eq!(segment_data_offset, SIZEOF_EHDR + ph_table_sz); + + // Verify correct segment data + let serialized_segment_data = &output_buf[ph_headers[0].p_offset as usize + ..(ph_headers[0].p_offset + ph_headers[0].p_filesz) as usize]; + assert_eq!(&segment_buffer, serialized_segment_data); + } + + #[rstest] + #[case(vec![1024, 1024], 1024, 3072)] + #[case(vec![2048, 1024], 512, 3584)] + #[case(vec![2048, 1024], 0, 3136)] + #[case(vec![2048, 1024], 1, 3136)] + fn test_output_size_calculation( + #[case] segment_sizes: Vec, + #[case] alignment: usize, + #[case] expected_size: usize, + ) { + let mem_stream = Vec::new(); + let mut output_buf = Vec::new(); + let mut core_writer = build_test_writer(Cursor::new(mem_stream), &mut output_buf); + + segment_sizes.iter().for_each(|size| { + core_writer.add_segment( + ProgramHeader { + p_type: elf::program_header::PT_LOAD, + p_filesz: *size as u64, + p_align: alignment.try_into().unwrap(), + ..Default::default() + }, + SegmentData::ProcessMemory, + ); + }); + + let output_size = core_writer.calc_output_size(); + assert_eq!(output_size, expected_size); + } + + #[test] + fn test_read_fail() { + let mut output_buf = Vec::new(); + let mut writer = build_test_writer(FailReader, &mut output_buf); + let segment_size = 1024usize; + + writer.add_segment( + ProgramHeader { + p_type: elf::program_header::PT_LOAD, + p_filesz: segment_size.try_into().unwrap(), + ..Default::default() + }, + SegmentData::ProcessMemory, + ); + + writer.write().unwrap(); + + let header = writer.elf_header; + let segment_offset = + header.e_phoff as usize + header.e_phentsize as usize * header.e_phnum as usize; + let segment_end = segment_offset + segment_size; + assert_eq!( + output_buf[segment_offset..segment_end], + vec![FILL_BYTE; segment_size] + ); + } + + fn build_test_writer( + mem_stream: T, + output_buf: &mut Vec, + ) -> CoreWriterImpl>> { + let elf_header = + build_test_header(ELF_TARGET_CLASS, ELF_TARGET_ENDIANNESS, ELF_TARGET_MACHINE); + let output_stream = StreamPositionTracker::new(output_buf); + CoreWriterImpl::new(elf_header, output_stream, mem_stream) + } + + /// Test reader that always returns failure when reading. + struct FailReader; + + impl Read for FailReader { + fn read(&mut self, _: &mut [u8]) -> io::Result { + Err(io::Error::new(io::ErrorKind::Other, "read failed")) + } + } + + impl Seek for FailReader { + fn seek(&mut self, _: SeekFrom) -> io::Result { + Ok(0) + } + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/find_dynamic.rs b/memfaultd/src/cli/memfault_core_handler/find_dynamic.rs new file mode 100644 index 0000000..5cbfda9 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/find_dynamic.rs @@ -0,0 +1,392 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{Read, Seek, SeekFrom}; +use std::mem::size_of; + +use elf::dynamic::{Dyn, DT_DEBUG}; +use elf::program_header::{ProgramHeader, PT_DYNAMIC, PT_PHDR, SIZEOF_PHDR}; +use eyre::{eyre, Result}; +use itertools::Itertools; +use libc::PATH_MAX; +use log::{debug, warn}; +use scroll::Pread; + +use crate::cli::memfault_core_handler::core_reader::read_program_headers; +use crate::cli::memfault_core_handler::elf; +use crate::cli::memfault_core_handler::memory_range::MemoryRange; +use crate::cli::memfault_core_handler::procfs::read_proc_mem; +use crate::cli::memfault_core_handler::r_debug::{LinkMap, RDebug, RDebugIter}; +use crate::cli::memfault_core_handler::ElfPtrSize; + +/// This function attempts to find the ranges of memory in which the dynamic linker has stored the +/// information about the loaded shared objects. This is needed for our backend processing and +/// debuggers like GDB to be able to find and load the symbol files (with debug info) for those +/// loaded shared objects. In more detail, this function does the following: +/// +/// - The function takes the AT_PHDR address from the auxiliary vector as input, the number of +/// program headers via AT_PHNUM, as well as the /proc//mem stream. The AT_PHDR address is +/// the virtual address of the program headers of the main executable. +/// - The function then reads the main executable's program headers, finds the dynamic segment, and +/// reads the DT_DEBUG entry from it. +/// - The DT_DEBUG entry contains the virtual address of the r_debug structure. This is the +/// "rendezvous structure" used by the dynamic linker to communicate details of shared object +/// loading to the debugger. +/// - The r_debug structure contains the head of a linked list of link_map structures. Each link_map +/// represents a loaded shared object. The link_map structure contains the virtual address of the +/// string buffer containing the path to the shared object. The linked list is traversed to the +/// end, collecting the memory regions of the link_map structures and the string buffers along the +/// way. +/// +/// # Arguments +/// * `memory_maps` - The list of the memory regions of the process's memory mappings. This +/// is used to bound the size reads from /proc//mem, when determining the size of path strings. +/// * `output` - Memory regions for all of the aforementioned structures are added to this vector. +pub fn find_dynamic_linker_ranges( + proc_mem_stream: &mut P, + phdr_vaddr: ElfPtrSize, + phdr_num: ElfPtrSize, + memory_maps: &[MemoryRange], + output: &mut Vec, +) -> Result<()> { + debug!( + "Detecting dynamic linker ranges from vaddr 0x{:x}", + phdr_vaddr + ); + + let phdr = read_main_executable_phdr(proc_mem_stream, phdr_vaddr, phdr_num)?; + let main_reloc_addr = calc_relocation_addr(phdr_vaddr, phdr); + + // Add the main executable's program headers. + // Note: find_elf_headers_and_build_id_note_ranges() may also find this range, but that's OK. + output.push(MemoryRange::from_start_and_size( + main_reloc_addr + phdr.p_vaddr, + phdr.p_memsz, + )); + + let main_program_headers = + read_main_exec_program_headers(proc_mem_stream, &phdr, main_reloc_addr)?; + let dynamic_ph = find_dynamic_program_header(&main_program_headers)?; + // Add the dynamic segment contents: + output.push(MemoryRange::from_start_and_size( + main_reloc_addr + dynamic_ph.p_vaddr, + dynamic_ph.p_memsz, + )); + + let r_debug_addr = find_r_debug_addr(proc_mem_stream, main_reloc_addr, dynamic_ph)?; + // Add the r_debug struct itself: + output.push(MemoryRange::from_start_and_size( + r_debug_addr, + size_of::() as ElfPtrSize, + )); + + let mut name_vaddrs: Vec = vec![]; + + RDebugIter::new(proc_mem_stream, r_debug_addr)?.for_each(|(vaddr, link_map)| { + // Add the link_map node itself: + output.push(MemoryRange::from_start_and_size( + vaddr, + size_of::() as ElfPtrSize, + )); + + // Stash the vaddr to the C-string with the path name of the shared object. + // Because the proc_mem_stream is already mutably borrowed, we can't read the string buffer + // here, so we'll do it after. + name_vaddrs.push(link_map.l_name); + }); + + // Add the memory region for each string buffer containing the shared object's path: + name_vaddrs.into_iter().for_each(|name_vaddr| { + output.push(find_c_string_region( + proc_mem_stream, + memory_maps, + name_vaddr, + )); + }); + + Ok(()) +} + +fn find_c_string_region( + proc_mem_stream: &mut P, + memory_maps: &[MemoryRange], + c_string_vaddr: ElfPtrSize, +) -> MemoryRange { + // Read up to PATH_MAX bytes from the given virtual address, or until the end of the memory of + // the memory mapping in which the address resides, whichever is smaller: + let read_size = memory_maps + .iter() + .find(|r| r.contains(c_string_vaddr)) + .map_or(PATH_MAX as ElfPtrSize, |r| r.end - c_string_vaddr) + .min(PATH_MAX as ElfPtrSize); + + read_proc_mem(proc_mem_stream, c_string_vaddr, read_size) + .map(|data| { + data.iter().find_position(|b| **b == 0).map_or_else( + || MemoryRange::from_start_and_size(c_string_vaddr, read_size), + |(idx, _)| { + // +1 for the NUL terminator: + let string_size = (idx + 1).min(read_size as usize); + MemoryRange::from_start_and_size(c_string_vaddr, string_size as ElfPtrSize) + }, + ) + }) + .unwrap_or_else(|e| { + warn!("Failed to read C-string at 0x{:x}: {}", c_string_vaddr, e); + MemoryRange::from_start_and_size(c_string_vaddr, read_size) + }) +} + +fn find_r_debug_addr( + proc_mem_stream: &mut P, + main_reloc_addr: ElfPtrSize, + dynamic_ph: &ProgramHeader, +) -> Result { + let dyn_data = read_proc_mem( + proc_mem_stream, + main_reloc_addr + dynamic_ph.p_vaddr, + dynamic_ph.p_memsz, + ) + .map_err(|e| eyre!("Failed to read dynamic segment: {}", e))?; + + let mut dyn_iter = DynIter::new(dyn_data); + + match find_dt_debug(&mut dyn_iter) { + Some(addr) => Ok(addr), + None => Err(eyre!("Missing DT_DEBUG entry")), + } +} + +/// Finds the virtual address of the r_debug structure in the DT_DEBUG entry, given a dynamic segment iterator. +fn find_dt_debug(dyn_iter: &mut impl Iterator) -> Option { + dyn_iter + .find(|dyn_entry| dyn_entry.d_tag == DT_DEBUG as ElfPtrSize) + .map(|dyn_entry| dyn_entry.d_val) +} + +/// Iterator over the entries in a dynamic segment. +struct DynIter { + data: Vec, + offset: usize, +} + +impl DynIter { + fn new(data: Vec) -> Self { + Self { data, offset: 0 } + } +} +impl Iterator for DynIter { + type Item = Dyn; + + fn next(&mut self) -> Option { + self.data.gread::(&mut self.offset).ok() + } +} + +/// Finds the PT_DYNAMIC program header in the main executable's program headers. +fn find_dynamic_program_header(program_headers: &[ProgramHeader]) -> Result<&ProgramHeader> { + match program_headers.iter().find(|ph| ph.p_type == PT_DYNAMIC) { + Some(ph) => Ok(ph), + None => Err(eyre!("No PT_DYNAMIC found")), + } +} + +/// Reads the main executable's program headers, given the PHDR program header, relocation address, +/// and the /proc//mem stream. +fn read_main_exec_program_headers( + proc_mem_stream: &mut P, + phdr: &ProgramHeader, + main_reloc_addr: ElfPtrSize, +) -> Result> { + // Ignore unnecessary cast here as it is needed on 32-bit systems. + #[allow(clippy::unnecessary_cast)] + proc_mem_stream.seek(SeekFrom::Start((main_reloc_addr + phdr.p_vaddr) as u64))?; + let count = phdr.p_memsz / (SIZEOF_PHDR as ElfPtrSize); + read_program_headers(proc_mem_stream, count as usize) +} + +/// Reads the program header table from the main executable and searches for the `PT_PHDR` +/// header. From the ELF spec: +/// +/// "The array element, if present, specifies the location and size of the program header +/// table itself, both in the file and in the memory image of the program. This segment +/// type may not occur more than once in a file. Moreover, it may occur only if the +/// program header table is part of the memory image of the program. If it is present, +/// it must precede any loadable segment entry." +fn read_main_executable_phdr( + proc_mem_stream: &mut P, + phdr_vaddr: ElfPtrSize, + phdr_num: ElfPtrSize, +) -> Result { + // Ignore unnecessary cast here as it is needed on 32-bit systems. + #[allow(clippy::unnecessary_cast)] + proc_mem_stream.seek(SeekFrom::Start(phdr_vaddr as u64))?; + read_program_headers(proc_mem_stream, phdr_num as usize)? + .into_iter() + .find(|ph| ph.p_type == PT_PHDR) + .ok_or_else(|| eyre!("Main executable PT_PHDR not found")) +} + +/// Calculates the relocation address given the PHDR virtual address and PHDR program header. +fn calc_relocation_addr(phdr_vaddr: ElfPtrSize, phdr: ProgramHeader) -> ElfPtrSize { + phdr_vaddr - phdr.p_vaddr +} + +#[cfg(test)] +mod test { + use std::io::Cursor; + use std::path::PathBuf; + + use insta::assert_debug_snapshot; + use rstest::rstest; + use scroll::{IOwrite, Pwrite}; + + use crate::cli::memfault_core_handler::procfs::ProcMaps; + use crate::cli::memfault_core_handler::test_utils::{FakeProcMaps, FakeProcMem}; + + use super::*; + + #[test] + fn test_phdr_not_first_header() { + let pdyn_header = build_test_program_header(PT_DYNAMIC); + let phdr_header = build_test_program_header(PT_PHDR); + + let mut phdr_bytes = [0; SIZEOF_PHDR * 2]; + phdr_bytes.pwrite::(pdyn_header, 0).unwrap(); + phdr_bytes + .pwrite::(phdr_header, SIZEOF_PHDR) + .unwrap(); + + let mut proc_mem_stream = Cursor::new(phdr_bytes); + let actual_phdr = read_main_executable_phdr(&mut proc_mem_stream, 0, 2).unwrap(); + + assert_eq!(actual_phdr, phdr_header); + } + + #[test] + fn test_find_dynamic_linker_ranges() { + let input_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/cli/memfault_core_handler/fixtures/elf-core-runtime-ld-paths.elf"); + let mut proc_mem_stream = FakeProcMem::new_from_path(&input_path).unwrap(); + + // The coredump fixture contains part of the main ELF executable at offset 0x2000: + // Type Offset VirtAddr PhysAddr FileSiz MemSiz Flg Align + // ... + // LOAD 0x002000 0x00005587ae8bd000 0x0000000000000000 0x001000 0x001000 R 0x1000 + + // The embedded, partial ELF executable has this program table: + // Type Offset VirtAddr PhysAddr FileSiz MemSiz Flg Align + // PHDR 0x000040 0x0000000000000040 0x0000000000000040 0x0002d8 0x0002d8 R 0x8 + let phdr_vaddr = 0x5587ae8bd000 + 0x40; + let phdr_num = 0x02d8 / SIZEOF_PHDR as ElfPtrSize; + + let mut fake_proc_maps = FakeProcMaps::new_from_path(&input_path).unwrap(); + let memory_maps = fake_proc_maps.get_process_maps().unwrap(); + let memory_maps_ranges: Vec = + memory_maps.iter().map(MemoryRange::from).collect(); + + let mut output = vec![]; + find_dynamic_linker_ranges( + &mut proc_mem_stream, + phdr_vaddr, + phdr_num, + &memory_maps_ranges, + &mut output, + ) + .unwrap(); + assert_debug_snapshot!(output); + } + + #[rstest] + // Happy case: + #[case( + b"1hello\0brave\0new\0world!!\0", + vec![MemoryRange::from_start_and_size(0, 25)], + 1, + MemoryRange::from_start_and_size(1, 6), // hello\0 -> 6 bytes + )] + // Cannot find NUL terminator, clips to end of memory mapping region: + #[case( + b"1hello", + vec![MemoryRange::from_start_and_size(0, 6)], + 1, + MemoryRange::from_start_and_size(1, 5), + )] + // Clips to the end of memory mapping regions: + #[case( + b"1hello\0brave\0new\0world!!\0", + vec![MemoryRange::new(0, 4)], + 1, + MemoryRange::new(1, 4), + )] + // Falls back to PATH_MAX if no memory mapping region is found: + #[case( + b"1hello\0", + vec![], + 1, + MemoryRange::from_start_and_size(1, PATH_MAX as ElfPtrSize), + )] + // Clips to PATH_MAX if memory mapping region is longer than PATH_MAX and NUL is not found: + #[case( + &[b'A'; PATH_MAX as usize + 1], + vec![MemoryRange::from_start_and_size(0, PATH_MAX as ElfPtrSize + 1)], + 0, + MemoryRange::from_start_and_size(0, PATH_MAX as ElfPtrSize), + )] + fn test_find_c_string_region( + #[case] proc_mem: &[u8], + #[case] mmap_regions: Vec, + #[case] c_string_vaddr: ElfPtrSize, + #[case] expected: MemoryRange, + ) { + let mut proc_mem_stream = Cursor::new(proc_mem); + assert_eq!( + find_c_string_region(&mut proc_mem_stream, &mmap_regions, c_string_vaddr), + expected + ); + } + + #[rstest] + // Empty + #[case(vec![], vec![])] + // Some items + #[case( + vec![1, 2, 3, 4], + vec![ + Dyn { d_tag: 1, d_val: 2 }, + Dyn { d_tag: 3, d_val: 4 } + ], + )] + // Partial item at the end + #[case( + vec![1, 2, 3], + vec![ + Dyn { d_tag: 1, d_val: 2 }, + ], + )] + fn test_dyn_iter(#[case] input: Vec, #[case] expected: Vec) { + let data = make_dyn_fixture(input); + assert_eq!(DynIter::new(data).collect::>(), expected); + } + + fn make_dyn_fixture(values: Vec) -> Vec { + let mut cursor = Cursor::new(vec![]); + for value in values { + cursor.iowrite::(value).unwrap(); + } + cursor.into_inner() + } + + fn build_test_program_header(p_type: u32) -> ProgramHeader { + ProgramHeader { + p_type, + p_flags: 1, + p_offset: 2, + p_vaddr: 3, + p_paddr: 4, + p_filesz: 5, + p_memsz: 6, + p_align: 0, + } + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/find_elf_headers.rs b/memfaultd/src/cli/memfault_core_handler/find_elf_headers.rs new file mode 100644 index 0000000..6e95953 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/find_elf_headers.rs @@ -0,0 +1,168 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{Read, Seek}; + +use elf::program_header::{ProgramHeader, PT_NOTE}; +use eyre::Result; +use log::debug; + +use crate::cli::memfault_core_handler::core_elf_note::{iterate_elf_notes, ElfNote}; +use crate::cli::memfault_core_handler::core_reader::{CoreReader, CoreReaderImpl}; +use crate::cli::memfault_core_handler::memory_range::MemoryRange; +use crate::cli::memfault_core_handler::{elf, ElfPtrSize}; + +/// Detects whether the supplied stream is at an ELF file and if so, returns the ranges in the +/// stream, that contain the ELF header, program headers and build id note. The ranges are +/// relative to the stream's position when calling the function. +/// GDB requires the headers and note to be present, for debuginfod and build id based symbol file +/// lookups to work. +pub fn find_elf_headers_and_build_id_note_ranges( + vaddr_base: ElfPtrSize, + stream: &mut P, +) -> Result> { + debug!( + "Detecting ELF headers and build ID note ranges from vaddr 0x{:x}", + vaddr_base + ); + + let mut elf_reader = CoreReaderImpl::new(stream)?; + let elf_header = elf_reader.elf_header(); + let program_headers = elf_reader.read_program_headers()?; + let build_id_note_ph = program_headers + .iter() + .find(|ph| ph.p_type == PT_NOTE && contains_gnu_build_id_note(&mut elf_reader, ph)); + + match build_id_note_ph { + Some(build_id_note_ph) => { + let ranges = [ + // ELF header: + MemoryRange::from_start_and_size(vaddr_base, elf_header.e_ehsize as ElfPtrSize), + // Program header table: + MemoryRange::from_start_and_size( + vaddr_base + elf_header.e_phoff, + (elf_header.e_phentsize as ElfPtrSize) * (elf_header.e_phnum as ElfPtrSize), + ), + // Note segment containing GNU Build ID: + MemoryRange::from_start_and_size( + vaddr_base + build_id_note_ph.p_offset, + build_id_note_ph.p_filesz, + ), + ]; + + // FIXME: MFLT-11635 CoreElf .py requires ELF header + build ID in single segment + Ok(vec![MemoryRange::new( + ranges.iter().map(|r| r.start).min().unwrap(), + ranges.iter().map(|r| r.end).max().unwrap(), + )]) + } + None => Err(eyre::eyre!("Build ID note missing")), + } +} + +fn contains_gnu_build_id_note( + elf_reader: &mut CoreReaderImpl<&mut P>, + program_header: &ProgramHeader, +) -> bool { + match elf_reader.read_segment_data(program_header) { + Ok(data) => iterate_elf_notes(&data).any(|note| matches!(note, ElfNote::GnuBuildId(_))), + _ => false, + } +} + +#[cfg(test)] +mod test { + use std::fs::File; + use std::io::{Cursor, SeekFrom}; + use std::path::PathBuf; + + use super::*; + + #[test] + fn test_err_result_if_no_elf() { + let mut stream = Cursor::new(vec![0u8; 100]); + assert_eq!( + find_elf_headers_and_build_id_note_ranges(0, &mut stream) + .unwrap_err() + .to_string(), + "Invalid ELF header" + ); + } + + #[test] + fn test_err_result_if_missing_build_id() { + // A core.elf itself does not have a build id note: + let mut stream = core_elf_stream(); + assert_eq!( + find_elf_headers_and_build_id_note_ranges(0, &mut stream) + .unwrap_err() + .to_string(), + "Build ID note missing" + ); + } + + #[test] + #[ignore = "FIXME: MFLT-11635 CoreElf .py requires ELF header + build ID in single segment"] + fn test_ok_result_with_list_of_ranges() { + let mut stream = exe_elf_stream(); + assert_eq!( + find_elf_headers_and_build_id_note_ranges(0x1000, &mut stream).unwrap(), + // Note: see offsets/sizes from the readelf -Wl output in exe_elf_stream(). + vec![ + // ELF header + MemoryRange::from_start_and_size(0x1000, 0x40), + // Program header table: + MemoryRange::from_start_and_size(0x1000 + 0x40, 0x2d8), + // Note segment with GNU Build ID + MemoryRange::from_start_and_size(0x1000 + 0x358, 0x44), + ] + ); + } + + #[test] + fn test_ok_result_with_list_of_ranges_mflt_11635_work_around() { + let mut stream = exe_elf_stream(); + assert_eq!( + find_elf_headers_and_build_id_note_ranges(0x1000, &mut stream).unwrap(), + // Note: see offsets/sizes from the readelf -Wl output in exe_elf_stream(). + + // FIXME: MFLT-11635 CoreElf .py requires ELF header + build ID in single segment + vec![MemoryRange::new(0x1000, 0x1000 + 0x358 + 0x44),] + ); + } + + fn core_elf_stream() -> File { + File::open(core_elf_path()).unwrap() + } + + fn exe_elf_stream() -> File { + let mut stream = core_elf_stream(); + // The coredump fixture contains part of a captured ELF executable at offset 0x2000: + // Type Offset VirtAddr PhysAddr FileSiz MemSiz Flg Align + // ... + // LOAD 0x002000 0x00005587ae8bd000 0x0000000000000000 0x001000 0x001000 R 0x1000 + + // The embedded, partial ELF executable has this program table: + // Type Offset VirtAddr PhysAddr FileSiz MemSiz Flg Align + // PHDR 0x000040 0x0000000000000040 0x0000000000000040 0x0002d8 0x0002d8 R 0x8 + // INTERP 0x000318 0x0000000000000318 0x0000000000000318 0x00001c 0x00001c R 0x1 + // LOAD 0x000000 0x0000000000000000 0x0000000000000000 0x000648 0x000648 R 0x1000 + // LOAD 0x001000 0x0000000000001000 0x0000000000001000 0x000199 0x000199 R E 0x1000 + // LOAD 0x002000 0x0000000000002000 0x0000000000002000 0x0000e4 0x0000e4 R 0x1000 + // LOAD 0x002db0 0x0000000000003db0 0x0000000000003db0 0x000268 0x000270 RW 0x1000 + // DYNAMIC 0x002dc0 0x0000000000003dc0 0x0000000000003dc0 0x000200 0x000200 RW 0x8 + // NOTE 0x000338 0x0000000000000338 0x0000000000000338 0x000020 0x000020 R 0x8 + // NOTE 0x000358 0x0000000000000358 0x0000000000000358 0x000044 0x000044 R 0x4 + // GNU_PROPERTY 0x000338 0x0000000000000338 0x0000000000000338 0x000020 0x000020 R 0x8 + // GNU_EH_FRAME 0x002008 0x0000000000002008 0x0000000000002008 0x00002c 0x00002c R 0x4 + // GNU_STACK 0x000000 0x0000000000000000 0x0000000000000000 0x000000 0x000000 RW 0x10 + // GNU_RELRO 0x002db0 0x0000000000003db0 0x0000000000003db0 0x000250 0x000250 R 0x1 + stream.seek(SeekFrom::Start(0x2000)).unwrap(); + stream + } + + fn core_elf_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/cli/memfault_core_handler/fixtures/elf-core-runtime-ld-paths.elf") + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/find_stack.rs b/memfaultd/src/cli/memfault_core_handler/find_stack.rs new file mode 100644 index 0000000..1cf5c18 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/find_stack.rs @@ -0,0 +1,133 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::cmp::min; + +use crate::cli::memfault_core_handler::arch::{get_stack_pointer, ElfGRegSet}; +use crate::cli::memfault_core_handler::memory_range::MemoryRange; +use crate::cli::memfault_core_handler::ElfPtrSize; + +use psm::StackDirection; + +/// Attempts to find a MemoryRange for the stack based on the supplied register set. +/// The returned range is bound by the max_thread_size and the end of the segment in which +/// the stack is found. The assumption herein is that an anonymous memory mapping is created as +/// stack (and used exclusively as such). If the stack pointer is not contained in any segment, +/// None is returned. +pub fn find_stack( + regs: &ElfGRegSet, + mapped_memory_ranges: &[MemoryRange], + max_thread_size: usize, +) -> Option { + let stack_pointer = get_stack_pointer(regs) as ElfPtrSize; + let stack_direction = StackDirection::new(); + + find_stack_inner( + stack_pointer, + mapped_memory_ranges, + max_thread_size, + stack_direction, + ) +} + +fn find_stack_inner( + stack_pointer: ElfPtrSize, + mapped_memory_ranges: &[MemoryRange], + max_thread_size: usize, + stack_direction: StackDirection, +) -> Option { + // Iterate over all PT_LOAD segments and find the one that contains the stack + // pointer. If the stack pointer is not contained in any PT_LOAD segment, we + // ignore the thread. + // + // NOTE: This is an M*N operation, but both M(#segments) and N(#threads) are + // likely quite small so this should be fine. + for memory_range in mapped_memory_ranges { + if memory_range.contains(stack_pointer as ElfPtrSize) { + let (stack_start, stack_end) = match stack_direction { + StackDirection::Ascending => { + let stack_size = min( + stack_pointer.saturating_sub(memory_range.start), + max_thread_size as ElfPtrSize, + ); + + let stack_base = stack_pointer - stack_size; + (stack_base, stack_pointer) + } + StackDirection::Descending => { + let stack_size = min( + memory_range.end.saturating_sub(stack_pointer), + max_thread_size as ElfPtrSize, + ); + + let stack_base = stack_pointer + stack_size; + (stack_pointer, stack_base) + } + }; + + return Some(MemoryRange::new(stack_start, stack_end)); + } + } + + None +} + +#[cfg(test)] +mod test { + use super::*; + + use rstest::rstest; + + #[rstest] + #[case::stack_ascending( + StackDirection::Ascending, + 0x1500, + MemoryRange::new(0x500, 0x1500), + 0x1000 + )] + #[case::stack_descending( + StackDirection::Descending, + 0x1500, + MemoryRange::new(0x1500, 0x2500), + 0x1000 + )] + fn test_stack_calculation( + #[case] stack_direction: StackDirection, + #[case] stack_pointer: ElfPtrSize, + #[case] expected_stack: MemoryRange, + #[case] max_thread_size: usize, + ) { + let mapped_regions = program_header_fixture(); + + let stack = find_stack_inner( + stack_pointer, + &mapped_regions, + max_thread_size, + stack_direction, + ); + assert_eq!(stack, Some(expected_stack)); + } + + #[rstest] + #[case::below_regions(0x0050)] + #[case::between_regions(0x0400)] + #[case::above_regions(0x3000)] + fn test_stack_not_found(#[case] stack_pointer: ElfPtrSize) { + let mapped_ranges = program_header_fixture(); + + let stack = find_stack_inner( + stack_pointer, + &mapped_ranges, + 0x1000, + StackDirection::Ascending, + ); + assert!(stack.is_none()); + } + + fn program_header_fixture() -> Vec { + vec![ + MemoryRange::from_start_and_size(0x0100, 0x0250), + MemoryRange::from_start_and_size(0x0500, 0x2500), + ] + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/fixtures/elf-core-runtime-ld-paths.elf b/memfaultd/src/cli/memfault_core_handler/fixtures/elf-core-runtime-ld-paths.elf new file mode 100644 index 0000000000000000000000000000000000000000..4b6d6cb2c4c6d8c9f0c39b1f5c16c5359dd5adcc GIT binary patch literal 319488 zcmeHQ2Vhji*1jn~G>|Ncf_+s~EQElFK}0t+Hx!K&#q!)#AR3aGg0Zr6VnAcZ9u*az zEq3fHSWr=e6?;^aAnJ<^2`c-4=gv8s-Afh%g73Zmoq@?W=gc{0X1l7x06{!u|*UifM>IdeG3o~$W z*@5KOj;d0*zQN(gYO57LyFKx1+D1QXUYq-AQ{v$cxwg_TQ~X{}E1D{U0!se{U8XL? zr=M*-4nMZDvSN)vn4isO4g1x;5!oKAt<2pv@w539zosu+_e-^-TQVP^eu|&FG1AY& z4GM$4nwZq^Gr{hGMm@9>j&f?(ZBHZVvpF(esvB%ZqCZwbY0tgil4P1k{9u-<$|-H;#ceNbF;l*n4dY`xgUY8 z^poSV?#>Y#!;g32D}<(fV&5KLiT&5I>K=;p^B6lzFsz?8pLOil&K4 zXJrK`v32P9mU$DTwld*JzJpI1J(TPd=?|yyvgmd}f$SxVep}JabFsLS*@F}AygP!c zhC7VBej)2~n(VcbwaOgxwckXpxqm%(|7X+}XFTzzep9ju!ks6% z-NRY$>FU_CSsx9g9pZQ{avZX7Y>j5G4j#MEh)T*%ZYO6K6tSfZy~-z-^dhA@T)XXM z>YE=qA&!~?X7H`C-sTeBpt8Pr~M@a9}hBDP4+?Evb=-8S9c{-R);D& zvwt2V%>^Y7pV!DylV;TB90p}MJ>pjb%i?`8f$ao#;zA~$?+Ax9r<6Jxu;-YH5?AVz z9QF1fSVyo2m#@Fe90Pl@%)6PjAIm>HZ|aj-zK88u3;}Umfk6))nK4W_?jp6kr&w~% z=RAiKna|N=omioBHggHxsJ$uE_1N-SI%wtT;)RP zkes{lxyYNMXW8{p+P zp4XJ+eC-U$=Cw+?X~}CPzxIWAVA({quVL?$vWzUI46{O5fDJxFtXZ8ATvYFh5KL2OgEVD0$FxMI`~RIR zk9CmM-M$V^*S5vZi7WX{m7rYLrpqtn*Nv~kI8i=dASl-+3FTTkikl=aUzit^`%vU( zUk8$ls}8)L8xRclv)glXQx06kYQbPlTtV^B9R5+^Y)C5+-nBKZZ_FTj3+_C+iT z#eTvk^^tNK^XWu4>L|JGzn10n8_#}D9K!a3)CQ$7Uzg`?OM98@m*BR;SWY~`s4YQh z?_~J#)Uw6qL8f;!w-;?5 zeG>|bvI`3?$hRVlW3W_WY8DWV%60 zbL~d1*s}n);^M67C{v1oPbE;+o2iet>q+DR^a%Uz2XMYxjdcIb!!R+>6|tVSPS$5y};f-27`-% zl;pEqF&B)sVk%uRdv)xPP!rR^MS6PAzW!gqVEFle>-H9|AQ<}vuy>WW|7bN#9MG|GF_)OfI$WnD4yFJ%y5YjV$*PmT~>g9REf} z`Cd~umib%9R<{-AyCrh(qA@x0J3wMD?=$7Izw$k&)b~yMqAz*R$kOisw%>N`|R<`y%xL;wdIZ+znNv}H#f80An;I>?`v}l{(axFzso51j~m`L zJi3>xH{XZl?>b6{T=JbMQ79?{%78MU3@8K2fHI&ACcAvES+6+$ul#Z6)OKvGLXsZ=6`bV z_Mqhs{J)ohCOt0yy%V6V+YHciVkt*WuC06Kanv|m;cUx!G}i6DecRhofBxe4VN1XUDGO&yCc-X3iJZ`wUwAyySr! z&2z--RvP`-UmedC|F;vN9Z&|80cAiLPzIC%Wk4BF29yD1Kp9X5lmTTx8Bhk40cAiL zPzIC%Wk4BF29yD1;Qu!R{QUpW;|W5~|1UF7H1P27#}APwNmv#=|33+TJ=XE|bI9qL zfnw)G7e^M9=Pi(${AzYKPn2+?JXb!^T{t-N*fbvR|(2(U#u@h>v-8{$1bS zd)(0eH|LzX>#6tdGWO&nsqYknVn+L2fM(3sS9n;;^LSv%KT(c@aUvGZfmk1xK>s9y zT!Ea47*|q2*gXQ9-!XA_K^%k!udGFR2J%95_EY3D5NCJDFQA^DfATVHt|ecj6x1(8 z+@~S;L_1%hod*$fXZTEqT`BZQn5@0w|1#2N#?e!%Y`UkaOa)ISd0RM_-Ie;xo1 z1XB7M`Ucothj>d6>pGMlLjC8cdkne*AwLHFm9YB|vA!icl!qbywdfP2mr*|rW6g*B z59nt=o`t*tcDulzQaSwkqWmJpHv_o}wwJ@U2IVh-N5TG7;@&qv+4Xp=%chdN3tV0#Yq1CS}LhuuHX#$~9lMH?q0_eP&+ zy&nOc8+NZCMlx)=qV6Z?_lLe4f_{pA(Sh|TFft2<~A1V>&O2}`)=WDce z4f5^qI~(=u5%(qd9YAqmQw;k9(f%=rcQnesLiYgBg|>^Ke-CzVp!^+teuR7<@Hpr) zkbg(q(UAX%_A*hv1iEG1k1^1l3HepD+XH^jBmR|;Uqk*7KCi%jAGC7;Vy7Wz!R~t4 zbO*i$-}NZ}6MgM~x_FfP!S*5K-iSr%YU&I0&!WBctUC>UgOL-V%SZhE(AF)ey9ziJ zZQg-+H^FBbVwOXuGz&3rg#DMuH=u4c?EitfgW+=?`a+35qjwzQ9R}M9_&$g>zDAiI z?E4t>pTl+mY%YYn0rKgHwHh(zBEJdSYY_K0)PD}&64>~W`@rT|#J?PP1mxQh;|chF z3HtzSUq)Z|MjfSxp^JrWU&J{I`ZVML$VuqKmGHe6^+O?_0(l%_oQzoAP`(tpTM^?K z#5fLpSqgj!`Bmhzp??Rs3u1l+`;Xx_75%#a<-JgM3FONWmr@S&_aoj2=-xqHf9f}4 z--kRAag$LuA2w;Q9g6aKC^rH>gWVs9e=q#5gzi@48(|NyZiese$lpT#5^bId`vkO? z0H5|4Qzw*X!0r>s-C>i2@`jzo;qyB3D%gxh-W~cg5&sd` z4gp>Z?1TDS;d3$ME~x(r?XG}+Bw{~^_R|pib@`P%g6?ThYzdP#O zsLMpGhfuc>KC4l8ALI$Zb+EY~_Fo_;LbncmdksE&0JGuu1nQ53j|+A8qb)jXS^*pc z{ZG*40@ovM3hbw#E*bK^h`Sf^J@C5-`CXL%fx11>ze;#<$F)HEnAmYCd-8Zl~8~P)lKNxnk&`(89K;4tTM-jIcefkM?;}Gji*q#df2JJis z-2&L$ju@vwrc?#_Tlma|KmBgjdB``zt}oiW0sX5$c`a-7R z4eTh| zKhFK^jkAjWN6ooR92+{YQ|ypuQ{cZP*_O-Ajly6ZNxD{t5X3jPYE=TMIi% z@1u=pP+tUnSLhd^y&Ux8G?dRm-MP^JjCwz^2e~u!vFtk;^-Ezp4*4;(GYoh%Y%)-P z3hI_YJ{0z+Lv|z1Iml^{XR?21w6!PX8rby#50w5yoLx|V49^egjzs-K==*E%f1CAt z!R`U*`=Wj|+UkY6o6+{e#2<14?B_r}5c&^L{~T-&2c7{eLYtq!W)^Y|bhFW3N7%1H z%pV}{guZ+O`2^HIi+l;>8{pRtZQlT!8Hh`#)K^1)BJ3#DA?7OB?1Z{%;Lfo79Cg`{ z$0L6Qc`WSCM9xMz5x%D*_Nl152)ZQn^GnFzBhK6KD}v20uo(c|RMgj^?cbmqhPo2u z{h*6MJI}-4MI6G`1zSpkpnn>={Sk9H^qrCEP05pp@hi&9G2S_du>{x&!j>gOT92mNy7(Z~g`qtpR0ni01h;v^s@y+F|0%}WsPERGn*F(5 zpUJYHv5v8U(e~5t`Eu}IKgQ^V|B~BH|Ahb4bW!8*n!Mi2W7{|!9O zCdRh%zpeex@|u2bssDB3jGgC1gC#=^)<0y%mCE{U?Y}k0#A^tccumteKJ(u&+sMxK zpE}p*rT>3*{cmgk>;7f>UHd-svDRQc9~U)T?_B?NO~x)-|F?Gi|JDB2aKBUk%la?t z-`&Bi2Uj~@Kg_@EKmJ6cZ#c-RBnw)XsCTlb&P z{@;E3^nY8=f3~^*{6g(M#PmPf>vq<`o3D@Nx1;s4|H%1I?Fpuz&ht+>|FQVIq0VFM zLa$%EA3D!}{;vH~`tRZMv}paW=kpcU=+^u1=kpsme~Z@t?Y)0W|F?AhDX(9D_55vn zy?)8@yS?_G9ew;}9Ub|cLg(=nXy*5U>-qiOI)m=t4O(2E%6jp4i_94K)!2)j-(#Nd z;%B}1Iq{-LSGD>+QS6$IG5x2r-& z{)Ev#%(BFtlx}$4dx*gjmMaUdo29u@q&PTY)v7!xOp(PFH%jNZf8 z5H#z=&*)|hY&7~TM*p8i4)8jXj(9RRIUc3!ViJK{IzGghjt?s1)|;Ik<#qEE<9j!o z^&}|kd28dRaD0igwfO8UrXSmiqZV(Qzq6m)+WxlsRUBf7d;Ym!?J07~b9ujHdP5-d z6jC?}TRVSz8`x@mGIBi#KQh14%=T{oRt_BCb@?LWGREbM^^ET_u4Md#@hiq}8GmB@ zgRvd2_l}IaG48>*599uf2QePX*psn0<57(L83!_^GY(}O#dr$inT!%YhvkWkg^Xp4 z7c+ zS@ttt&6vr4m5d7*1B`VNhq00Mo?)ilNsKj&X*}*NVLi|1G0&vhuM=;x{4V4B4q4`J zCF=v+o*c(JdY$fQo_FJM?Wp7UB3?&3Y8}dPCb+fxH$Hk}r$hOyfj)yqN#@r`Eq}K* zK0cyjrzYlE*3*`7d$ljW2OC(^`s-7)e#nTU)jRdl->v0G&aZ-7;AdOo)91t}$?+|5 zjCRw%Tibv7oI9nR_<Md3N*rn}b+?Bx5?`NsPkdKOC~?1+%$6pRwjcKELL6OfT@c zye+){6FWJ--BlF-X3YAG-xqw;`naTfG@tgGUTf?czBMTI@)0=Ct!BNuPl$YfAjkVq zuJ<_VWxb5z`YcDiytx>|^?paa^h4%NWWka*&3)P|#)dbHEcMpyrr!ySZpLF6hco6d zRxn=9co*Yh#^sE4Kbp9ov%HQm;SQtk!gv_tAjT6JCo!JKIEV3C#=99`VqD4XeaLbn z<8O?~3r)LwGxlLjXI#(zqgXzNaTenO#(NoGWOV;z;tHQDS^qO*`#Vkj0gM9}hck|6 zJeTn@#(NkSGrr2Wit#7L-x*zZMfPJKmisUcV;s*ofpIG1&5VyQzR36y|HSe} z#+~jn>t#2_0~n8B9LPA7@ifLF#uXW^7Uk=TTiYRy_e;UY(I`=IWIlkVPDMhA8cRFvYh8Oxy|(w ziT_tI?E}OASjKF|3m6-_Z@#^(uRpWbMJi(nqn|PB8&e;xpMlk;-ou#6Si+dan8_F| zj%%ZdBkR2OPb15`Mf39{n9pyzIvDgb)-$Fux*45uQg=0ewaEqpoeh?_3|fr!Y?qb9 zc8oQ<7}>-A&iJl0Q=i$-po=k;F^e(4==9GTZ2aoe4gT_=`Mj+Y-GfZM(?2l8=xZ1o zI8G+Z7W+l>PaR?WJtGbJS*|^i^(;Hv71=e$*tr?A#u_=rZZv4!WUz+O&FEotw%_!$u@5k2J!51yW5ZKMZesMXT|LV+;>YNWZ!KZ} z0Jrmk!AzEGx!%v{VSNe9nQULfdT0FKuQ&Uf|1-1hr9K{s%856a`oLGM*B`|7ncucv ze>B(o|J8c^XIl#YX>HM8bxqi2!G1TuZ z6!3aDl}u`QUO#=r%%5}qonh!;p-hWHUSpj$ZIPvYX;=D@@n1-4~+z7Hii?`!m#btdPH>$%}B#>{@+6BWC% z++)5up5=apHOt8O%CVj386}kH>&KL&J)C#i{U3k4X$j}Kw#>KZ?dJoLy^J5{wRU}g zAAW=RwPoS8iRF0{P5a%+8OeR6kqg2A6TXZv1<^fo!ACly;{Wx$<8I6DAp4#C{%bZ&|LBy^q+0&UEGgG?*Bq=C!XyrZr^{gv7h_C!NnIDZ2Hz<$yw(5 zx|`dpJN*uXug zJRphz?e7ou-|hOZp1E`PHx`yp-1p>|ey#Qrb6(OgozH7dU1E#uIBDg(-ZGN2471IoaEF9TVHk>_3XKB=|j|H521 zz2s5zen;?(=kFe|aVoy&%qYGRhc7W$>sLp%Bj=3?JsqEC`5(VO=;!+Y!lM&s|4rxr za(!OrPv$GspD5@5e!g!IvTI{O&i`lrP2-dMAhJ%K<8wtDUqR$LZsYJQ8K2Cr@GJe0 z`y6$Tn(NzbjjMKlF9Yirn*C1J&EMO)wx$dy1ImChpbRJj%D~n$(AM)7vE$q4)>}th z9-FspKl*7RP0D~WpbRJj%78MU3@8K2fHI&ACNEbdkcK7M(zdqB;;3-hrxd=>ZSlG-GTB2C?AfxonZS7a250g z(B;||mQ?{8N~fUybm)fxDXnKD-YI23z6|x0YPpWe#~@$FdXg_fyy>VPN6@%YOT? zj_S{)KS*h)KMeXyVN-|lYv|J}C=WpS9%Ewvox(Bh1a3s#2hjRaz8H8O;Jd&Y*mMRChkO(8ePl{a&{Mh+c&RCd z{_ci&D`0;KW4krL@-*k1tMVaWTV{&b_Z;vvsOZbW?{&g8pzW{{;Kq zF!&1jLhi>S@V%AGVdB9KEb_UAtAU}g#gghv$ zPp82*09{O&?hJ0Po@17U)xp$C=K5ne{z1S)*!El0x8pL|E<#-j+mU=dm-{2;(P8>@ zF4u5d*RXsl;|7bD;)kPl&*UPG(7-CC6IhJ6m|4?w2$Pso%i8INInn33wPfbG7>pOV1oD#Tom zaUY2M19XQYKg%)uFz$;wI#KzB+aDJe^Lo@tI*awwp}P|0Tf*wdCJ*s$g8m-p+W`-P zZY?_>2>EElFJK>frO$`4H4Cg61*c$H=SE#wpNW#kLP{yw8AF zpuPy%nJ#6UQSe=fd?*Y&$k#wW0r_F*$n9X{bC7c((}_uc;Op!QbtvtF(%Y=hhYh6? z$V1?Bn-Q(K(7%eDi@=xiI15m&L_M9)ypK%hPTxa!67q1Ae}wKe;4i@6pp$e6x6uRm zIO;y6I4Iu;?8onZ4V>LvY1zNl*lJ*86(*nd9}AJ%ywuZ3L&+WHB)i-8A0?gsoB*ai6)#G3^< z3*|e3iNJoy{gEj(A>RZWddo5x`C;TF#QhU^H0*anc@N~J+{bg__cF>yL3bnS-{CkX zKz<$iyMV_)mkj?GA%6<_GL*-|meLcb`x81!Pb05H`4-e&1O0W}#>c?n9QSS3y#U*K zl+77CdE=bPfb|*lOJILK@?gZf3i9nJ|Bmt-zzdO^p_>o<1wIF&?B@O~fbIpz$*_F_ zy6LDdg-j_0x)~^c3Y*r_D(EM`=U|RC9!B}_$pGHVJ|o!ibLe|PcR%ZoW<~7Kx7` z*CN&vux-zF*Fr8qPCy&ZwvK3GxEy5^V|d z%lbJi?*^a ztlI^$J3Xn}pd{Czq=dmsUta!3?6RaDGuq9LpBFbjHnuS-fpjG;D$Cj%KHpt}eLkf* z_{mNXGLqb}b9PQj@ES(lP<5P&^RBe!U=qElO;2*oh#ivTiJh`@k}EwaVMwAyHmR^# z=xGjaq__EE6E`9TMnjVlMkF2q^;p>Wz0JYP*=BYuIy)l{!Q!riVK=xTV7lEft)ZE) zZ#bzr*hKai)cn{%N!{kb-aR{h2n}LLQdZn)u{$UANKa}RxACXe11h9uI@^yAOO<{<59l1P7(ALsoA&A}9h z{!!>X=QIatua-pi%l$Y;CN~Eg9qq1yzNEN0DEkpi?f(A({i1W5gG1Vr#w1!NU8Z3^ z&T9@nNd3et#J-PpGR&-#5~%v3zU0E@;Aon!3qtdS$(bFWPV*IewpoF(l`Ts!_RX-l z8)~oF#bLD{CAl+XMQ2d$_?F5^Xl4=GJac+;@Gfd|N^5OS;5Pj&jgT5s9&}W#~JYmo?6Bidr4>;7@la6 z?OfR2x1u@NZ4=wL3uw<6YVR4tZQdS*ZS6l&pPyDp)dIvdt@ApIFWrJ z^y7bP4qhxuU>@>J=mXGyuq%XNNv_Lj|GO-1P?CFo{Gg;B^V+2+rOs}jo|HDD!;qws zxI@v?G^o&Zn84J$4va{QJE(oDHMCPwJKv$5w8Ra;5yZ@v+v(GKJA)GAu8wQ9ok85r z*jeCv_YJ|pHeVq99gR%>WxXgtrLow@D}!9H!{t$loyXmPK$UNaQ! z(Cbqhg|^>y@JmmOeZn4xG42cKG6(y?fDOSTX|JXDi(ueQ zjN26xl`t-`9Dj!-4#VHkiE)Xf8xNpU1hb4ipxSp_eosyI= z)aHzoLleDFWh5nFvQNfxbB)CL46ROU2=Yycw30*`(J2-sY44Aq>3`DmgVeGSO%UTY)QXZ>g@ zf-Flvn^=Ck!n9BOJEg2yMn2Gpq4w$eA|>gkk#_uFKRQmSvBdB!E!IA@!REb*^@39G z-}f!M-c#9Vquj4a&q(f_Rh* zGG$^h3g!8g_&Whw7!~JbmuFik`ICGTO0y^D`zGZfoJdv*x%#q8OS3T)@{lsx%AQ=1 zi;fhRQ)Kkoj+0YXW_QkKb5vMRbdK=(SCRG?+7o1+=)M&tLANcn{Ba(Kbo__=FJpf} zIld)Ayc8JezF4^bckD0dcGx@p2cw?$RO!EqH%P&&4=@2_pOyYgoa0fw4ttXLFX9bH zaLB=}`x~vf&Pp7Lf71w4BUpQ`k!g(d+9d7E{(3TW)MfFnc@_eapu7$kt-NZ9WLc-7 zh;1s=|F^$3b%K?=q09KC9l;4Or#)R{+2;j2as~CD_I&BTX$9B)QJ_6v{AJ%4r0Y$T zWR9KwGf_u#A^!FJx*;g9OVWSw7yjtFKKTzqrWBx?5Mfx$fytjgU*g=33Yu5xU&{=cGZA_H6@OWGB46e3ciYkM{*#gJH-sk_3xbOvhqvFv z7c&K=pAti4!G|3FRxCHj*rNx{TuYL6Me_s-;qA+Ll%SkXh`n?CwA{&G_;-G;%)Kts z!!3arUjl~vNBbO=xJ5=Y_sqlnz3JwRP;lW~@iZvBiS||kOikp*eF(%KV$Ip2`a&1EvNGK=CT^LQV8Y5d~9;Bzzn1#cP5 zL|u3if5DBY4EJwj|3>y-AMT8@_zBYIK*IeS8ce$l^#)tc*G(Ytld$x;4$?{gzhnRJ zqWC-Ke;00F;t7gBz3&fCPXF+}hugVVnEvli24Tq|hSg|M8Bhk40cGHSfq_iFxgN07 z-sXCOjM-xuIs7wZQnxv*jS}xvjwjcxHWwqjt|>~qQjSOOxwa)a$6Lwqoac#Kik;#x zDBhb zziQve^Wx$4YcVF;pY+@`y5F&08{X!1J>RTX`drtSI6229d;4>9i}^bE;wR1V@y9ZQ z6nS7Q&gA((+|qfnTtBXp_PL+y<$7)4NBg?x;`bu=S869b$@%=GS*E{>XGR`>_pq^l z(7v)ogu7dr^CIo3GQ98dJ}bg0Oep!ZW&EU;{UNJzx!wOT(`_os`%$?bC-0LaSgr8K z3hBFJH?w+`PcG?|VmZ|k=5I&ZZhKXx_4D=X+Y{GAD=K^9+AS$_c@OryZu95i{Wj|i zb3MNPwa9%}_?7`AIObFQTxS}8i?NpD1tcz`hwC#Lvltt=zKK!V6D*0MmwG3d^Zgacdm1_}4i6MDN4~wyS4sVyt7VW%M(aFgoLGFFCM-<8QBiwlz*2&yR=KZ{{O! z*y}jSax|ED|Eitzv#q%C-RcMCjKgQCuDI5|^xnwwPmC3N7T9koK8vCk`?j{{w43(x zLnAigq-I@(d^^Qyv2}7cz8^0A7Vqw??@~};E+}$E3hS@o@r!+b)-Q1Ahp>KzLw^$M zXFBw!vwoICpU?W)4t)vh=Q#8ivVN{ZKb!Sc4*eCZf5@S~iS-XV^mnuVF^B$P*3Wt? z*IozzDmVADoji-)$+PI4ahf?!fa4EJ`qQRf>}4Ld7Qe_h`>AvOzi0gvUe6M@nf1Fm z)s35X8mrCb_cM2cZYr`>$^JiC$YYpLw_3U z_i*TQSzqIr=OWf$;n1JY`YRp!nXJFcp}(BTCh+UJ^LX8r_0HpUU)I+-`hO_v zoyT!s);o{m<5~ZRBhGNv%RVpbc^vDV$9)#-A9ut#hxKy2N}P*XFZ)C^{SrRjoX4H$ zoyV`}CEnKJzFWAH#a*@%j(eJNcW$dMAHntatL^i826r-Iwzgk)xQa zS{YCVlmTTx8Bhk40cAiL_@87T+VlSc{7?c9v(k+oxW&nX=l|_wdGOtcuJ%19=e0z! zBb>`uiv;CDD4nO<_wMC_rUO@;Xl`=}hWpv=W$|q=2mW!RX}1B_#2$wj zzx6!~iXXK?er|ejO)FT-Z#V?ydMx#kavFEjiQOp}k|5bqlIQWQf8IDcvAca034V;? z_gA0C=j20VH-7*`P~KlryYw9&VuhZiH)!^=oliJx;($F)TvPCC^O}llulZ&Os_eV9 zLMi#s_n0IG)!P?Mfd?bg6dZ(1&!yN0`2b{E7Q!jDwI4DiT08WaOrn84`|n1kbRaU_ zU!c#j%6E`Zl-s#fj7ajmsR(75K;Oe63c^3PA{RjSXS?vvX3O`c!a~@^?yQi{Vaw-W zJ3~*O+o$`}lIU~bl86uDi?|~m+J-?;QWJ4}bdP2e%d+J^Zmn zS9fVTDfy^VuUt{`{ml=LJ@NDJi!UGOsu;WKg?$@;c>B_ere8Cse!#4?dw+2DhTF#7 zbLXJ!+plcD@U`E%y^wO|#mff-U;l7m;kuQ`{hx?E^@qovo%NA*=8~J^JzWpXd*)d0 zXX7hLdU~c# zXRx)hH?7i^HGVNBqOT%L zH%E9qGU@%?&kIS3^aR`c`C(NOpFZDB_E~<@pFy^@c^(z<){S}V!TPh<-kNUgm#}^v z>r;mq{Xo|5!|l5G{+8?$sjT(4U8OS`2U->Nr(YYkXcQQ}_}MNiMGq;~7LU74>JIDW(Nrp0`YzdP({ zTy@O{WWNYgt;M;J8XrrL0d_q{G_Sei{Kg-&R1Shl$}$U@0*xk?wdk`Fulk(+OquIic;!YX<7cczTycJ%JLDM;@J+_ zdH6;jwME{fMCZ#3a(x9w6N+tps3m*@FduR3Zs7}#wxN-;OD9gD28c-c)?rCuc5eQp z;=;Uqg!dJdlo$GPCuJ8+%x8A+{m036u<#M0ReTjOd`5hwrWbbXyaHcYJ~-o^(Whpp z*SRGXz6szT!!#UC&MmGeqCwh2o=})Q(dfgQl1@>J1t7FEzofXd+?PKkzoC)0- z!%zzoX1ZxPOwJCqQ&w;R=qNQ`ZM28RGj9jLmpjeJd(i8i{rb{}d-_->jLaA`*w-he zPs&lD@{yrZ-%#nuu%cLIj?OqGee6(L_8shhvG(8gVdWNm9NUnlz17a&0k7-vC>e_m zIJ+^WcA@jB7|Zgk%k)Y0n{C8~=)@*;1l7?O;RK`^AQiZL%R-N@UX>HgxS@Z0I;A8?vQDHZ(rE4?s4Q zXfGxkvXvCr&+NZ+K1t^(l9q+(>6}uMoZr$p3MG#`T#;W#>3oUKeFNAPk}FIv zuQ^oLElhu)7z3O5{m0^cOn-wkKuyYkGN2471ImChpbRJj%78MU4D0{~&gyBtcS4^R zZp&MbhW*Xw^5y!{z>Vhf#-Xm&o6kSHUGwbELCfc{6Pk>^=JUw!J<0WNxlUgDS!BKa zuq*rt#W!yOEjQoqaEDb;aiaavbGp2cQdCstsBA@)w~R7&zFcZwUvI^H6P0N%zp#Lw z+mupvPCqW(qP}%(HWzBW4)t?j`5x}a)?j(600p3 zK5=4CTyuMwLr0GtH+m2n;$+5$OF$`fIjC$>c4>Z|FE=}P5-v2+1sP|GHa>@+hI0Pjfs>s=)Qb!p*MFXzNAdZ3#_#vOaN@(KoqxN?A_S zqcGGS4=i;2xlK!38gIn-CRa|#E-T+`7eno_Rj9o%ug$e-+Ozjhe(0O83>Q+vyjw3y zOle=Rb?X+bX;0Qm%l?^DQFeZdRrJ!ftxwrMi%WbZ*{yG!+@y0H(fE}8^Q25)W;$)0 zoHkuAW#z?fSTDk7s6D#Ci6nX>ZTdH|y&~MpD9@jaOX0XA-s%g5jL+jSvADhDKZDQP zimhvpo{f0Urkx7YwDol^%q5~|#wYu=xihnwmj_%aj!Ypw?O#fZtZPkb;d9b?x!E|{ zv&%L+Kqp_)o_wxB+Q5se_te4ZnZ8l!V@8M)XxiiTIeD_(Vr5wUmWpU^s?&!0#Org^ zD7(q=p#^H%i#YzuOV77YVQ@1mzm#rHW#>fS0i?aQ&TGW@%FWDG6wwxg*ZIPN$+%w> zk)qFEgjrOfW&Y@Vq9Df+eiOg8j$;!?R`ar@e^zYy0G(2Y+e)aB^0Z zjYDEH3smM$jte;+!ubnZpXsBAo|ukfV!YHxtQY%uXmuMC^UWOZA(Qa)UHf?0^n6!d z-{~o*n>+j5^m9a^@%c@AGVYevBTEtE&8;AynJvXTPaVz=+vwEc!( z9|n7e4jvJDaUl1%-Zjtrc*S5~xxu=34W{m8Ff+!WwAi%pZu@)4A`3Qtkk$XaM=W?EDxuqj~qRSxm9;x=5f>ej}_9p<$=K#o} zMIU)7NAz;sm6he2Pdd@*LBGE6%$`ztRPR2%-d4d;Y39>*1^u{y2X~*BUs#A=PszuL zb;+axm^+f6*P(~?e#d%j=wz;Mg`Nx6WUimdbGJ(VFmfuRhtc}e=>06qbjlhI6q@I^ zH=SqDb&Nsl6qW<#dz&Q(HrvN@-3lYuK4t0yy^Ox$Bcpc=DzbhbdA^bDR=x??9I2Gbi$S5^0yKcNi@C zr{SrNFq7=SMB7y(-}956>^>)dVf`G&ilK)!=FodF7opQa=z+(E}fsB z7wc@Nmi=oOQyDWEvl#0c8yK4yOBnr(&iL*qdZ(Spby4gCQS{>XSJA`o$5I(H8J%%m zq8D@;Y$w^hgX3?fZnim$hv(13>sK7*{i?`Y!ltWQ-uKdbQF=d3??vnQ{dqlO)s~`$^78cw@sWLD{3(&o;oR2l%lAv< zd(+z*=l`ppTo)4l>#jEIQREsKIIqnHxA`2J)Jt`s-t6y!)}H(vS;hvg7ZkrOp|m5{ zKm2?>E#&7oPDA=F6y$s4^0_*TpI4i8K+E%Qw-92xRDX6fhslh5?zc0zYz_X;49`9I zI+)lAPlB0i#6v7LkMcU5>U{n6obVt$S9R;6VXx!s_@P$$^NF?mI|qSI=5va2{Z;H_ zo<*L^_9EA_Um%J78KpjpxagM;qIXX6E5>rZPJ1+zx$~Lw4G?RnFW_aoXD6j<(B``Qi1O*2yZ*Sb*?UiuxwhE-YHA}?aycRG@t#X-vPq2 zL3&O(vRA9>KE3dfH2$$5fiR5;={N6#<#+ky_w?j<@T6TuWk4BF29yD1Kp9X5lmTTx z8Bhk40cAiLPzIC%Wk4BF29yD1Kp9X5lmTTx8Bhk40cAiLPzIC%Wk4BF29yD1Kp9X5 zlmTTx8Bhk40cAiLPzIC%Wk4BF29yD1Kp9X5lmTTx8Bhk40cAiLPzIC%Wk4BF29yD1 zKp9X5lmTTx8Bhk40cAiLPzIC%Wk4BF29yD1Kp9X5lmTTx8Bhk40cAiLPzIC%Wk4BF z29yD1Kp9X5lmTTx8Bhk40cAiLPzIC%Wk4BF29yD1Kp9X5lmTTx8Bhk40cAiLPzIC% zWk4BF29yD1Kp9X5lmTTx8Bhk40cAiLPzIC%Wk4BF29yD1Kp9X5lmTTx8Bhk40cAiL zPzIC%Wk4BF29yD1Kp9X5lmTTx8Bhk40cAiLPzIC%Wk4BF29yD1Kp9X5lmTTx8Bhk4 z0cAiLPzIC%Wk4BF29yD1Kp9X5lmTTx8Bhk40cAiLPzIC%Wk4BF29yD1Kp9X5lmTTx z8Bhk40cAiLPzIC%Wk4BF29yD1Kp9X5lmTTx8Bhk40cAiLPzIC%Wk4BF29yD1Kp9X5 zlmTTx8Bhk40cAiLPzIC%Wk4BF29yD1Kp9X5lmTTx8Bhk40cAiLPzIC%Wk4BF29yD1 zKp9X5lmTTx8Bhk40cAiLPzIC%Wk4BF29yD1Kp9X5lmTTx8Bhk40cAiLPzIC%Wk4BF z29yD1Kp9X5lmTTx8Bhk40cAiLPzIC%Wk4BF29yD1Kp9X5lmTTx8Bhk40cAiLPzIC% zWk4BF29yD1Kp9X5lmTTx8Bhk40cAiLPzIC%Wk4BF29yD1Kp9X5lmTTx8Bhk40cAiL zPzIC%Wk4BF29yD1Kp9X5lmTTx8Bhk40cAiLPzIC%Wk4BF29yD1Kp9X5lmTTx8Bhk4 z0cAiLPzIC%Wk4BF29yD1Kp9X5lmTTx8Bhk40cAiLPzIC%Wk4BF29yD1Kp9X5lmTTx z8Bhk40cAiLPzIC%Wk4BF29yD1Kp9X5lmTTx8Bhk40cAiLPzIC%Wk4BF29yD1Kp9X5 zlmTTx8Bhk40cAiLPzIC%Wk4BF29yD1Kp9X5lmTTx8Tj`xu&C?$w4$kD?PtZ>LfV)} zv|M2}D8`7$_Eue2Tl2@gkr?u0yLRHo_Ey8*wkGA{NDTS0U8p^_w>7B0R0hDn*&`#@d&#wTkJyOy?+$FRkAv~MoBH;!jYsw%Q}WnFj$5p63ISw6|a=1BC z1;hP_oBBpG0w*pw$&A2%FdwlzK-x{nFKaJzygJb`+l?hCd=e*=GnsEEx=}~TZU40_ zr(X&CIdKTv3sM`DIv`WoIWCc9zG}!AN!lCpb$K57Ir}HFn~ag%_TLu2O!m8v*NYQ7 zmKz5_>F3GtBR<=jq~D{tJsFP^Q;#+g1$h=N&z#@v<72-L=J@#R$MRQ$$(H51&Ky6o zJ}90|`IW5a4t85%jxXsi*^oaaI{sw;mUbH7t$tw6IOqqz9(sM^skH6yF|O8WgZ8yo zj#HL?e?N^49WZ{ZS%=c^x+G&ToQ%14{SmueVh-?dW0Wr%Zusd{Sdi1J zGOgc|{5S0WXEA@j>}Tp}zadxYr-YGu(F;nwpy&lfADRzV>?{?GsWM*TQw7iy1Xu@A$ZvePG%WvyWz-(P-}r*#|V+Sj%z5FO%&gzO&wcqiHWN&%|qZ+w8Bd zd(HkLa>F!kkNa_TPg7t2nz8@aW6#c_Hdih*cGjH+ALaN@^)~i=c5MX~nR@YG%ywe` z8SmSH8%+QFKC}P%UpM=>pT}9r_`$l}W2yZ{9*3+K($VWe_vNNN+2^EPs;BhrOOf?P zi+zIoCG9s}YV2ySGAQ#RM?&G|kdoX%#<#ob1=*e|)7=5s{^*cJ_l=QJfEuRnP0)qzo zGD(*H%e<}{muffp3y#0=w)=8?X@AoD{QhLVSr;;YGX8+{i`TEr=XkakS=ta;;-s;^ zj8EDVdnYdDd89aW{Xo{Wtc9#qW*&rRIUeMAl65NM_j5eyPpCcDJo`K_;ZDQ*;!Z}E z_iN7ed%$gG{Ym>)PLn;~60ermZ3Cm|C4S5Lb$h5z+LQYAd|o8{iGS@kk^4#~u2{gw zpZNQ+@PonF(ECki|0P}}@3Vmf^ZGFTUK770O8XMOrT-?9v;Ptw??gN;WhzC=vR<9* zSA0aD>&wqBEj-_sB^6HOouQW_InDlVc}MqOjt72^=Ks6Lx2^4+`glisY5(>9n;rcR zd1(k8Fh(!?y@xyc&wn0fmm0^md*uqhI7J_Y9EZ;R;_NTX`$nmEozO)ZynKV?u#kM zU;fq+Bkp%C+wU@Cz3E?xD`@?^)7s+Q>(N^(&zpT#?6X*&!Z?+2HsfWC^^9LKivRvnXnM%L&UPfrO?+U;`?TQi)%Jzs z&@~ON=*;$cjB;K^^F!AMX~lmbC{LufNOiDi;mp_>%LWADB13Bx2pe_AQaPbSp2A)vI6nBzy71YjKUi z*RSs;W~xvIlmTTx8Bhk40cAiLPzIC%Wk4BF29yD1Kp9X5lmTTx8Bhk40cGItVW3y} z}_SH zkM$1hRZ&*jtFSmXyRcVTPC?NzVRA@psjyW>MA15F}(ajI)7L?~%laKu@Y-!YiW4Uobd2Y7+{@7Ty-vp!hQTOOs*z~+^ za+T+3qiE4@kI*>eS=#grZR$2Xi(C3j{?rFTsv~=xNrKU!FQ?_J~>Ebh>Qc>BE;N6kc6<%SC(KF}Uc`!(Kl4;JTdr$>Znn zFAu=6mDB^e1JJ_~e!#cTMeXeZeX2u0knR1fui%*d*xsq9aZ-F|9Qrjritph#DI8}! z>r?pw<2SROe$R^HIPLdmduRWrLEqW>ag(tRXM20nP(A{g(#gma->Jv1mQ2YjEA~yy zFE1~coL@X4FZ+DakwVJ3cp%3)Vzr5~G`~!A2$Wk=AyV1IN?(3ad44JW<`!m8$+rp% ziYh95QkyCG-`g6QapJh~zCI~LFxErs|7h))f44pVL#HmVB)-li7bKd?So~dA@jx8a zQ6hdQ$#dJqR-Qv|YWY}#;HDr*zitc`6Ha1T){DJ>p_k{@FXMVzuav3{5W{xr#)#iV zu;VS_5AE_V~2zq=xIJ_EhM1a9+}dy z$f?NjXp`Et(vw`XV+VI?KV*K~y!atA+Qlx~dFhblgWgDg8BtD29JTY(K`j#6hf4|a zSPb?lifQ|2bZS3nUYvJEeC$H9w|z4csr{Rf8!;|fE2OJKUIm35_oRCfnTQRyTM5-4 zP{=+*<0M`t!A_1{h*o#xc=W{&y}f71n9^s&#h!>^4C>gP+L#fHsn|O?rg=h*_k}Or z-ks`uzuo)Q-s^e?hxQH(?fvS~3QY zlf41&Z(n+o@2huLckTav#hu=mWzME2q$KY*lW4QXd8@uoAD2Ehqw1G2-rUb;{GOaV z=Sk=n)4W!FICB1O7c342-)X2?GVRM5*vyk>zKe>X)qAF|TN;x*>v0I_)fdD!F5HOz zRW0$(3RGU~&HXu}dUyi#-MlY+6F)NMeQ))!9^U>d%69f1`kFWS!4JGK_372a_x4s# zai>@BlHUKFiu=9qG2W|U=Xo0=Fez?a%A$5l|!o!Pha;E82r^{ z@WS}U9Bh4t!Q{C|BLY|)RXrHtN7yW8%s=)>Z}ka1GWyq-?L6|(C7^MIP2&jAcq(Wd zklz2#ibWYQKY_`N>QZ+`+@Kzy%`liedjZ;MmDrzbQZuT$bYF8jP2!BDvEKPd&HRRX zv(g^>{EqFbkMpd1C8lEZsOp%E>WqXT)%&LR|Gi>OM$D3o>OL89!xKivyzHIUY|r1W z-f3%O`i$zl?{N@RjqIM_jcLlLitoOrKN=($bxi|}r)t+~KP}bts=@JVpPV)@ ziA~CP2RlhHUYv#6kyRgztokgu-P-HH$l6uZXXarVYO4o+0#)OIzY7nQuVFV@4jnl7 z8c^NoEfV^_Plw-6yl$#HJ!e}Cd>UYL@KMDg;vlZwLyy8bZs6?zEEsc4cHZQOtOAo8 z*Q_=Zc@DZdA$h+cmbGR8G!t6(7mU6?H8g*>K8QDM;1T$XWRu{n_P8;(<#FDs4mg;D zOP5cmS#u7?5#~Q5n0(%u2=dEUc98EM!B*KkZQ$2H6J|)gy;Z%_aL-lPo+^4Q$|^r% zv9;4c%R6gbd7}3fdi}J#Rl^c`2U<1^>=V_~8&R?L!^K|%lG{bp)wHC=7oz&-& z-WkX4VO8|O5tiVc_B*|*SA6HK-orcJKG#i-`M_KK57#S$6XI>#3%gkr2ViC4HEHp; z80*?!sk&-#!i1^|J5H!7OY&BYajpHpTlEDUr&V8hXMC0How2GZGxHA++b#Kl#$dg7 ze%{`$8SAY}o?LS_LL@W?gP90kYPDiX@$qOUxi?;~oJgFVFnI7W?jGZEDvHW0+`apx^hrtWd1Qqx_CAyqI0Y!1 zR90SEo}FW*Of0HMnUq~N$x6vPzo_i|$p*_yP0f`2(z1f$q6op4XWL{?F37b~ii*qg zt(5#pz6qt-lkOcYQTR&pOA51d^C#nc+)BwU zo@^H|%D> zJ5VxyX-BXR{OLV`$nrixFpXQH{*#ULpVd}63bX}C{01%v$~*|);!o#_l<55;PO)3Y zPm@WBajcO-erw{`NMAay=IW|C9=L zka3HD5&KL3-{AMlKLvndEbAJ z?{MyU_I=ko0K_f>ygxkj*pWk?!Bys;mW!{cJ@;idj$a4Ne9?`scXw0%&$iw>^wysX zG6vH~Nf{F#ylHg5#Dqt-iSlXe!hvVK`r`7UwZ5NT+^zPRvDO;l6)mHgSAI4=ed;!{ z%|rue?3Z!fqxx$7X!Spgv;yq#Y8JiMcJIX)8twZ=BTs;|L=6+g#yVMfONVm)5WR z@zY6jY2Is#R(R=ZOH!>_3sa(vIlK0k3t~vE8G|X>Ja!2JPrlYG8zw9{SzJRc?+0x%`IJ znR`%1RrH%VPG!_Y-%ni`^%h+j4bg9*zcNb4o5w@0ylwR>>936HJ68XQx-x2_e@0yy zbvbHC);H&*w@uZ-AftDj)rDx)O&UM7k%Dx%*+z4EQ} zS69pNkYi_Ty&q>~ltlOMcV(1C_w!8|718~5sR;vTrGNHIo)F#dR~aSI{rg23WzqTZ zMn*++|Km*=_3xVdCD+bb{RWOl8O2|%{xWrClte#4T^abz`un;vxx-#mb`|GIRv8c*-Er;Xt#P7}HQAU^S2k+$@0>X%j z9i*;|3DJjDZ{0s}dlk&2r)Q4^3(2Fw^t9{br{fuK>*dCSbO`62L^|gb{uWr13{Ryq zP9eP7whf&=w^(dv3;A#+JsCbWG%!BUuc?!oo;6}BmkK)h{7gPnmf@VR3b56u(w6~SZ{Pcb9RMx9RJxde0hLv@PVA)ZMDLbqx;K@ATew zDZk;}?HR83&}yDrdB(ZmKg{!CP;AjVKwVjB);0otrN42|@;8swEd6LX2cG%;^3-{I zK3)BO(4M204(w@|Gh9Ba^GqYUPjtS2t61~9z@m3hxXnw7!=sOmJ?U{LC$?5j?D7w) p)|a*0|E2%>TJ3msBG8FICjy-abRy7+KqmsLN8pLEF>ePB`~|J4@NED9 literal 0 HcmV?d00001 diff --git a/memfaultd/src/cli/memfault_core_handler/log_wrapper.rs b/memfaultd/src/cli/memfault_core_handler/log_wrapper.rs new file mode 100644 index 0000000..33d86c6 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/log_wrapper.rs @@ -0,0 +1,123 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Logger wrapper to capture logs that happen during coredump capture. +//! +//! These logs are sent to a channel that is read by the `CoreElfTransformer` and written to a note +//! in the coredump. + +use std::sync::mpsc::SyncSender; + +use log::{LevelFilter, Log, Metadata, Record}; + +pub const CAPTURE_LOG_CHANNEL_SIZE: usize = 128; +pub const CAPTURE_LOG_MAX_LEVEL: LevelFilter = LevelFilter::Debug; + +/// Logger wrapper to capture all error and warning logs that happen during coredump capture. +pub struct CoreHandlerLogWrapper { + log: Box, + capture_logs_tx: SyncSender, + level: LevelFilter, +} + +impl CoreHandlerLogWrapper { + pub fn new(log: Box, capture_logs_tx: SyncSender, level: LevelFilter) -> Self { + Self { + log, + capture_logs_tx, + level, + } + } +} + +impl Log for CoreHandlerLogWrapper { + fn enabled(&self, metadata: &Metadata) -> bool { + self.log.enabled(metadata) + } + + fn log(&self, record: &Record) { + if record.level() <= CAPTURE_LOG_MAX_LEVEL { + let entry = build_log_string(record); + + // Errors are ignored here because the only options are to panic or log the error. + // Panicking is not a great option because this isn't critical functionality. Logging + // the error isn't an option because we'd risk infinite recursion since we're already + // inside the logger. + let _ = self.capture_logs_tx.try_send(entry); + } + + if record.level() <= self.level { + self.log.log(record) + } + } + + fn flush(&self) { + self.log.flush() + } +} + +/// Build a log string from a log record. +/// +/// The log string is formatted as follows: +/// +/// ```text +/// : - +/// ``` +fn build_log_string(record: &Record) -> String { + match record.line() { + Some(line) => format!( + "{} {}:{} - {}", + record.level(), + record.target(), + line, + record.args() + ), + None => format!("{} {} - {}", record.level(), record.target(), record.args()), + } +} + +#[cfg(test)] +mod test { + use super::*; + + use std::sync::mpsc::sync_channel; + + use insta::assert_json_snapshot; + use log::Level; + + #[test] + fn test_log_saving() { + let mut logger = stderrlog::new(); + logger.module("memfaultd").verbosity(10); + + let (capture_logs_tx, capture_logs_rx) = sync_channel(5); + let wrapper = + CoreHandlerLogWrapper::new(Box::new(logger), capture_logs_tx, CAPTURE_LOG_MAX_LEVEL); + + let error_record = build_log_record(Level::Error); + let warn_record = build_log_record(Level::Warn); + let info_record = build_log_record(Level::Info); + let debug_record = build_log_record(Level::Debug); + let trace_record = build_log_record(Level::Trace); + + wrapper.log(&error_record); + wrapper.log(&warn_record); + wrapper.log(&info_record); + wrapper.log(&debug_record); + wrapper.log(&trace_record); + + let errors: Vec = capture_logs_rx.try_iter().collect(); + assert_json_snapshot!(errors); + } + + fn build_log_record(level: Level) -> Record<'static> { + Record::builder() + .args(format_args!("Test message")) + .level(level) + .target("test") + .file(Some("log_wrapper.rs")) + .line(Some(71)) + .module_path(Some("core_handler")) + .build() + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/memory_range.rs b/memfaultd/src/cli/memfault_core_handler/memory_range.rs new file mode 100644 index 0000000..c6a57a5 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/memory_range.rs @@ -0,0 +1,127 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::cli::memfault_core_handler::ElfPtrSize; +use std::cmp::max; +use std::fmt::{Debug, Formatter}; + +/// Convenience struct to manage memory address ranges +#[derive(PartialEq, Eq, Clone)] +pub struct MemoryRange { + pub start: ElfPtrSize, + pub end: ElfPtrSize, +} + +impl MemoryRange { + pub fn new(start: ElfPtrSize, end: ElfPtrSize) -> Self { + Self { start, end } + } + + pub fn from_start_and_size(start: ElfPtrSize, size: ElfPtrSize) -> Self { + Self { + start, + end: start + size, + } + } + + /// Returns true if the two ranges overlap + pub fn overlaps(&self, other: &Self) -> bool { + self.start <= other.end && self.end > other.start + } + + pub fn size(&self) -> ElfPtrSize { + self.end - self.start + } + + pub fn contains(&self, addr: ElfPtrSize) -> bool { + self.start <= addr && addr < self.end + } +} + +impl Debug for MemoryRange { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MemoryRange") + .field("start", &format!("{:#x}", self.start)) + .field("end", &format!("{:#x}", self.end)) + .finish() + } +} + +/// Merge overlapping memory ranges. +/// +/// This is used to merge memory ranges before turning them into PT_LOAD program +/// headers. +pub fn merge_memory_ranges(mut ranges: Vec) -> Vec { + // First, sort by start address. This lets us merge overlapping ranges in a single pass + // by knowing that we only need to check the last range in the merged list. + ranges.sort_by_key(|r| r.start); + + // Next, iterate over the sorted ranges and merge overlapping ranges. If the current range + // overlaps with the last range in the merged list, we extend the last range to include the + // current range. Otherwise, we add the current range to the merged list. + let mut merged_ranges: Vec = Vec::new(); + for range in ranges { + if let Some(last) = merged_ranges.last_mut() { + if last.overlaps(&range) { + last.end = max(last.end, range.end); + continue; + } + } + merged_ranges.push(range); + } + + merged_ranges +} + +#[cfg(test)] +mod test { + use rstest::rstest; + + use super::*; + + #[rstest] + // Two ranges with matching boundaries + #[case( + vec![MemoryRange::new(0x1000, 0x2000), MemoryRange::new(0x2000, 0x3000)], + vec![MemoryRange::new(0x1000, 0x2000), MemoryRange::new(0x2000, 0x3000)], + )] + // Two ranges with overlapping boundaries + #[case( + vec![MemoryRange::new(0x1000, 0x2000), MemoryRange::new(0x1500, 0x3000)], + vec![MemoryRange::new(0x1000, 0x3000)], + )] + // Two ranges with non-overlapping boundaries + #[case( + vec![MemoryRange::new(0x1000, 0x2000), MemoryRange::new(0x3000, 0x4000)], + vec![MemoryRange::new(0x1000, 0x2000), MemoryRange::new(0x3000, 0x4000)], + )] + // Three overlapping regions, unsorted + #[case( + vec![ + MemoryRange::new(0x1500, 0x3000), + MemoryRange::new(0x1000, 0x2000), + MemoryRange::new(0x3000, 0x5000), + ], + vec![MemoryRange::new(0x1000, 0x3000), MemoryRange::new(0x3000, 0x5000)] + )] + fn test_memory_range_merge( + #[case] input: Vec, + #[case] expected: Vec, + ) { + let merged = merge_memory_ranges(input); + assert_eq!(merged, expected); + } + + #[rstest] + #[case(MemoryRange::new(0x1000, 0x2000), 0x1000, true)] + #[case(MemoryRange::new(0x1000, 0x2000), 0x2000, false)] + #[case(MemoryRange::new(0x1000, 0x2000), 0x0fff, false)] + #[case(MemoryRange::new(0x1000, 0x2000), 0x2001, false)] + fn test_memory_range_contains( + #[case] input_range: MemoryRange, + #[case] addr: ElfPtrSize, + #[case] contains: bool, + ) { + assert_eq!(input_range.contains(addr), contains); + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/mod.rs b/memfaultd/src/cli/memfault_core_handler/mod.rs new file mode 100644 index 0000000..d4425df --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/mod.rs @@ -0,0 +1,306 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +mod arch; +mod auxv; +mod core_elf_memfault_note; +mod core_elf_note; +mod core_reader; +mod core_transformer; +mod core_writer; +mod find_dynamic; +mod find_elf_headers; +mod find_stack; +mod log_wrapper; +mod memory_range; +mod procfs; +mod r_debug; +#[cfg(test)] +mod test_utils; + +use self::log_wrapper::CoreHandlerLogWrapper; +use self::procfs::{proc_mem_stream, read_proc_cmdline, ProcMapsImpl}; +use self::{arch::coredump_thread_filter_supported, log_wrapper::CAPTURE_LOG_CHANNEL_SIZE}; +use self::{core_elf_memfault_note::CoredumpMetadata, core_transformer::CoreTransformerOptions}; +use self::{core_reader::CoreReaderImpl, core_transformer::CoreTransformerLogFetcher}; +use self::{core_writer::CoreWriterImpl, log_wrapper::CAPTURE_LOG_MAX_LEVEL}; +use crate::cli; +use crate::config::{Config, CoredumpCompression}; +use crate::mar::manifest::{CompressionAlgorithm, Metadata}; +use crate::mar::mar_entry_builder::MarEntryBuilder; +use crate::network::NetworkConfig; +use crate::util::disk_size::get_disk_space; +use crate::util::io::{ForwardOnlySeeker, StreamPositionTracker}; +use crate::util::persistent_rate_limiter::PersistentRateLimiter; +use argh::FromArgs; +use eyre::{eyre, Result, WrapErr}; +use flate2::write::GzEncoder; +use kernlog::KernelLog; +use log::{debug, error, info, warn, LevelFilter, Log}; +use prctl::set_dumpable; +use std::io::BufWriter; +use std::path::Path; +use std::thread::scope; +use std::{cmp::max, io::BufReader}; +use std::{cmp::min, fs::File}; +use std::{ + env::{set_var, var}, + sync::mpsc::SyncSender, +}; +use std::{io::Write, sync::mpsc::sync_channel}; +use uuid::Uuid; + +#[cfg(target_pointer_width = "64")] +pub use goblin::elf64 as elf; + +#[cfg(target_pointer_width = "64")] +pub type ElfPtrSize = u64; + +#[cfg(target_pointer_width = "32")] +pub use goblin::elf32 as elf; + +use super::MemfaultdClient; + +#[cfg(target_pointer_width = "32")] +pub type ElfPtrSize = u32; + +#[derive(FromArgs)] +/// Accepts a kernel-generated core.elf from stdin and processes it. +/// This is intended to be called by the kernel as the coredump handler. It is not intended to be +/// called by users directly. memfaultd is expected to set up the handler with the kernel by writing +/// the appropriate configuration to /proc/sys/kernel/core_pattern. +/// See https://mflt.io/linux-coredumps for more information. +struct MemfaultCoreHandlerArgs { + /// use configuration file + #[argh(option, short = 'c')] + config_file: Option, + + #[argh(positional)] + pid: i32, + + /// verbose output + #[argh(switch, short = 'V')] + verbose: bool, +} + +pub fn main() -> Result<()> { + // Disable coredumping of this process + let dumpable_result = set_dumpable(false); + + let args: MemfaultCoreHandlerArgs = argh::from_env(); + + let (capture_logs_tx, capture_logs_rx) = sync_channel(CAPTURE_LOG_CHANNEL_SIZE); + let log_level = if args.verbose { + LevelFilter::Trace + } else { + LevelFilter::Info + }; + // When the kernel executes a core dump handler, the stdout/stderr go nowhere. + // Let's log to the kernel log to aid debugging: + init_kernel_logger(log_level, capture_logs_tx); + + if let Err(e) = dumpable_result { + warn!("Failed to set dumpable: {}", e); + }; + + let config_path = args.config_file.as_ref().map(Path::new); + let config = + Config::read_from_system(config_path).wrap_err(eyre!("Unable to load configuration"))?; + + if !config.config_file.enable_data_collection { + error!("Data collection disabled, not processing corefile"); + return Ok(()); + } + + let (app_logs_tx, app_logs_rx) = sync_channel(1); + let log_fetcher = CoreTransformerLogFetcher::new(capture_logs_rx, app_logs_rx); + + // Asynchronously notify memfaultd that a crash occurred and fetch any crash logs. + scope(|s| { + s.spawn(|| { + let client = MemfaultdClient::from_config(&config); + match client { + Ok(client) => { + if let Err(e) = client.notify_crash() { + debug!("Failed to notify memfaultd of crash: {:?}", e); + } + + debug!("Getting crash logs"); + match client.get_crash_logs() { + Ok(logs) => { + if let Err(e) = app_logs_tx.send(logs) { + debug!("Application logs channel rx already dropped: {:?}", e); + } + } + Err(e) => { + debug!("Failed to get crash logs: {:?}", e); + } + } + } + Err(e) => { + debug!("Failed to create memfaultd client: {:?}", e); + } + } + }); + process_corefile(&config, args.pid, log_fetcher) + .wrap_err(format!("Error processing coredump for PID {}", args.pid)) + }) +} + +pub fn process_corefile( + config: &Config, + pid: i32, + log_fetcher: CoreTransformerLogFetcher, +) -> Result<()> { + let rate_limiter = if !config.config_file.enable_dev_mode { + config.coredump_rate_limiter_file_path(); + let mut rate_limiter = PersistentRateLimiter::load( + config.coredump_rate_limiter_file_path(), + config.config_file.coredump.rate_limit_count, + chrono::Duration::from_std(config.config_file.coredump.rate_limit_duration)?, + ) + .with_context(|| { + format!( + "Unable to open coredump rate limiter {}", + config.coredump_rate_limiter_file_path().display() + ) + })?; + if !rate_limiter.check() { + info!("Coredumps limit reached, not processing corefile"); + return Ok(()); + } + Some(rate_limiter) + } else { + None + }; + + let max_size = calculate_available_space(config)?; + if max_size == 0 { + error!("Not processing corefile, disk usage limits exceeded"); + return Ok(()); + } + + let mar_staging_path = config.mar_staging_path(); + let mar_builder = MarEntryBuilder::new(&mar_staging_path)?; + let compression = config.config_file.coredump.compression; + let capture_strategy = config.config_file.coredump.capture_strategy; + let output_file_name = generate_tmp_file_name(compression); + let output_file_path = mar_builder.make_attachment_path_in_entry_dir(&output_file_name); + + let cmd_line_file_name = format!("/proc/{}/cmdline", pid); + let mut cmd_line_file = File::open(cmd_line_file_name)?; + let cmd_line = read_proc_cmdline(&mut cmd_line_file)?; + let metadata = CoredumpMetadata::new(config, cmd_line); + let thread_filter_supported = coredump_thread_filter_supported(); + let transformer_options = CoreTransformerOptions { + max_size, + capture_strategy, + thread_filter_supported, + }; + + let output_file = BufWriter::new(File::create(&output_file_path)?); + let output_stream: Box = match compression { + CoredumpCompression::Gzip => { + Box::new(GzEncoder::new(output_file, flate2::Compression::default())) + } + CoredumpCompression::None => Box::new(output_file), + }; + let output_stream = StreamPositionTracker::new(output_stream); + + let input_stream = ForwardOnlySeeker::new(BufReader::new(std::io::stdin())); + let proc_maps = ProcMapsImpl::new(pid); + let core_reader = CoreReaderImpl::new(input_stream)?; + let core_writer = CoreWriterImpl::new( + core_reader.elf_header(), + output_stream, + proc_mem_stream(pid)?, + ); + let mut core_transformer = core_transformer::CoreTransformer::new( + core_reader, + core_writer, + proc_mem_stream(pid)?, + transformer_options, + metadata, + proc_maps, + log_fetcher, + )?; + + match core_transformer.run_transformer() { + Ok(()) => { + info!("Successfully captured coredump"); + let network_config = NetworkConfig::from(config); + let mar_entry = mar_builder + .set_metadata(Metadata::new_coredump(output_file_name, compression.into())) + .add_attachment(output_file_path) + .save(&network_config)?; + + debug!("Coredump MAR entry generated: {}", mar_entry.path.display()); + + if let Some(rate_limiter) = rate_limiter { + rate_limiter.save()?; + } + + Ok(()) + } + Err(e) => Err(eyre!("Failed to capture coredump: {}", e)), + } +} + +fn generate_tmp_file_name(compression: CoredumpCompression) -> String { + let id = Uuid::new_v4(); + let extension = match compression { + CoredumpCompression::Gzip => "elf.gz", + CoredumpCompression::None => "elf", + }; + format!("core-{}.{}", id, extension) +} + +fn calculate_available_space(config: &Config) -> Result { + let min_headroom = config.tmp_dir_min_headroom(); + let available = get_disk_space(&config.tmp_dir())?; + let has_headroom = available.exceeds(&min_headroom); + if !has_headroom { + return Ok(0); + } + Ok(min( + (available.bytes - min_headroom.bytes) as usize, + config.config_file.coredump.coredump_max_size, + )) +} + +impl From for CompressionAlgorithm { + fn from(compression: CoredumpCompression) -> Self { + match compression { + CoredumpCompression::Gzip => CompressionAlgorithm::Gzip, + CoredumpCompression::None => CompressionAlgorithm::None, + } + } +} + +fn init_kernel_logger(level: LevelFilter, capture_logs_tx: SyncSender) { + // kernlog::init() reads from the KERNLOG_LEVEL to set the level. There's no public interface + // to set it otherwise, so: if this environment variable is not set, set it according to the + // --verbose flag: + if var("KERNLOG_LEVEL").is_err() { + set_var("KERNLOG_LEVEL", level.as_str()); + } + // We fallback to standard output if verbose mode is enabled or if kernel is not available. + + let logger: Box = match KernelLog::from_env() { + Ok(logger) => Box::new(logger), + Err(_) => Box::new(cli::build_logger(level)), + }; + + let logger = Box::new(CoreHandlerLogWrapper::new( + logger, + capture_logs_tx, + CAPTURE_LOG_MAX_LEVEL, + )); + log::set_boxed_logger(logger).unwrap(); + + // Set the max log level to the max of the log level and the capture log level. + // This is necessary because the log macros will completely disable calls if the level + // is below the max level. + let max_level = max(level, CAPTURE_LOG_MAX_LEVEL); + log::set_max_level(max_level); +} diff --git a/memfaultd/src/cli/memfault_core_handler/procfs.rs b/memfaultd/src/cli/memfault_core_handler/procfs.rs new file mode 100644 index 0000000..e697f91 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/procfs.rs @@ -0,0 +1,79 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::fs::File; +use std::io::{Read, Seek, SeekFrom}; + +use eyre::{eyre, Result}; +use procfs::process::{MemoryMap, MemoryMaps}; + +use crate::cli::memfault_core_handler::memory_range::MemoryRange; +use crate::cli::memfault_core_handler::ElfPtrSize; + +/// Opens /proc//mem for reading. +pub fn proc_mem_stream(pid: i32) -> Result { + let proc_mem_stream = File::open(format!("/proc/{}/mem", pid))?; + Ok(proc_mem_stream) +} + +/// Reads memory from /proc//mem into a buffer. +pub fn read_proc_mem( + proc_mem_stream: &mut P, + vaddr: ElfPtrSize, + size: ElfPtrSize, +) -> Result> { + // Ignore unnecessary cast here as it is needed on 32-bit systems. + #[allow(clippy::unnecessary_cast)] + proc_mem_stream.seek(SeekFrom::Start(vaddr as u64))?; + let mut buf = vec![0; size as usize]; + proc_mem_stream.read_exact(&mut buf)?; + Ok(buf) +} + +pub fn read_proc_cmdline(cmd_line_stream: &mut P) -> Result { + let mut cmd_line_buf = Vec::new(); + cmd_line_stream.read_to_end(&mut cmd_line_buf)?; + + Ok(String::from_utf8_lossy(&cmd_line_buf).into_owned()) +} + +/// Wrapper trait for reading /proc//maps. +/// +/// Provides a useful abstraction that can be mocked out for testing. +pub trait ProcMaps { + fn get_process_maps(&mut self) -> Result>; +} + +/// Direct implementation of ProcMaps that reads from /proc//maps file. +/// +/// This is the default implementation used in production. It simply reads directly from +/// the file and returns the parsed memory ranges. +#[derive(Debug)] +pub struct ProcMapsImpl { + pid: i32, +} + +impl ProcMapsImpl { + pub fn new(pid: i32) -> Self { + Self { pid } + } +} + +impl ProcMaps for ProcMapsImpl { + fn get_process_maps(&mut self) -> Result> { + let maps_file_name = format!("/proc/{}/maps", self.pid); + + Ok(MemoryMaps::from_path(maps_file_name) + .map_err(|e| eyre!("Failed to read /proc/{}/maps: {}", self.pid, e))? + .memory_maps) + } +} + +impl From<&MemoryMap> for MemoryRange { + fn from(m: &MemoryMap) -> Self { + MemoryRange { + start: m.address.0 as ElfPtrSize, + end: m.address.1 as ElfPtrSize, + } + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/r_debug.rs b/memfaultd/src/cli/memfault_core_handler/r_debug.rs new file mode 100644 index 0000000..cb0442b --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/r_debug.rs @@ -0,0 +1,97 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{Read, Seek, SeekFrom}; + +use eyre::{eyre, Result}; + +use crate::cli::memfault_core_handler::ElfPtrSize; +use crate::util::mem::AsBytes; + +/// "Rendezvous structures used by the run-time dynamic linker to +/// communicate details of shared object loading to the debugger." +/// See glibc's elf/link.h +/// https://sourceware.org/git/?p=glibc.git;a=blob;f=elf/link.h;h=3b5954d9818e8ea9f35638c55961f861f6ae6057 + +// TODO: MFLT-11643 Add support for r_debug_extended + +/// The r_debug C structure from elf/link.h +#[derive(Debug, Clone, Copy, Default)] +#[repr(C)] +pub struct RDebug { + pub version: u32, + pub r_map: ElfPtrSize, + pub r_brk: ElfPtrSize, + pub r_state: u32, + pub r_ldbase: ElfPtrSize, +} + +/// The link_map C structure from elf/link.h +#[derive(Debug, Clone, Copy, Default)] +#[repr(C)] +pub struct LinkMap { + pub l_addr: ElfPtrSize, + /// Pointer to C-string. + pub l_name: ElfPtrSize, + pub l_ld: ElfPtrSize, + /// Pointer to next link map. + pub l_next: ElfPtrSize, + pub l_prev: ElfPtrSize, +} + +pub struct RDebugIter<'a, P: Read + Seek> { + proc_mem_stream: &'a mut P, + l_next: ElfPtrSize, +} + +impl<'a, P: Read + Seek> RDebugIter<'a, P> { + pub fn new(proc_mem_stream: &'a mut P, r_debug_addr: ElfPtrSize) -> Result { + // Ignore unnecessary cast here as it is needed on 32-bit systems. + #[allow(clippy::unnecessary_cast)] + proc_mem_stream.seek(SeekFrom::Start(r_debug_addr as u64))?; + let mut r_debug = RDebug::default(); + // SAFETY: From the point of view of this program, + // RDebug only contains scalar values where any value is allowed. + let data = unsafe { r_debug.as_mut_bytes() }; + proc_mem_stream + .read_exact(data) + .map_err(|e| eyre!("Failed to read r_debug: {}", e))?; + Ok(Self { + proc_mem_stream, + l_next: r_debug.r_map, + }) + } + + /// Returns the next tuple of the link map's virtual address and the link map itself, + /// or None if the end of the linked list has been reached. + fn read_next(&mut self) -> Result<(ElfPtrSize, LinkMap)> { + let vaddr = self.l_next; + // Ignore unnecessary cast here as it is needed on 32-bit systems. + #[allow(clippy::unnecessary_cast)] + self.proc_mem_stream.seek(SeekFrom::Start(vaddr as u64))?; + let mut link_map = LinkMap::default(); + // SAFETY: From the point of view of this program, + // LinkMap only contains scalar values where any value is allowed. + let data = unsafe { link_map.as_mut_bytes() }; + self.proc_mem_stream + .read_exact(data) + .map_err(|e| eyre!("Failed to read link_map: {}", e))?; + Ok((vaddr, link_map)) + } +} + +impl<'a, P: Read + Seek> Iterator for RDebugIter<'a, P> { + /// Tuple of the link map's virtual address and the link map itself. + type Item = (ElfPtrSize, LinkMap); + + fn next(&mut self) -> Option { + if self.l_next == 0 { + return None; + } + + self.read_next().ok().map(|(vaddr, link_map)| { + self.l_next = link_map.l_next; + (vaddr, link_map) + }) + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_debug_data.snap b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_debug_data.snap new file mode 100644 index 0000000..fbc0986 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_debug_data.snap @@ -0,0 +1,36 @@ +--- +source: memfaultd/src/cli/memfault_core_handler/core_elf_memfault_note.rs +expression: deser_capture_logs +--- +Map( + [ + ( + Text( + "schema_version", + ), + Integer( + Integer( + 1, + ), + ), + ), + ( + Text( + "capture_logs", + ), + Array( + [ + Text( + "Error 1", + ), + Text( + "Error 2", + ), + Text( + "Error 3", + ), + ], + ), + ), + ], +) diff --git a/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_metadata_as_map@app_logs.snap b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_metadata_as_map@app_logs.snap new file mode 100644 index 0000000..03d4449 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_metadata_as_map@app_logs.snap @@ -0,0 +1,165 @@ +--- +source: memfaultd/src/cli/memfault_core_handler/core_elf_memfault_note.rs +expression: deser_map +--- +Map( + [ + ( + Integer( + Integer( + 1, + ), + ), + Integer( + Integer( + 1, + ), + ), + ), + ( + Integer( + Integer( + 2, + ), + ), + Text( + "SDK_VERSION", + ), + ), + ( + Integer( + Integer( + 3, + ), + ), + Integer( + Integer( + 1234, + ), + ), + ), + ( + Integer( + Integer( + 4, + ), + ), + Text( + "12345678", + ), + ), + ( + Integer( + Integer( + 5, + ), + ), + Text( + "evt", + ), + ), + ( + Integer( + Integer( + 6, + ), + ), + Text( + "main", + ), + ), + ( + Integer( + Integer( + 7, + ), + ), + Text( + "1.0.0", + ), + ), + ( + Integer( + Integer( + 8, + ), + ), + Text( + "binary -a -b -c", + ), + ), + ( + Integer( + Integer( + 9, + ), + ), + Map( + [ + ( + Text( + "type", + ), + Text( + "kernel_selection", + ), + ), + ], + ), + ), + ( + Integer( + Integer( + 10, + ), + ), + Map( + [ + ( + Text( + "logs", + ), + Array( + [ + Text( + "Error 1", + ), + Text( + "Error 2", + ), + Text( + "Error 3", + ), + ], + ), + ), + ( + Text( + "format", + ), + Map( + [ + ( + Text( + "id", + ), + Text( + "v1", + ), + ), + ( + Text( + "serialization", + ), + Text( + "json-lines", + ), + ), + ], + ), + ), + ], + ), + ), + ], +) diff --git a/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_metadata_as_map@kernel_selection.snap b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_metadata_as_map@kernel_selection.snap new file mode 100644 index 0000000..690ef76 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_metadata_as_map@kernel_selection.snap @@ -0,0 +1,119 @@ +--- +source: memfaultd/src/cli/memfault_core_handler/core_elf_memfault_note.rs +expression: deser_map +--- +Map( + [ + ( + Integer( + Integer( + 1, + ), + ), + Integer( + Integer( + 1, + ), + ), + ), + ( + Integer( + Integer( + 2, + ), + ), + Text( + "SDK_VERSION", + ), + ), + ( + Integer( + Integer( + 3, + ), + ), + Integer( + Integer( + 1234, + ), + ), + ), + ( + Integer( + Integer( + 4, + ), + ), + Text( + "12345678", + ), + ), + ( + Integer( + Integer( + 5, + ), + ), + Text( + "evt", + ), + ), + ( + Integer( + Integer( + 6, + ), + ), + Text( + "main", + ), + ), + ( + Integer( + Integer( + 7, + ), + ), + Text( + "1.0.0", + ), + ), + ( + Integer( + Integer( + 8, + ), + ), + Text( + "binary -a -b -c", + ), + ), + ( + Integer( + Integer( + 9, + ), + ), + Map( + [ + ( + Text( + "type", + ), + Text( + "kernel_selection", + ), + ), + ], + ), + ), + ( + Integer( + Integer( + 10, + ), + ), + Null, + ), + ], +) diff --git a/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_metadata_as_map@threads.snap b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_metadata_as_map@threads.snap new file mode 100644 index 0000000..65d4c87 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_metadata_as_map@threads.snap @@ -0,0 +1,129 @@ +--- +source: memfaultd/src/cli/memfault_core_handler/core_elf_memfault_note.rs +expression: deser_map +--- +Map( + [ + ( + Integer( + Integer( + 1, + ), + ), + Integer( + Integer( + 1, + ), + ), + ), + ( + Integer( + Integer( + 2, + ), + ), + Text( + "SDK_VERSION", + ), + ), + ( + Integer( + Integer( + 3, + ), + ), + Integer( + Integer( + 1234, + ), + ), + ), + ( + Integer( + Integer( + 4, + ), + ), + Text( + "12345678", + ), + ), + ( + Integer( + Integer( + 5, + ), + ), + Text( + "evt", + ), + ), + ( + Integer( + Integer( + 6, + ), + ), + Text( + "main", + ), + ), + ( + Integer( + Integer( + 7, + ), + ), + Text( + "1.0.0", + ), + ), + ( + Integer( + Integer( + 8, + ), + ), + Text( + "binary -a -b -c", + ), + ), + ( + Integer( + Integer( + 9, + ), + ), + Map( + [ + ( + Text( + "type", + ), + Text( + "threads", + ), + ), + ( + Text( + "max_thread_size_kib", + ), + Integer( + Integer( + 32, + ), + ), + ), + ], + ), + ), + ( + Integer( + Integer( + 10, + ), + ), + Null, + ), + ], +) diff --git a/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_note__test__iterate_elf_notes_with_fixture.snap b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_note__test__iterate_elf_notes_with_fixture.snap new file mode 100644 index 0000000..5812d39 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_note__test__iterate_elf_notes_with_fixture.snap @@ -0,0 +1,756 @@ +--- +source: memfaultd/src/cli/memfault_core_handler/core_elf_note.rs +expression: notes +--- +[ + ProcessStatus( + ProcessStatusNote { + si_signo: 11, + si_code: 0, + si_errno: 0, + pr_cursig: 11, + pad0: 0, + pr_sigpend: 0, + pr_sighold: 0, + pr_pid: 1757063, + pr_ppid: 4514, + pr_pgrp: 1756783, + pr_sid: 1706201, + pr_utime: ProcessTimeVal { + tv_sec: 0, + tv_usec: 0, + }, + pr_stime: ProcessTimeVal { + tv_sec: 0, + tv_usec: 0, + }, + pr_cutime: ProcessTimeVal { + tv_sec: 0, + tv_usec: 0, + }, + pr_cstime: ProcessTimeVal { + tv_sec: 0, + tv_usec: 0, + }, + pr_reg: user_regs_struct { + r15: 140723164295904, + r14: 1, + r13: 140723164295824, + r12: 140723155918848, + rbp: 0, + rbx: 140723155914752, + r11: 582, + r10: 140192614273224, + r9: 140192606909824, + r8: 0, + rax: 0, + rcx: 42, + rdx: 21885, + rsi: 0, + rdi: 0, + orig_rax: 18446744073709551615, + rip: 93999345134665, + cs: 51, + eflags: 66118, + rsp: 140723164290544, + ss: 43, + fs_base: 140192606909824, + gs_base: 0, + ds: 0, + es: 0, + fs: 0, + gs: 0, + }, + pr_fpvalid: 1, + pad1: 0, + }, + ), + Auxv( + [ + AuxvEntry { + key: 33, + value: 140723164557312, + }, + AuxvEntry { + key: 51, + value: 3632, + }, + AuxvEntry { + key: 16, + value: 3219913727, + }, + AuxvEntry { + key: 6, + value: 4096, + }, + AuxvEntry { + key: 17, + value: 100, + }, + AuxvEntry { + key: 3, + value: 93999343464512, + }, + AuxvEntry { + key: 4, + value: 56, + }, + AuxvEntry { + key: 5, + value: 14, + }, + AuxvEntry { + key: 7, + value: 140192614031360, + }, + AuxvEntry { + key: 8, + value: 0, + }, + AuxvEntry { + key: 9, + value: 93999344771504, + }, + AuxvEntry { + key: 11, + value: 1000, + }, + AuxvEntry { + key: 12, + value: 1000, + }, + AuxvEntry { + key: 13, + value: 1000, + }, + AuxvEntry { + key: 14, + value: 1000, + }, + AuxvEntry { + key: 23, + value: 0, + }, + AuxvEntry { + key: 25, + value: 140723164297705, + }, + AuxvEntry { + key: 26, + value: 2, + }, + AuxvEntry { + key: 31, + value: 140723164307423, + }, + AuxvEntry { + key: 15, + value: 140723164297721, + }, + AuxvEntry { + key: 0, + value: 0, + }, + ], + ), + File( + FileNote { + page_size: 4096, + mapped_files: [ + MappedFile { + path: Some( + "/home/blake/git/memfault-linux-sdk-internal/meta-memfault/recipes-memfault/memfaultd/files/target/debug/memfaultctl", + ), + start_addr: 93999343464448, + end_addr: 93999344754688, + page_offset: 0, + }, + MappedFile { + path: Some( + "/home/blake/git/memfault-linux-sdk-internal/meta-memfault/recipes-memfault/memfaultd/files/target/debug/memfaultctl", + ), + start_addr: 93999344754688, + end_addr: 93999353466880, + page_offset: 315, + }, + MappedFile { + path: Some( + "/home/blake/git/memfault-linux-sdk-internal/meta-memfault/recipes-memfault/memfaultd/files/target/debug/memfaultctl", + ), + start_addr: 93999353466880, + end_addr: 93999355916288, + page_offset: 2442, + }, + MappedFile { + path: Some( + "/home/blake/git/memfault-linux-sdk-internal/meta-memfault/recipes-memfault/memfaultd/files/target/debug/memfaultctl", + ), + start_addr: 93999355916288, + end_addr: 93999356571648, + page_offset: 3039, + }, + MappedFile { + path: Some( + "/home/blake/git/memfault-linux-sdk-internal/meta-memfault/recipes-memfault/memfaultd/files/target/debug/memfaultctl", + ), + start_addr: 93999356571648, + end_addr: 93999356583936, + page_offset: 3199, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgpg-error.so.0.32.1", + ), + start_addr: 140192606912512, + end_addr: 140192606928896, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgpg-error.so.0.32.1", + ), + start_addr: 140192606928896, + end_addr: 140192607019008, + page_offset: 4, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgpg-error.so.0.32.1", + ), + start_addr: 140192607019008, + end_addr: 140192607059968, + page_offset: 26, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgpg-error.so.0.32.1", + ), + start_addr: 140192607059968, + end_addr: 140192607064064, + page_offset: 35, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgpg-error.so.0.32.1", + ), + start_addr: 140192607064064, + end_addr: 140192607068160, + page_offset: 36, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcrypt.so.20.3.4", + ), + start_addr: 140192607068160, + end_addr: 140192607129600, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcrypt.so.20.3.4", + ), + start_addr: 140192607129600, + end_addr: 140192608071680, + page_offset: 15, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcrypt.so.20.3.4", + ), + start_addr: 140192608071680, + end_addr: 140192608325632, + page_offset: 245, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcrypt.so.20.3.4", + ), + start_addr: 140192608325632, + end_addr: 140192608329728, + page_offset: 307, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcrypt.so.20.3.4", + ), + start_addr: 140192608329728, + end_addr: 140192608342016, + page_offset: 307, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcrypt.so.20.3.4", + ), + start_addr: 140192608342016, + end_addr: 140192608366592, + page_offset: 310, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libcap.so.2.44", + ), + start_addr: 140192608370688, + end_addr: 140192608382976, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libcap.so.2.44", + ), + start_addr: 140192608382976, + end_addr: 140192608399360, + page_offset: 3, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libcap.so.2.44", + ), + start_addr: 140192608399360, + end_addr: 140192608407552, + page_offset: 7, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libcap.so.2.44", + ), + start_addr: 140192608407552, + end_addr: 140192608411648, + page_offset: 8, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libcap.so.2.44", + ), + start_addr: 140192608411648, + end_addr: 140192608415744, + page_offset: 9, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblz4.so.1.9.3", + ), + start_addr: 140192608415744, + end_addr: 140192608423936, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblz4.so.1.9.3", + ), + start_addr: 140192608423936, + end_addr: 140192608526336, + page_offset: 2, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblz4.so.1.9.3", + ), + start_addr: 140192608526336, + end_addr: 140192608534528, + page_offset: 27, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblz4.so.1.9.3", + ), + start_addr: 140192608534528, + end_addr: 140192608538624, + page_offset: 29, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblz4.so.1.9.3", + ), + start_addr: 140192608538624, + end_addr: 140192608542720, + page_offset: 29, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblz4.so.1.9.3", + ), + start_addr: 140192608542720, + end_addr: 140192608546816, + page_offset: 30, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libzstd.so.1.4.8", + ), + start_addr: 140192608546816, + end_addr: 140192608587776, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libzstd.so.1.4.8", + ), + start_addr: 140192608587776, + end_addr: 140192609316864, + page_offset: 10, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libzstd.so.1.4.8", + ), + start_addr: 140192609316864, + end_addr: 140192609386496, + page_offset: 188, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libzstd.so.1.4.8", + ), + start_addr: 140192609386496, + end_addr: 140192609390592, + page_offset: 204, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libzstd.so.1.4.8", + ), + start_addr: 140192609390592, + end_addr: 140192609394688, + page_offset: 205, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblzma.so.5.2.5", + ), + start_addr: 140192609394688, + end_addr: 140192609406976, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblzma.so.5.2.5", + ), + start_addr: 140192609406976, + end_addr: 140192609517568, + page_offset: 3, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblzma.so.5.2.5", + ), + start_addr: 140192609517568, + end_addr: 140192609562624, + page_offset: 30, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblzma.so.5.2.5", + ), + start_addr: 140192609562624, + end_addr: 140192609566720, + page_offset: 40, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblzma.so.5.2.5", + ), + start_addr: 140192609566720, + end_addr: 140192609570816, + page_offset: 41, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libm.so.6", + ), + start_addr: 140192609570816, + end_addr: 140192609628160, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libm.so.6", + ), + start_addr: 140192609628160, + end_addr: 140192610136064, + page_offset: 14, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libm.so.6", + ), + start_addr: 140192610136064, + end_addr: 140192610508800, + page_offset: 138, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libm.so.6", + ), + start_addr: 140192610508800, + end_addr: 140192610512896, + page_offset: 228, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libm.so.6", + ), + start_addr: 140192610512896, + end_addr: 140192610516992, + page_offset: 229, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libc.so.6", + ), + start_addr: 140192610516992, + end_addr: 140192610680832, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libc.so.6", + ), + start_addr: 140192610680832, + end_addr: 140192612339712, + page_offset: 40, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libc.so.6", + ), + start_addr: 140192612339712, + end_addr: 140192612700160, + page_offset: 445, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libc.so.6", + ), + start_addr: 140192612700160, + end_addr: 140192612716544, + page_offset: 532, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libc.so.6", + ), + start_addr: 140192612716544, + end_addr: 140192612724736, + page_offset: 536, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcc_s.so.1", + ), + start_addr: 140192612790272, + end_addr: 140192612802560, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcc_s.so.1", + ), + start_addr: 140192612802560, + end_addr: 140192612896768, + page_offset: 3, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcc_s.so.1", + ), + start_addr: 140192612896768, + end_addr: 140192612913152, + page_offset: 26, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcc_s.so.1", + ), + start_addr: 140192612913152, + end_addr: 140192612917248, + page_offset: 29, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcc_s.so.1", + ), + start_addr: 140192612917248, + end_addr: 140192612921344, + page_offset: 30, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libconfig.so.9.2.0", + ), + start_addr: 140192612929536, + end_addr: 140192612941824, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libconfig.so.9.2.0", + ), + start_addr: 140192612941824, + end_addr: 140192612966400, + page_offset: 3, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libconfig.so.9.2.0", + ), + start_addr: 140192612966400, + end_addr: 140192612978688, + page_offset: 9, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libconfig.so.9.2.0", + ), + start_addr: 140192612978688, + end_addr: 140192612982784, + page_offset: 11, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libconfig.so.9.2.0", + ), + start_addr: 140192612982784, + end_addr: 140192612986880, + page_offset: 12, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libsystemd.so.0.32.0", + ), + start_addr: 140192612986880, + end_addr: 140192613064704, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libsystemd.so.0.32.0", + ), + start_addr: 140192613064704, + end_addr: 140192613584896, + page_offset: 19, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libsystemd.so.0.32.0", + ), + start_addr: 140192613584896, + end_addr: 140192613761024, + page_offset: 146, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libsystemd.so.0.32.0", + ), + start_addr: 140192613761024, + end_addr: 140192613765120, + page_offset: 189, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libsystemd.so.0.32.0", + ), + start_addr: 140192613765120, + end_addr: 140192613793792, + page_offset: 189, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libsystemd.so.0.32.0", + ), + start_addr: 140192613793792, + end_addr: 140192613797888, + page_offset: 196, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libz.so.1.2.11", + ), + start_addr: 140192613801984, + end_addr: 140192613810176, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libz.so.1.2.11", + ), + start_addr: 140192613810176, + end_addr: 140192613879808, + page_offset: 2, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libz.so.1.2.11", + ), + start_addr: 140192613879808, + end_addr: 140192613904384, + page_offset: 19, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libz.so.1.2.11", + ), + start_addr: 140192613904384, + end_addr: 140192613908480, + page_offset: 25, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libz.so.1.2.11", + ), + start_addr: 140192613908480, + end_addr: 140192613912576, + page_offset: 25, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libz.so.1.2.11", + ), + start_addr: 140192613912576, + end_addr: 140192613916672, + page_offset: 26, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2", + ), + start_addr: 140192614031360, + end_addr: 140192614039552, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2", + ), + start_addr: 140192614039552, + end_addr: 140192614211584, + page_offset: 2, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2", + ), + start_addr: 140192614211584, + end_addr: 140192614256640, + page_offset: 44, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2", + ), + start_addr: 140192614260736, + end_addr: 140192614268928, + page_offset: 55, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2", + ), + start_addr: 140192614268928, + end_addr: 140192614277120, + page_offset: 57, + }, + ], + incomplete: false, + }, + ), +] diff --git a/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_transformer__test__transform@kernel_selection.snap b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_transformer__test__transform@kernel_selection.snap new file mode 100644 index 0000000..dc66b7e --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_transformer__test__transform@kernel_selection.snap @@ -0,0 +1,387 @@ +--- +source: memfaultd/src/cli/memfault_core_handler/core_transformer.rs +expression: segments +--- +[ + ( + ProgramHeader { + p_type: "PT_NOTE", + p_flags: 0x0, + p_offset: 0x660, + p_vaddr: 0x0, + p_paddr: 0x0, + p_filesz: 0xdec, + p_memsz: 0x0, + p_align: 0, + }, + Buffer( + [], + ), + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x2000, + p_vaddr: 0x5587ae8bd000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x5, + p_offset: 0x3000, + p_vaddr: 0x5587ae8be000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x3000, + p_vaddr: 0x5587ae8bf000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x3000, + p_vaddr: 0x5587ae8c0000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x4000, + p_vaddr: 0x5587ae8c1000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x5000, + p_vaddr: 0x7f6e38cb1000, + p_paddr: 0x0, + p_filesz: 0x3000, + p_memsz: 0x3000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x8000, + p_vaddr: 0x7f6e38cb4000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x26000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x5, + p_offset: 0x9000, + p_vaddr: 0x7f6e38cda000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x155000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x9000, + p_vaddr: 0x7f6e38e2f000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x53000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x9000, + p_vaddr: 0x7f6e38e82000, + p_paddr: 0x0, + p_filesz: 0x4000, + p_memsz: 0x4000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0xd000, + p_vaddr: 0x7f6e38e86000, + p_paddr: 0x0, + p_filesz: 0x2000, + p_memsz: 0x2000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0xf000, + p_vaddr: 0x7f6e38e88000, + p_paddr: 0x0, + p_filesz: 0xd000, + p_memsz: 0xd000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x1c000, + p_vaddr: 0x7f6e38e99000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x10000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x5, + p_offset: 0x1d000, + p_vaddr: 0x7f6e38ea9000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x73000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x1d000, + p_vaddr: 0x7f6e38f1c000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x5a000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x1d000, + p_vaddr: 0x7f6e38f76000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x1e000, + p_vaddr: 0x7f6e38f77000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x1f000, + p_vaddr: 0x7f6e38f78000, + p_paddr: 0x0, + p_filesz: 0x2000, + p_memsz: 0x2000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x21000, + p_vaddr: 0x7f6e38f7a000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x5, + p_offset: 0x22000, + p_vaddr: 0x7f6e38f7b000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x25000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x22000, + p_vaddr: 0x7f6e38fa0000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0xa000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x22000, + p_vaddr: 0x7f6e38faa000, + p_paddr: 0x0, + p_filesz: 0x2000, + p_memsz: 0x2000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x24000, + p_vaddr: 0x7f6e38fac000, + p_paddr: 0x0, + p_filesz: 0x2000, + p_memsz: 0x2000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x26000, + p_vaddr: 0x7ffe4ba5b000, + p_paddr: 0x0, + p_filesz: 0x21000, + p_memsz: 0x21000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x47000, + p_vaddr: 0x7ffe4bb1b000, + p_paddr: 0x0, + p_filesz: 0x4000, + p_memsz: 0x4000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x5, + p_offset: 0x4b000, + p_vaddr: 0x7ffe4bb1f000, + p_paddr: 0x0, + p_filesz: 0x2000, + p_memsz: 0x2000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x1, + p_offset: 0x4d000, + p_vaddr: 0xffffffffff600000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_NOTE", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x0, + p_paddr: 0x0, + p_filesz: 0xb8, + p_memsz: 0x0, + p_align: 0, + }, + Buffer( + [], + ), + ), +] diff --git a/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_transformer__test__transform@threads_32k.snap b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_transformer__test__transform@threads_32k.snap new file mode 100644 index 0000000..451f0ba --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_transformer__test__transform@threads_32k.snap @@ -0,0 +1,244 @@ +--- +source: memfaultd/src/cli/memfault_core_handler/core_transformer.rs +expression: segments +--- +[ + ( + ProgramHeader { + p_type: "PT_NOTE", + p_flags: 0x0, + p_offset: 0x660, + p_vaddr: 0x0, + p_paddr: 0x0, + p_filesz: 0xdec, + p_memsz: 0x0, + p_align: 0, + }, + Buffer( + [], + ), + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x5587ae8bd000, + p_paddr: 0x0, + p_filesz: 0x39c, + p_memsz: 0x39c, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x5587ae8c0dc0, + p_paddr: 0x0, + p_filesz: 0x200, + p_memsz: 0x200, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38cb4000, + p_paddr: 0x0, + p_filesz: 0x3b4, + p_memsz: 0x3b4, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38e99000, + p_paddr: 0x0, + p_filesz: 0x30c, + p_memsz: 0x30c, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38f781d0, + p_paddr: 0x0, + p_filesz: 0xc, + p_memsz: 0xc, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38f781e0, + p_paddr: 0x0, + p_filesz: 0x28, + p_memsz: 0x28, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38f78740, + p_paddr: 0x0, + p_filesz: 0x20, + p_memsz: 0x20, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38f78760, + p_paddr: 0x0, + p_filesz: 0x28, + p_memsz: 0x28, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38f7a000, + p_paddr: 0x0, + p_filesz: 0x25c, + p_memsz: 0x25c, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38facad0, + p_paddr: 0x0, + p_filesz: 0x28, + p_memsz: 0x28, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38fad118, + p_paddr: 0x0, + p_filesz: 0x28, + p_memsz: 0x28, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38fad2e0, + p_paddr: 0x0, + p_filesz: 0x28, + p_memsz: 0x28, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38fad8a0, + p_paddr: 0x0, + p_filesz: 0x1, + p_memsz: 0x1, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38fad8b0, + p_paddr: 0x0, + p_filesz: 0x28, + p_memsz: 0x28, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7ffe4ba7a3c0, + p_paddr: 0x0, + p_filesz: 0x1c40, + p_memsz: 0x1c40, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7ffe4bb1f000, + p_paddr: 0x0, + p_filesz: 0x554, + p_memsz: 0x554, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_NOTE", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x0, + p_paddr: 0x0, + p_filesz: 0xc8, + p_memsz: 0x0, + p_align: 0, + }, + Buffer( + [], + ), + ), +] diff --git a/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_transformer__test__transform@threads_32k_no_filter_support.snap b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_transformer__test__transform@threads_32k_no_filter_support.snap new file mode 100644 index 0000000..dc66b7e --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_transformer__test__transform@threads_32k_no_filter_support.snap @@ -0,0 +1,387 @@ +--- +source: memfaultd/src/cli/memfault_core_handler/core_transformer.rs +expression: segments +--- +[ + ( + ProgramHeader { + p_type: "PT_NOTE", + p_flags: 0x0, + p_offset: 0x660, + p_vaddr: 0x0, + p_paddr: 0x0, + p_filesz: 0xdec, + p_memsz: 0x0, + p_align: 0, + }, + Buffer( + [], + ), + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x2000, + p_vaddr: 0x5587ae8bd000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x5, + p_offset: 0x3000, + p_vaddr: 0x5587ae8be000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x3000, + p_vaddr: 0x5587ae8bf000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x3000, + p_vaddr: 0x5587ae8c0000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x4000, + p_vaddr: 0x5587ae8c1000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x5000, + p_vaddr: 0x7f6e38cb1000, + p_paddr: 0x0, + p_filesz: 0x3000, + p_memsz: 0x3000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x8000, + p_vaddr: 0x7f6e38cb4000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x26000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x5, + p_offset: 0x9000, + p_vaddr: 0x7f6e38cda000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x155000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x9000, + p_vaddr: 0x7f6e38e2f000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x53000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x9000, + p_vaddr: 0x7f6e38e82000, + p_paddr: 0x0, + p_filesz: 0x4000, + p_memsz: 0x4000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0xd000, + p_vaddr: 0x7f6e38e86000, + p_paddr: 0x0, + p_filesz: 0x2000, + p_memsz: 0x2000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0xf000, + p_vaddr: 0x7f6e38e88000, + p_paddr: 0x0, + p_filesz: 0xd000, + p_memsz: 0xd000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x1c000, + p_vaddr: 0x7f6e38e99000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x10000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x5, + p_offset: 0x1d000, + p_vaddr: 0x7f6e38ea9000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x73000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x1d000, + p_vaddr: 0x7f6e38f1c000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x5a000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x1d000, + p_vaddr: 0x7f6e38f76000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x1e000, + p_vaddr: 0x7f6e38f77000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x1f000, + p_vaddr: 0x7f6e38f78000, + p_paddr: 0x0, + p_filesz: 0x2000, + p_memsz: 0x2000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x21000, + p_vaddr: 0x7f6e38f7a000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x5, + p_offset: 0x22000, + p_vaddr: 0x7f6e38f7b000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x25000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x22000, + p_vaddr: 0x7f6e38fa0000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0xa000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x22000, + p_vaddr: 0x7f6e38faa000, + p_paddr: 0x0, + p_filesz: 0x2000, + p_memsz: 0x2000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x24000, + p_vaddr: 0x7f6e38fac000, + p_paddr: 0x0, + p_filesz: 0x2000, + p_memsz: 0x2000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x26000, + p_vaddr: 0x7ffe4ba5b000, + p_paddr: 0x0, + p_filesz: 0x21000, + p_memsz: 0x21000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x47000, + p_vaddr: 0x7ffe4bb1b000, + p_paddr: 0x0, + p_filesz: 0x4000, + p_memsz: 0x4000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x5, + p_offset: 0x4b000, + p_vaddr: 0x7ffe4bb1f000, + p_paddr: 0x0, + p_filesz: 0x2000, + p_memsz: 0x2000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x1, + p_offset: 0x4d000, + p_vaddr: 0xffffffffff600000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_NOTE", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x0, + p_paddr: 0x0, + p_filesz: 0xb8, + p_memsz: 0x0, + p_align: 0, + }, + Buffer( + [], + ), + ), +] diff --git a/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__find_dynamic__test__find_dynamic_linker_ranges.snap b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__find_dynamic__test__find_dynamic_linker_ranges.snap new file mode 100644 index 0000000..9848a1f --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__find_dynamic__test__find_dynamic_linker_ranges.snap @@ -0,0 +1,58 @@ +--- +source: memfaultd/src/cli/memfault_core_handler/find_dynamic.rs +expression: output +--- +[ + MemoryRange { + start: "0x5587ae8bd040", + end: "0x5587ae8bd318", + }, + MemoryRange { + start: "0x5587ae8c0dc0", + end: "0x5587ae8c0fc0", + }, + MemoryRange { + start: "0x7f6e38fad118", + end: "0x7f6e38fad140", + }, + MemoryRange { + start: "0x7f6e38fad2e0", + end: "0x7f6e38fad308", + }, + MemoryRange { + start: "0x7f6e38fad8b0", + end: "0x7f6e38fad8d8", + }, + MemoryRange { + start: "0x7f6e38f781e0", + end: "0x7f6e38f78208", + }, + MemoryRange { + start: "0x7f6e38f78760", + end: "0x7f6e38f78788", + }, + MemoryRange { + start: "0x7f6e38facad0", + end: "0x7f6e38facaf8", + }, + MemoryRange { + start: "0x7f6e38fad8a0", + end: "0x7f6e38fad8a1", + }, + MemoryRange { + start: "0x7ffe4bb1f371", + end: "0x7ffe4bb1f381", + }, + MemoryRange { + start: "0x7f6e38f781d0", + end: "0x7f6e38f781dc", + }, + MemoryRange { + start: "0x7f6e38f78740", + end: "0x7f6e38f78760", + }, + MemoryRange { + start: "0x5587ae8bd318", + end: "0x5587ae8bd334", + }, +] diff --git a/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__log_wrapper__test__log_saving.snap b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__log_wrapper__test__log_saving.snap new file mode 100644 index 0000000..415c90b --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__log_wrapper__test__log_saving.snap @@ -0,0 +1,10 @@ +--- +source: memfaultd/src/cli/memfault_core_handler/log_wrapper.rs +expression: errors +--- +[ + "ERROR test:71 - Test message", + "WARN test:71 - Test message", + "INFO test:71 - Test message", + "DEBUG test:71 - Test message" +] diff --git a/memfaultd/src/cli/memfault_core_handler/test_utils.rs b/memfaultd/src/cli/memfault_core_handler/test_utils.rs new file mode 100644 index 0000000..f5d4a51 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/test_utils.rs @@ -0,0 +1,231 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::cli::memfault_core_handler::core_writer::{CoreWriter, SegmentData}; +use crate::cli::memfault_core_handler::elf; +use crate::cli::memfault_core_handler::procfs::ProcMaps; +use elf::{header::Header, program_header::ProgramHeader}; +// NOTE: Using the "universal" (width-agnostic types in the test): +use goblin::elf::{ + program_header::{PF_R, PF_W, PF_X, PT_LOAD}, + Elf, ProgramHeader as UniversalProgramHeader, +}; +use procfs::process::{MMPermissions, MMapPath, MemoryMap}; +use std::fs::read; +use std::io::{Cursor, Error, ErrorKind, Read, Seek, SeekFrom, Take}; +use std::path::Path; +use take_mut::take; + +pub fn build_test_header(class: u8, endianness: u8, machine: u16) -> Header { + let mut e_ident = [0u8; 16]; + e_ident[..elf::header::SELFMAG].copy_from_slice(elf::header::ELFMAG); + e_ident[elf::header::EI_CLASS] = class; + e_ident[elf::header::EI_DATA] = endianness; + e_ident[elf::header::EI_VERSION] = elf::header::EV_CURRENT; + + Header { + e_phoff: elf::header::SIZEOF_EHDR.try_into().unwrap(), + e_phentsize: elf::program_header::SIZEOF_PHDR.try_into().unwrap(), + e_ehsize: elf::header::SIZEOF_EHDR.try_into().unwrap(), + e_version: elf::header::EV_CURRENT.try_into().unwrap(), + e_phnum: 0, + e_machine: machine, + e_ident, + ..Default::default() + } +} + +pub struct MockCoreWriter<'a> { + pub output_size: usize, + pub segments: &'a mut Vec<(ProgramHeader, SegmentData)>, +} + +impl<'a> MockCoreWriter<'a> { + pub fn new(segments: &mut Vec<(ProgramHeader, SegmentData)>) -> MockCoreWriter { + MockCoreWriter { + output_size: 0, + segments, + } + } +} + +impl<'a> CoreWriter for MockCoreWriter<'a> { + fn add_segment(&mut self, program_header: ProgramHeader, data: SegmentData) { + self.segments.push((program_header, data)); + } + + fn write(&mut self) -> eyre::Result<()> { + Ok(()) + } + + fn calc_output_size(&self) -> usize { + self.output_size + } +} + +/// A fake `ProcMaps` implementation that uses the memory ranges from PT_LOAD segments. +pub struct FakeProcMaps { + maps: Vec, +} + +impl FakeProcMaps { + /// Creates a new `FakeProcMaps` from the given core.elf file. + pub fn new_from_path>(core_elf_path: P) -> eyre::Result { + let data = &read(core_elf_path)?; + + let load_segments = load_segments_from_buffer(data)?; + let maps = load_segments + .iter() + .map(|ph| MemoryMap { + address: (ph.p_vaddr, ph.p_vaddr + ph.p_memsz), + perms: ph_flags_to_perms(ph.p_flags), + // All below is (likely) incorrect, but we cannot get this info from the ELF file: + offset: 0, + dev: (0, 0), + inode: 0, + pathname: MMapPath::Other("Unknown".into()), + extension: Default::default(), + }) + .collect(); + + Ok(Self { maps }) + } +} + +impl ProcMaps for FakeProcMaps { + fn get_process_maps(&mut self) -> eyre::Result> { + Ok(self.maps.clone()) + } +} + +/// A fake `/proc//mem` stream that uses the memory contents from PT_LOAD segments of a +/// core.elf as the data in the `/proc//mem` file. +pub struct FakeProcMem { + // Note: as per the ELF spec, are expected to be sorted by v_addr: + load_segments: Vec, + inner: Take>>, +} + +impl FakeProcMem { + pub fn new_from_path>(core_elf_path: P) -> eyre::Result { + let data = read(core_elf_path)?; + Self::new(data) + } + pub fn new(data: Vec) -> eyre::Result { + let load_segments = load_segments_from_buffer(&data)?; + + Ok(FakeProcMem { + inner: FakeProcMem::make_inner(data, &load_segments[0])?, + load_segments, + }) + } + + /// Creates a Read stream that corresponds to the given program header. + /// The stream is a "view" into the segment in the coredump data buffer. + fn make_inner( + data: Vec, + ph: &UniversalProgramHeader, + ) -> eyre::Result>>> { + let mut cursor = Cursor::new(data); + // Ignore unnecessary cast here as it is needed on 32-bit systems. + #[allow(clippy::unnecessary_cast)] + cursor.seek(SeekFrom::Start(ph.p_offset as u64))?; + Ok(cursor.take(ph.p_filesz)) + } + + fn file_offset_to_vaddr(&self, offset: u64) -> Option { + self.load_segments + .iter() + .find(|ph| { + let start = ph.p_offset; + let end = start + ph.p_filesz; + offset >= start && offset < end + }) + .map(|ph| ph.p_vaddr + (offset - ph.p_offset)) + } +} + +impl Read for FakeProcMem { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.inner.read(buf) + } +} + +impl Seek for FakeProcMem { + fn seek(&mut self, pos: SeekFrom) -> std::io::Result { + if let SeekFrom::End(_) = pos {} + + // The seek offset in /proc//mem is the virtual address of the process memory: + let vaddr = match pos { + SeekFrom::Start(pos) => Ok(pos), + SeekFrom::Current(pos) => self + .stream_position() + .map(|p| pos.checked_add(p as i64).unwrap() as u64), + SeekFrom::End(_) => Err(Error::new(ErrorKind::Other, "Not implemented")), + } + .unwrap(); + + // Find the PT_LOAD segment's program header that contains the requested vaddr: + let ph = self.load_segments.iter().find(|ph| { + let start = ph.p_vaddr; + let end = start + ph.p_memsz; + vaddr >= start && vaddr < end + }); + + match ph { + Some(ph) => { + // When seek() is called, always create a new inner stream that contains the + // requested seek position (vaddr), even if the seek position is within the current + // inner: + take(&mut self.inner, |inner| { + let data = inner.into_inner().into_inner(); + FakeProcMem::make_inner(data, ph).unwrap() + }); + // Seek within the new inner stream to the requested vaddr: + self.inner + .get_mut() + .seek(SeekFrom::Start(ph.p_offset + vaddr - ph.p_vaddr)) + } + None => Err(Error::new( + ErrorKind::Other, + format!("Invalid seek position: {:#x}", vaddr), + )), + } + } + + fn stream_position(&mut self) -> std::io::Result { + let inner_pos = self.inner.get_mut().stream_position()?; + self.file_offset_to_vaddr(inner_pos) + .ok_or_else(|| Error::new(ErrorKind::Other, "Invalid stream position")) + } +} + +fn load_segments_from_buffer(data: &[u8]) -> eyre::Result> { + let elf = Elf::parse(data)?; + let load_segments: Vec = elf + .program_headers + .iter() + .filter_map(|ph| { + if ph.p_type == PT_LOAD { + Some(ph.clone()) + } else { + None + } + }) + .collect(); + Ok(load_segments) +} + +fn ph_flags_to_perms(flags: u32) -> MMPermissions { + let mut perms = MMPermissions::empty(); + if flags & PF_R != 0 { + perms |= MMPermissions::READ; + } + if flags & PF_W != 0 { + perms |= MMPermissions::WRITE; + } + if flags & PF_X != 0 { + perms |= MMPermissions::EXECUTE; + } + perms +} diff --git a/memfaultd/src/cli/memfault_watch/buffer.rs b/memfaultd/src/cli/memfault_watch/buffer.rs new file mode 100644 index 0000000..5715289 --- /dev/null +++ b/memfaultd/src/cli/memfault_watch/buffer.rs @@ -0,0 +1,120 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + io::{BufRead, Result, Write}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }, +}; + +use log::error; + +pub(crate) static STOP_THREADS: AtomicBool = AtomicBool::new(false); + +/// This function monitors the reader and calls `read_and_double_write` on the two inputs. +/// It handles the buffering and the quitting in the case where the `read()` call returns +/// an error (in the case of an OS-level problem retrieving the process's stream) and +/// also the case where the program no longer as any more bytes to read. +pub(crate) fn monitor_and_buffer( + reader: impl BufRead, + mut io_writer: &mut impl Write, + file_writer: Arc>, +) { + // Create an initial buffer with a large starting capacity, to avoid the rapid + // allocation that can occur when inserting over and over again one by one + // Refs: https://blog.mozilla.org/nnethercote/2020/08/05/how-to-speed-up-the-rust-compiler-some-more-in-2020/ + // (under miscellaneous, discusses how rust-lang solved an adjacent issue by + // making the default allocation a minimum of 4) + + // The default value of 240 was chosen as a rough estimate of the average line size + // of a log. If there are lines significantly longer, the higher capacity will + // persist after clearing. + let mut buffer = Vec::with_capacity(240); + let mut reader = reader; + loop { + // Here we catch the case where we need to stop the threads because of an error + // reaching the process. We call join() in the upper loop, but this just waits + // for each thread to stop naturally. + if STOP_THREADS.load(Ordering::Relaxed) { + break; + } + + // Utilize the Take adapter to temporarily limit the amount we + // can ever read from *this limiter*. This avoids the problem later on in + // `read_and_double_write`, which calls `read_until` in order to read + // uninterrupted until reaching a \n. If a program's output prints extremely long + // lines, without ever printing a \n (for example, if a print subroutine breaks), + // we still want to handle that by limiting to this value. + let mut temporary_limiter = reader.take(8192); + let read_result = read_and_double_write( + &mut buffer, + &mut temporary_limiter, + &mut io_writer, + file_writer.clone(), + ); + + //Here we catch the case where we successfully read zero bytes. This should + // _only_ happen when there is _no more_ data to be read - because otherwise, + // any implementation of `Read` should _always block_ upon trying to read, + // waiting until there's _something_ to return (even Ok(1)). + + // TLDR: Ok(0) should only occur if the program exited. So, break. + if let Ok(0) = read_result { + break; + } + // Here we catch any `io`-related error when reading. + if let Err(e) = read_result { + error!("Error reading data: {e}"); + break; + } + // Clear the buffer - zeroes out the contents but does not affect the capacity + // Leaving the capacity untouched for two reasons: + // 1. Avoiding reallocating if lines are consistently longer than 240 bytes + // 2. When considering the allocation, it's unlikely for some extremely long + // line to hold a large buffer + buffer.clear(); + // Here, we reset the reader to remove the limiter, by consuming the Take and + // saving it back as the wrapped reader. + reader = temporary_limiter.into_inner(); + } + + if let Err(e) = file_writer + .lock() + .expect("Failed to lock file writer") + .flush() + { + error!("Error flushing file writer. {e}"); + } +} + +/// This function is meant to be called in a loop - it reads as much as it can from the +/// `reader` into the buffer, and writes to each of the writers. It handles the case of +/// no data currently available, as well as a broken reader, by returning an +/// std::io::Result. The contained usize success value is the number of bytes read. +/// Returns an error in the case of a generic io::Error, whether in the read, or in +/// either of the two writes/flushes. +/// The file writer is not flushed, as it's wrapped in a BufWriter and only needs to be +/// flushed at the end. +#[inline(always)] +pub(crate) fn read_and_double_write( + buffer: &mut Vec, + reader: &mut impl BufRead, + io_writer: &mut impl Write, + file_writer: Arc>, +) -> Result { + // We use read_until here to continuously read bytes until we reach a newline. + match reader.read_until(b'\n', buffer)? { + 0 => Ok(0), + bytes_read => { + io_writer.write_all(buffer)?; + io_writer.flush()?; + file_writer + .lock() + .expect("Failed to lock file writer") + .write_all(buffer)?; + Ok(bytes_read) + } + } +} diff --git a/memfaultd/src/cli/memfault_watch/mod.rs b/memfaultd/src/cli/memfault_watch/mod.rs new file mode 100644 index 0000000..2d63479 --- /dev/null +++ b/memfaultd/src/cli/memfault_watch/mod.rs @@ -0,0 +1,218 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + fs::File, + io::{stderr, stdout, BufReader, BufWriter}, + path::Path, + process::{Command, ExitStatus, Stdio}, + sync::{atomic::Ordering, Arc, Mutex}, + time::{Duration, Instant}, +}; + +use argh::{FromArgs, TopLevelCommand}; +use buffer::{monitor_and_buffer, STOP_THREADS}; +use chrono::Local; +use eyre::{eyre, Result}; +use flate2::{write::ZlibEncoder, Compression}; +use log::{error, info, trace, LevelFilter}; + +use crate::{ + cli::{init_logger, MemfaultdClient}, + config::Config, + mar::{CompressionAlgorithm, Metadata}, +}; + +use super::memfaultctl::WrappedArgs; + +mod buffer; + +#[derive(FromArgs)] +/// A command line utility to run a specified command and send its output to our backend +struct MemfaultWatchArgs { + /// use configuration file + #[argh(option, short = 'c')] + config_file: Option, + /// verbose output + #[argh(switch, short = 'V')] + verbose: bool, + + /// read in positional command argument + #[argh(positional)] + command: Vec, +} + +pub fn main() -> Result<()> { + let args: MemfaultWatchArgs = from_env(); + let exit_status = run_from_args(args)?; + std::process::exit(exit_status) +} + +fn run_from_args(args: MemfaultWatchArgs) -> Result { + init_logger(if args.verbose { + LevelFilter::Trace + } else { + LevelFilter::Info + }); + + let config_path = args.config_file.as_ref().map(Path::new); + let config = Config::read_from_system(config_path)?; + + // Set up log files + let file_name = format!("mfw-log-{}", Local::now().to_rfc3339()); + let stdio_log_file_name = format!("{file_name}.zlib"); + let stdio_log_file = File::create(&stdio_log_file_name) + .map_err(|_| eyre!("Failed to create output file on filesystem!"))?; + + let (command, additional_args) = args + .command + .split_first() + .ok_or_else(|| eyre!("No command given!"))?; + + trace!("Running command: {:?}", command); + + let start = Instant::now(); + + let mut child = Command::new(command) + .args(additional_args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| eyre!("Failed to run command! Does it exist in path?\nError: {e}"))?; + + let child_stdout = child + .stdout + .take() + .ok_or_else(|| eyre!("Failed to take stdout handle from child process."))?; + let child_stderr = child + .stderr + .take() + .ok_or_else(|| eyre!("Failed to take stderr handle from child process."))?; + + let mut child_stdout_reader = BufReader::new(child_stdout); + let mut child_stderr_reader = BufReader::new(child_stderr); + + let compression_encoder_stdio = ZlibEncoder::new(stdio_log_file, Compression::fast()); + + let stdio_file_writer = Arc::new(Mutex::new(BufWriter::new(compression_encoder_stdio))); + + let child_stdout_writer = stdio_file_writer; + let child_stderr_writer = child_stdout_writer.clone(); + + let child_stdout_monitor = std::thread::spawn(move || { + monitor_and_buffer(&mut child_stdout_reader, &mut stdout(), child_stdout_writer); + }); + + let child_stderr_monitor = std::thread::spawn(move || { + monitor_and_buffer(&mut child_stderr_reader, &mut stderr(), child_stderr_writer); + }); + + let mut get_status = || match child.try_wait() { + Ok(Some(status)) => { + trace!("Command completed with status {status}!"); + Ok(ProcessStatus::Exited(status)) + } + Ok(None) => Ok(ProcessStatus::Running), + Err(e) => { + error!("Failed to check command status! {e}"); + Err(e) + } + }; + let either_thread_still_running = + || !child_stdout_monitor.is_finished() || !child_stderr_monitor.is_finished(); + + // Check condition of process + while matches!(get_status(), Ok(ProcessStatus::Running)) { + std::thread::sleep(Duration::from_millis(100)); + } + + // Now that the process is no longer running, send message to threads + STOP_THREADS.store(true, Ordering::Relaxed); + + // While threads still cleaning up + while either_thread_still_running() { + std::thread::sleep(Duration::from_millis(100)); + } + + let child_stdout_monitor = child_stdout_monitor.join(); + let child_stderr_monitor = child_stderr_monitor.join(); + + let status = match get_status() { + Ok(ProcessStatus::Exited(status)) => status, + _ => return Err(eyre!("Failed to retrieve exit status!")), + }; + + let exit_code = status + .code() + .ok_or_else(|| eyre!("Failed to retrieve exit status code!"))?; + + match (&child_stdout_monitor, &child_stderr_monitor) { + (Ok(_), Ok(_)) => { + trace!("Execution completed and monitor threads shut down.") + } + _ => { + error!( + "Error shutting down monitor threads. \n{:?} | {:?}", + &child_stdout_monitor, &child_stderr_monitor + ) + } + } + + let duration = start.elapsed(); + + trace!("Command completed in {} ms", duration.as_millis()); + + let _metadata = Metadata::LinuxMemfaultWatch { + cmdline: args.command, + exit_code, + duration, + stdio_log_file_name, + compression: CompressionAlgorithm::Zlib, + }; + + if !status.success() { + info!("Command failed with exit code {exit_code}!"); + let client = MemfaultdClient::from_config(&config) + .map_err(|report| eyre!("Failed to create Memfaultd client from config! {report}"))?; + + if client.notify_crash().is_err() { + error!("Unable to contact memfaultd. Is it running?"); + } + } + + Ok(exit_code) +} + +/// Utilizes the WrappedArgs which provide version information +pub fn from_env() -> T { + argh::from_env::>().0 +} + +enum ProcessStatus { + Running, + Exited(ExitStatus), +} + +use sealed_test::prelude::*; + +#[sealed_test] +fn test_success_propagates() { + let args: MemfaultWatchArgs = MemfaultWatchArgs { + config_file: None, + verbose: false, + command: vec!["ls".into()], + }; + + assert_eq!(run_from_args(args).unwrap(), 0); +} + +#[sealed_test] +fn test_error_propagates() { + let args: MemfaultWatchArgs = MemfaultWatchArgs { + config_file: None, + verbose: false, + command: vec!["bash".into(), "-c".into(), "exit 7".into()], + }; + + assert_eq!(run_from_args(args).unwrap(), 7); +} diff --git a/memfaultd/src/cli/memfaultctl/add_battery_reading.rs b/memfaultd/src/cli/memfaultctl/add_battery_reading.rs new file mode 100644 index 0000000..24c8d50 --- /dev/null +++ b/memfaultd/src/cli/memfaultctl/add_battery_reading.rs @@ -0,0 +1,19 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::config::Config; +use eyre::{eyre, Result}; + +use crate::cli::memfaultd_client::MemfaultdClient; + +pub fn add_battery_reading(config: &Config, reading_string: &str) -> Result<()> { + let client = MemfaultdClient::from_config(config)?; + + match client.add_battery_reading(reading_string) { + Ok(()) => { + eprintln!("Successfully published battery reading to memfaultd"); + Ok(()) + } + Err(e) => Err(eyre!("add-battery-reading failed: {:#}", e)), + } +} diff --git a/memfaultd/src/cli/memfaultctl/config_file.rs b/memfaultd/src/cli/memfaultctl/config_file.rs new file mode 100644 index 0000000..236858f --- /dev/null +++ b/memfaultd/src/cli/memfaultctl/config_file.rs @@ -0,0 +1,176 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Cli commands for modifying the Memfaultd config file. + +use crate::util::string::capitalize; +use crate::{config::Config, service_manager::MemfaultdServiceManager}; + +use eyre::Result; +use urlencoding::encode; + +/// Set the developer mode flag in the config file and restart memfaultd. +pub fn set_developer_mode( + config: &mut Config, + service_manager: &impl MemfaultdServiceManager, + enable_dev_mode: bool, +) -> Result<()> { + let already_set = check_already_set( + "developer mode", + config.config_file.enable_dev_mode, + enable_dev_mode, + ); + + print_server_side_developer_mode_url(config); + + if already_set { + return Ok(()); + } + + config.config_file.enable_dev_mode = enable_dev_mode; + write_bool_to_config_and_restart_memfaultd( + config, + "enable_dev_mode", + enable_dev_mode, + service_manager, + ) +} + +fn print_server_side_developer_mode_url(config: &Config) { + let device_serial = encode(&config.device_info.device_id); + let project_key = encode(&config.config_file.project_key); + println!( + "⚠️ Enable 'server-side developer mode' to bypass rate limits in Memfault cloud:\n\ + https://mflt.io/developer-mode?d={device_serial}&p={project_key}" + ); +} + +/// Set the data collection flag in the config file and restart memfaultd. +pub fn set_data_collection( + config: &mut Config, + service_manager: &impl MemfaultdServiceManager, + enable_data_collection: bool, +) -> Result<()> { + if check_already_set( + "data collection", + config.config_file.enable_data_collection, + enable_data_collection, + ) { + return Ok(()); + } + + config.config_file.enable_data_collection = enable_data_collection; + write_bool_to_config_and_restart_memfaultd( + config, + "enable_data_collection", + enable_data_collection, + service_manager, + ) +} + +fn check_already_set(module: &str, config_val: bool, new_value: bool) -> bool { + let is_set = config_val == new_value; + if is_set { + let enable_string = if new_value { "enabled" } else { "disabled" }; + println!("{} is already {}", capitalize(module), enable_string); + } else { + let enable_string = if new_value { "Enabling" } else { "Disabling" }; + println!("{} {}", enable_string, module); + } + + is_set +} + +fn write_bool_to_config_and_restart_memfaultd( + config: &Config, + key: &str, + value: bool, + service_manager: &impl MemfaultdServiceManager, +) -> Result<()> { + config + .config_file + .set_and_write_bool_to_runtime_config(key, value)?; + + service_manager.restart_memfaultd_if_running() +} + +#[cfg(test)] +mod test { + use super::*; + + use crate::{service_manager::MockMemfaultdServiceManager, util::path::AbsolutePath}; + + use rstest::rstest; + use tempfile::tempdir; + + struct TestContext { + config: Config, + mock_service_manager: MockMemfaultdServiceManager, + _tmpdir: tempfile::TempDir, + } + + impl TestContext { + fn new() -> Self { + let mut config = Config::test_fixture(); + let mut mock_service_manager = MockMemfaultdServiceManager::new(); + + let tmpdir = tempdir().unwrap(); + config.config_file.persist_dir = + AbsolutePath::try_from(tmpdir.path().to_path_buf()).unwrap(); + + mock_service_manager + .expect_restart_memfaultd_if_running() + .returning(|| Ok(())); + + Self { + config, + mock_service_manager, + _tmpdir: tmpdir, + } + } + } + + #[rstest] + #[case(true, false)] + #[case(false, true)] + #[case(false, false)] + #[case(true, true)] + fn test_set_developer_mode(#[case] enable_developer_mode: bool, #[case] initial_state: bool) { + let mut test_context = TestContext::new(); + test_context.config.config_file.enable_dev_mode = initial_state; + + set_developer_mode( + &mut test_context.config, + &test_context.mock_service_manager, + enable_developer_mode, + ) + .unwrap(); + + assert_eq!( + test_context.config.config_file.enable_dev_mode, + enable_developer_mode, + ); + } + + #[rstest] + #[case(true, false)] + #[case(false, true)] + #[case(false, false)] + #[case(true, true)] + fn test_set_data_collection(#[case] enable_data_collection: bool, #[case] initial_state: bool) { + let mut test_context = TestContext::new(); + test_context.config.config_file.enable_data_collection = initial_state; + + set_data_collection( + &mut test_context.config, + &test_context.mock_service_manager, + enable_data_collection, + ) + .unwrap(); + + assert_eq!( + test_context.config.config_file.enable_data_collection, + enable_data_collection, + ); + } +} diff --git a/memfaultd/src/cli/memfaultctl/coredump.rs b/memfaultd/src/cli/memfaultctl/coredump.rs new file mode 100644 index 0000000..0f5fe20 --- /dev/null +++ b/memfaultd/src/cli/memfaultctl/coredump.rs @@ -0,0 +1,87 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use argh::FromArgValue; +use eyre::Result; + +use crate::config::Config; + +/// Strategy to trigger a crash. +/// +/// This is used to test the crash reporting functionality. +#[derive(Debug)] +pub enum ErrorStrategy { + SegFault, + FPException, +} + +impl FromArgValue for ErrorStrategy { + fn from_arg_value(value: &str) -> Result { + match value { + "segfault" => Ok(ErrorStrategy::SegFault), + "divide-by-zero" => Ok(ErrorStrategy::FPException), + _ => Err("valid strategies are 'segfault' and 'divide-by-zero'".to_string()), + } + } +} + +pub fn trigger_coredump(config: &Config, error_type: ErrorStrategy) -> Result<()> { + trigger_coredump_inner(config, error_type) +} + +#[cfg(feature = "coredump")] +fn trigger_coredump_inner(config: &Config, error_type: ErrorStrategy) -> Result<()> { + use crate::util::ipc::send_flush_signal; + + trigger_crash(error_type)?; + + if config.config_file.enable_dev_mode { + println!("Signaling memfaultd to upload coredump event..."); + + // Give the kernel and memfault-core-handler time to process the coredump + std::thread::sleep(std::time::Duration::from_secs(3)); + + send_flush_signal()?; + } + + Ok(()) +} + +#[cfg(not(feature = "coredump"))] +fn trigger_coredump_inner(_config: &Config, _error_type: ErrorStrategy) -> Result<()> { + println!( + "You must enable the coredump feature when building memfault SDK to report coredumps." + ); + + Ok(()) +} + +#[cfg(feature = "coredump")] +fn trigger_crash(error_type: ErrorStrategy) -> Result<()> { + use memfaultc_sys::coredump::memfault_trigger_fp_exception; + use nix::unistd::{fork, ForkResult}; + + println!("Triggering coredump ..."); + match unsafe { fork() } { + Ok(ForkResult::Parent { .. }) => Ok(()), + Ok(ForkResult::Child) => { + match error_type { + ErrorStrategy::FPException => { + // This still needs to be implemented in C because Rust automatically + // generates code to prevent divide-by-zero errors. This can be moved to Rust when + // [unchecked_div](https://doc.rust-lang.org/std/intrinsics/fn.unchecked_div.html) + // is stabilized. + unsafe { + memfault_trigger_fp_exception(); + } + } + ErrorStrategy::SegFault => { + unsafe { std::ptr::null_mut::().write(42) }; + } + } + + unreachable!("Child process should have crashed"); + } + Err(e) => Err(eyre::eyre!("Failed to fork process: {}", e)), + } +} diff --git a/memfaultd/src/cli/memfaultctl/export.rs b/memfaultd/src/cli/memfaultctl/export.rs new file mode 100644 index 0000000..831eb1a --- /dev/null +++ b/memfaultd/src/cli/memfaultctl/export.rs @@ -0,0 +1,54 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{copy, BufWriter}; + +use crate::config::Config; +use eyre::{eyre, Context, Result}; + +use crate::cli::memfaultctl::ExportArgs; + +use crate::cli::memfaultd_client::{ExportDeleteResponse, ExportGetResponse, MemfaultdClient}; + +pub fn export(config: &Config, args: &ExportArgs) -> Result<()> { + let client = MemfaultdClient::from_config(config)?; + + let delete_token = match client + .export_get(&args.format) + .wrap_err("Unable to fetch latest export")? + { + ExportGetResponse::NoData => { + eprintln!("Nothing to export right now. You may want to try `memfaultctl sync`."); + return Ok(()); + } + ExportGetResponse::Data { + delete_token, + mut data, + } => { + let mut file = BufWriter::new(args.output.get_output_stream()?); + copy(&mut data, &mut file).wrap_err("Unable to write server response")?; + delete_token + } + }; + + if !args.do_not_delete { + match client + .export_delete(delete_token) + .wrap_err("Error while deleting data")? + { + ExportDeleteResponse::Ok => { + eprintln!("Export saved and data cleared from memfaultd."); + Ok(()) + } + ExportDeleteResponse::ErrorWrongDeleteToken => { + Err(eyre!("Unexpected response: wrong hash")) + } + ExportDeleteResponse::Error404 => { + Err(eyre!("Unexpected response: 404 (no data to delete)")) + } + } + } else { + eprintln!("Export saved. Data preserved in memfaultd."); + Ok(()) + } +} diff --git a/memfaultd/src/cli/memfaultctl/mod.rs b/memfaultd/src/cli/memfaultctl/mod.rs new file mode 100644 index 0000000..0d0be11 --- /dev/null +++ b/memfaultd/src/cli/memfaultctl/mod.rs @@ -0,0 +1,322 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use argh::{FromArgs, TopLevelCommand}; +use std::{path::Path, str::FromStr}; + +mod add_battery_reading; +mod config_file; +mod coredump; +mod export; +mod report_sync; +mod session; +mod sync; +mod write_attributes; + +use crate::{ + cli::version::format_version, + mar::{DeviceAttribute, ExportFormat, Metadata}, + metrics::{KeyedMetricReading, SessionName}, + reboot::{write_reboot_reason_and_reboot, RebootReason}, + service_manager::get_service_manager, +}; +use crate::{mar::MarEntryBuilder, util::output_arg::OutputArg}; + +use crate::cli::init_logger; +use crate::cli::memfaultctl::add_battery_reading::add_battery_reading; +use crate::cli::memfaultctl::config_file::{set_data_collection, set_developer_mode}; +use crate::cli::memfaultctl::coredump::{trigger_coredump, ErrorStrategy}; +use crate::cli::memfaultctl::export::export; +use crate::cli::memfaultctl::report_sync::report_sync; +use crate::cli::memfaultctl::sync::sync; +use crate::cli::show_settings::show_settings; +use crate::config::Config; +use crate::network::NetworkConfig; +use eyre::{eyre, Context, Result}; +use log::LevelFilter; + +use self::session::{end_session, start_session}; + +#[derive(FromArgs)] +/// A command line utility to adjust memfaultd configuration and trigger specific events for +/// testing purposes. For further reference, see: +/// https://docs.memfault.com/docs/linux/reference-memfaultctl-cli +struct MemfaultctlArgs { + #[argh(subcommand)] + command: MemfaultctlCommand, + + /// use configuration file + #[argh(option, short = 'c')] + config_file: Option, + + /// show version information + #[argh(switch, short = 'v')] + #[allow(dead_code)] + version: bool, + + /// verbose output + #[argh(switch, short = 'V')] + verbose: bool, +} + +/// Wrapper around argh to support flags acting as subcommands, like --version. +/// Inspired by https://gist.github.com/suluke/e0c672492126be0a4f3b4f0e1115d77c +pub struct WrappedArgs(pub T); +impl TopLevelCommand for WrappedArgs {} +impl FromArgs for WrappedArgs { + fn from_args(command_name: &[&str], args: &[&str]) -> Result { + /// Pseudo subcommands that look like flags. + #[derive(FromArgs)] + struct CommandlikeFlags { + /// show version information + #[argh(switch, short = 'v')] + version: bool, + } + + match CommandlikeFlags::from_args(command_name, args) { + Ok(CommandlikeFlags { version: true }) => Err(argh::EarlyExit { + output: format_version(), + status: Ok(()), + }), + _ => T::from_args(command_name, args).map(Self), + } + } +} + +pub fn from_env() -> T { + argh::from_env::>().0 +} + +#[derive(FromArgs)] +#[argh(subcommand)] +enum MemfaultctlCommand { + EnableDataCollection(EnableDataCollectionArgs), + DisableDataCollection(DisableDataCollectionArgs), + EnableDevMode(EnableDevModeArgs), + DisableDevMode(DisableDevModeArgs), + Export(ExportArgs), + Reboot(RebootArgs), + RequestMetrics(RequestMetricsArgs), + ShowSettings(ShowSettingsArgs), + Synchronize(SyncArgs), + TriggerCoredump(TriggerCoredumpArgs), + WriteAttributes(WriteAttributesArgs), + AddBatteryReading(AddBatteryReadingArgs), + ReportSyncSuccess(ReportSyncSuccessArgs), + ReportSyncFailure(ReportSyncFailureArgs), + StartSession(StartSessionArgs), + EndSession(EndSessionArgs), +} + +#[derive(FromArgs)] +/// enable data collection and restart memfaultd +#[argh(subcommand, name = "enable-data-collection")] +struct EnableDataCollectionArgs {} + +#[derive(FromArgs)] +/// disable data collection and restart memfaultd +#[argh(subcommand, name = "disable-data-collection")] +struct DisableDataCollectionArgs {} + +#[derive(FromArgs)] +/// enable developer mode and restart memfaultd +#[argh(subcommand, name = "enable-dev-mode")] +struct EnableDevModeArgs {} + +#[derive(FromArgs)] +/// disable developer mode and restart memfaultd +#[argh(subcommand, name = "disable-dev-mode")] +struct DisableDevModeArgs {} + +#[derive(FromArgs)] +/// export (and delete) memfault data +#[argh(subcommand, name = "export")] +pub struct ExportArgs { + #[argh(switch, short = 'n')] + /// do not delete the data from memfault mar_staging + do_not_delete: bool, + #[argh(option, short = 'o')] + /// where to write the MAR data (or '-' for standard output) + output: OutputArg, + + #[argh(option, short = 'f', default = "ExportFormat::Mar")] + /// output format (mar, chunk or chunk-wrapped) + format: ExportFormat, +} + +#[derive(FromArgs)] +/// register reboot reason and call 'reboot' +#[argh(subcommand, name = "reboot")] +struct RebootArgs { + /// a reboot reason ID from https://docs.memfault.com/docs/platform/reference-reboot-reason-ids + #[argh(option)] + reason: String, +} + +#[derive(FromArgs)] +/// flush collectd metrics to Memfault now +#[argh(subcommand, name = "request-metrics")] +struct RequestMetricsArgs {} + +#[derive(FromArgs)] +/// show memfaultd settings +#[argh(subcommand, name = "show-settings")] +struct ShowSettingsArgs {} + +#[derive(FromArgs)] +/// Upload all pending data to Memfault now +#[argh(subcommand, name = "sync")] +struct SyncArgs {} + +#[derive(FromArgs)] +/// trigger a coredump and immediately reports it to Memfault (defaults to segfault) +#[argh(subcommand, name = "trigger-coredump")] +struct TriggerCoredumpArgs { + /// a strategy, either 'segfault' or 'divide-by-zero' + #[argh(positional, default = "ErrorStrategy::SegFault")] + strategy: ErrorStrategy, +} + +#[derive(FromArgs)] +/// write device attribute(s) to memfaultd +#[argh(subcommand, name = "write-attributes")] +struct WriteAttributesArgs { + /// attributes to write, in the format + #[argh(positional)] + attributes: Vec, +} + +#[derive(FromArgs)] +/// add a reading to memfaultd's battery metrics in format "[status string]:[0.0-100.0]". +#[argh(subcommand, name = "add-battery-reading")] +struct AddBatteryReadingArgs { + // Valid status strings are "Charging", "Not charging", "Discharging", "Unknown", and "Full" + // These are based off the values that can appear in /sys/class/power_supply//status + // See: https://www.kernel.org/doc/Documentation/ABI/testing/sysfs-class-power + #[argh(positional)] + reading_string: String, +} + +#[derive(FromArgs)] +/// Report a successful sync for connectivity metrics +#[argh(subcommand, name = "report-sync-success")] +struct ReportSyncSuccessArgs {} + +#[derive(FromArgs)] +/// Report a failed sync for connectivity metrics +#[argh(subcommand, name = "report-sync-failure")] +struct ReportSyncFailureArgs {} + +#[derive(FromArgs)] +/// Begin a session and start capturing metrics for it +#[argh(subcommand, name = "start-session")] +struct StartSessionArgs { + // session name (needs to be defined in memfaultd.conf) + #[argh(positional)] + session_name: SessionName, + // List of metric key value pairs to write in the format + #[argh(positional)] + readings: Vec, +} + +#[derive(FromArgs)] +/// End a session and dump its metrics to MAR staging directory +#[argh(subcommand, name = "end-session")] +struct EndSessionArgs { + // session name (needs to be defined in memfaultd.conf) + #[argh(positional)] + session_name: SessionName, + // List of metric key value pairs to write in the format + #[argh(positional)] + readings: Vec, +} + +fn check_data_collection_enabled(config: &Config, do_what: &str) -> Result<()> { + match config.config_file.enable_data_collection { + true => Ok(()), + false => { + let msg = format!( + "Cannot {} because data collection is disabled. \ + Hint: enable it with 'memfaultctl enable-data-collection'.", + do_what + ); + Err(eyre!(msg)) + } + } +} + +pub fn main() -> Result<()> { + let args: MemfaultctlArgs = from_env(); + + init_logger(if args.verbose { + LevelFilter::Trace + } else { + LevelFilter::Info + }); + + let config_path = args.config_file.as_ref().map(Path::new); + let mut config = Config::read_from_system(config_path)?; + let network_config = NetworkConfig::from(&config); + let mar_staging_path = config.mar_staging_path(); + + let service_manager = get_service_manager(); + + match args.command { + MemfaultctlCommand::EnableDataCollection(_) => { + set_data_collection(&mut config, &service_manager, true) + } + MemfaultctlCommand::DisableDataCollection(_) => { + set_data_collection(&mut config, &service_manager, false) + } + MemfaultctlCommand::EnableDevMode(_) => { + set_developer_mode(&mut config, &service_manager, true) + } + MemfaultctlCommand::DisableDevMode(_) => { + set_developer_mode(&mut config, &service_manager, false) + } + MemfaultctlCommand::Export(args) => export(&config, &args).wrap_err("Error exporting data"), + MemfaultctlCommand::Reboot(args) => { + let reason = RebootReason::from_str(&args.reason) + .wrap_err(eyre!("Failed to parse {}", args.reason))?; + println!("Rebooting with reason {:?}", reason); + write_reboot_reason_and_reboot( + &config.config_file.reboot.last_reboot_reason_file, + reason, + ) + } + MemfaultctlCommand::RequestMetrics(_) => sync(), + MemfaultctlCommand::ShowSettings(_) => show_settings(config_path), + MemfaultctlCommand::Synchronize(_) => sync(), + MemfaultctlCommand::TriggerCoredump(TriggerCoredumpArgs { strategy }) => { + trigger_coredump(&config, strategy) + } + MemfaultctlCommand::WriteAttributes(WriteAttributesArgs { attributes }) => { + // argh does not have a way to specify the minimum number of repeating arguments, so check here: + // https://github.com/google/argh/issues/110 + if attributes.is_empty() { + Err(eyre!( + "No attributes given. Please specify them as KEY=VALUE pairs." + )) + } else { + check_data_collection_enabled(&config, "write attributes")?; + MarEntryBuilder::new(&mar_staging_path)? + .set_metadata(Metadata::new_device_attributes(attributes)) + .save(&network_config) + .map(|_entry| ()) + } + } + MemfaultctlCommand::AddBatteryReading(AddBatteryReadingArgs { reading_string }) => { + add_battery_reading(&config, &reading_string) + } + MemfaultctlCommand::ReportSyncSuccess(_) => report_sync(&config, true), + MemfaultctlCommand::ReportSyncFailure(_) => report_sync(&config, false), + MemfaultctlCommand::StartSession(StartSessionArgs { + session_name, + readings, + }) => start_session(&config, session_name, readings), + MemfaultctlCommand::EndSession(EndSessionArgs { + session_name, + readings, + }) => end_session(&config, session_name, readings), + } +} diff --git a/memfaultd/src/cli/memfaultctl/report_sync.rs b/memfaultd/src/cli/memfaultctl/report_sync.rs new file mode 100644 index 0000000..199653f --- /dev/null +++ b/memfaultd/src/cli/memfaultctl/report_sync.rs @@ -0,0 +1,25 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::config::Config; +use eyre::{eyre, Result}; + +use crate::cli::memfaultd_client::MemfaultdClient; + +pub fn report_sync(config: &Config, success: bool) -> Result<()> { + let client = MemfaultdClient::from_config(config)?; + let status = if success { "successful" } else { "failed" }; + let command_string = if success { + "report-sync-success " + } else { + "report-sync-failure" + }; + + match client.report_sync(success) { + Ok(()) => { + eprintln!("Reported a {} sync to memfaultd", status); + Ok(()) + } + Err(e) => Err(eyre!("{} failed: {:#}", command_string, e)), + } +} diff --git a/memfaultd/src/cli/memfaultctl/session.rs b/memfaultd/src/cli/memfaultctl/session.rs new file mode 100644 index 0000000..96c2f40 --- /dev/null +++ b/memfaultd/src/cli/memfaultctl/session.rs @@ -0,0 +1,54 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::{eyre, Result}; + +use crate::config::Config; +use crate::{ + cli::memfaultd_client::MemfaultdClient, + metrics::{KeyedMetricReading, SessionName}, +}; + +pub fn start_session( + config: &Config, + session_name: SessionName, + metric_readings: Vec, +) -> Result<()> { + let client = MemfaultdClient::from_config(config)?; + if config.config_file.enable_data_collection { + match client.start_session(session_name.clone(), metric_readings) { + Ok(()) => { + eprintln!("Started new {} session", session_name); + Ok(()) + } + Err(e) => Err(eyre!("start-session failed: {:?}", e)), + } + } else { + Err(eyre!( + "Cannot start session with data collection disabled.\n\ + You can enable data collection with \"memfaultctl enable-data-collection\"" + )) + } +} + +pub fn end_session( + config: &Config, + session_name: SessionName, + readings: Vec, +) -> Result<()> { + let client = MemfaultdClient::from_config(config)?; + if config.config_file.enable_data_collection { + match client.end_session(session_name.clone(), readings) { + Ok(()) => { + eprintln!("Ended ongoing {} session", session_name); + Ok(()) + } + Err(e) => Err(eyre!("end-session failed: {:?}", e)), + } + } else { + Err(eyre!( + "Cannot end session with data collection disabled.\n\ + You can enable data collection with \"memfaultctl enable-data-collection\"" + )) + } +} diff --git a/memfaultd/src/cli/memfaultctl/sync.rs b/memfaultd/src/cli/memfaultctl/sync.rs new file mode 100644 index 0000000..87bb40d --- /dev/null +++ b/memfaultd/src/cli/memfaultctl/sync.rs @@ -0,0 +1,18 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::{eyre, Result}; + +use crate::util::ipc::send_flush_signal; + +pub fn sync() -> Result<()> { + match send_flush_signal() { + Ok(()) => Ok(()), + Err(e) => Err(eyre!( + "Error: {} If you are not running memfaultd as a daemon you \ + can force it to sync data with \ + 'killall -USR1 memfaultd'.", + e + )), + } +} diff --git a/memfaultd/src/cli/memfaultctl/write_attributes.rs b/memfaultd/src/cli/memfaultctl/write_attributes.rs new file mode 100644 index 0000000..b213c21 --- /dev/null +++ b/memfaultd/src/cli/memfaultctl/write_attributes.rs @@ -0,0 +1,82 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::mar::DeviceAttribute; +use crate::metrics::MetricStringKey; +use argh::FromArgValue; +use serde_json::Value; +use std::str::FromStr; + +impl FromArgValue for DeviceAttribute { + fn from_arg_value(value: &str) -> Result { + let (key, value_str) = value + .split_once('=') + .ok_or("Each attribute should be specified as KEY=VALUE")?; + + // Let's ensure the key is valid first: + let metric_key = MetricStringKey::from_str(key)?; + let value = match serde_json::from_str::(value_str) { + Ok(value) => { + if value.is_array() || value.is_object() { + return Err("Invalid value: arrays or objects are not allowed".to_string()); + } + value + } + // Default to string value: + Err(_) => value_str.into(), + }; + Ok(DeviceAttribute::new(metric_key, value)) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case("")] + #[case("KEY")] + #[case("KEY:VALUE")] + fn split_failed(#[case] input: &str) { + assert!(DeviceAttribute::from_arg_value(input) + .err() + .unwrap() + .contains("Each attribute should be specified as KEY=VALUE")); + } + + #[rstest] + fn invalid_key() { + assert_eq!( + DeviceAttribute::from_arg_value("\u{1F4A9}=smelly") + .err() + .unwrap(), + "Invalid key: must be ASCII" + ); + } + + #[rstest] + #[case("key=[]")] + #[case("key={}")] + fn invalid_value(#[case] input: &str) { + assert_eq!( + DeviceAttribute::from_arg_value(input).err().unwrap(), + "Invalid value: arrays or objects are not allowed" + ); + } + + #[rstest] + #[case("key=", ("key", "").try_into())] + #[case("key=my_string", ("key", "my_string").try_into())] + #[case("key=123", ("key", 123).try_into())] + #[case("key=123.456", ("key", 123.456).try_into())] + #[case("key=true", ("key", true).try_into())] + #[case("key=false", ("key", false).try_into())] + #[case("key=\"false\"", ("key", "false").try_into())] + #[case("key=\"[]\"", ("key", "[]").try_into())] + #[case("key=\"{}\"", ("key", "{}").try_into())] + fn parsed_ok(#[case] input: &str, #[case] expected: Result) { + assert_eq!(DeviceAttribute::from_arg_value(input), expected); + } +} diff --git a/memfaultd/src/cli/memfaultd.rs b/memfaultd/src/cli/memfaultd.rs new file mode 100644 index 0000000..f73db2c --- /dev/null +++ b/memfaultd/src/cli/memfaultd.rs @@ -0,0 +1,154 @@ +#![allow(clippy::print_stdout, clippy::print_stderr)] +// +// Copyright (c) Memfault, Inc. +// See License.txt for details + +use std::{ + env::args_os, fs::create_dir_all, os::unix::process::CommandExt, path::Path, process::Command, +}; + +use crate::{ + config::Config, + memfaultd::{memfaultd_loop, MemfaultLoopResult}, + util::pid_file::{get_pid_from_file, is_pid_file_about_me, remove_pid_file, write_pid_file}, +}; + +use eyre::{eyre, Context, Result}; +use log::{error, info, warn, LevelFilter}; + +use crate::cli::show_settings::show_settings; +use crate::cli::version::format_version; +use argh::FromArgs; + +use super::init_logger; + +#[derive(FromArgs)] +/// Memfault daemon. +struct MemfaultDArgs { + /// use configuration file + #[argh(option, short = 'c')] + config_file: Option, + + #[argh(switch, short = 's')] + /// show settings and exit immediately + show_settings: bool, + + #[argh(switch, short = 'Z')] + /// daemonize (fork to background) + daemonize: bool, + + #[argh(switch, short = 'v')] + /// show version + version: bool, + + #[argh(switch, short = 'V')] + /// verbose output + verbose: bool, + + #[argh(switch, short = 'q')] + /// quiet - no output + quiet: bool, +} + +pub fn main() -> Result<()> { + let args: MemfaultDArgs = argh::from_env(); + let config_path = args.config_file.as_ref().map(Path::new); + + init_logger(match (args.quiet, args.verbose) { + (true, _) => LevelFilter::Off, + (false, true) => LevelFilter::Trace, + _ => LevelFilter::Info, + }); + + if args.version { + println!("{}", format_version()); + return Ok(()); + } + + let config = + Config::read_from_system(config_path).wrap_err(eyre!("Unable to load configuration"))?; + + // Create directories early so we can fail early if we can't create them. + mkdir_if_needed(&config.config_file.persist_dir)?; + mkdir_if_needed(&config.tmp_dir())?; + + // Always show settings when starting + show_settings(config_path)?; + + if args.show_settings { + // Already printed above. We're done. + return Ok(()); + } + + if !args.daemonize && get_pid_from_file().is_ok() { + return Err(eyre!("memfaultd is already running")); + } + + if config.config_file.enable_dev_mode { + info!("memfaultd:: Starting with developer mode enabled"); + } + if !config.config_file.enable_data_collection { + warn!("memfaultd:: Starting with data collection DISABLED"); + } + + #[cfg(feature = "swupdate")] + { + use crate::swupdate::generate_swupdate_config; + + generate_swupdate_config(&config)?; + } + #[cfg(feature = "coredump")] + { + use crate::coredump::coredump_configure_kernel; + if config.config_file.enable_data_collection { + coredump_configure_kernel(&config.config_file_path)?; + } + } + + // Only daemonize when asked to AND not already running (aka don't fork when reloading) + let need_daemonize = args.daemonize && !is_pid_file_about_me(); + if need_daemonize { + daemonize()?; + } + + let result = memfaultd_loop(config, || { + if need_daemonize { + // All subcomponents are ready, write the pid file now to indicate we've started up completely. + write_pid_file()?; + } + Ok(()) + })?; + if result == MemfaultLoopResult::Relaunch { + // If reloading the config, execv ourselves (replace our program in memory by a new copy of us) + let mut args = args_os().collect::>(); + let arg0 = args.remove(0); + + let err = Command::new(arg0).args(&args).exec(); + // This next line will only be executed if we failed to exec(). + error!("Unable to restart {:?}: {:?}", args, err); + }; + + if args.daemonize { + remove_pid_file()? + } + Ok(()) +} + +fn mkdir_if_needed(path: &Path) -> Result<()> { + if path.exists() && path.is_dir() { + return Ok(()); + } + create_dir_all(path).wrap_err(eyre!("Unable to create directory {}", path.display())) +} + +fn daemonize() -> Result<()> { + #[cfg(target_os = "linux")] + { + nix::unistd::daemon(true, true).wrap_err("Unable to daemonize") + } + #[cfg(not(target_os = "linux"))] + { + warn!("Daemonizing is not supported on this platform"); + Ok(()) + } +} diff --git a/memfaultd/src/cli/memfaultd_client.rs b/memfaultd/src/cli/memfaultd_client.rs new file mode 100644 index 0000000..3afc538 --- /dev/null +++ b/memfaultd/src/cli/memfaultd_client.rs @@ -0,0 +1,206 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{io::Read, str::from_utf8, time::Duration}; + +use eyre::{eyre, Context, Result}; +use reqwest::{blocking::Client, header::ACCEPT, StatusCode}; + +use crate::{ + config::Config, + http_server::SessionRequest, + mar::{ExportFormat, EXPORT_MAR_URL}, + metrics::{KeyedMetricReading, SessionName}, +}; + +/// Client to Memfaultd localhost HTTP API +pub struct MemfaultdClient { + base_url: String, + client: Client, +} + +pub struct DeleteToken(String); + +pub enum ExportGetResponse { + Data { + delete_token: DeleteToken, + data: Box, + }, + NoData, +} + +pub enum ExportDeleteResponse { + Ok, + ErrorWrongDeleteToken, + Error404, +} + +impl MemfaultdClient { + pub fn from_config(config: &Config) -> Result { + Ok(MemfaultdClient { + client: Client::builder().timeout(Duration::from_secs(10)).build()?, + base_url: format!("http://{}", config.config_file.http_server.bind_address), + }) + } + + pub fn export_get(&self, format: &ExportFormat) -> Result { + let r = self + .client + .get(format!("{}{}", self.base_url, EXPORT_MAR_URL)) + .header(ACCEPT, format.to_content_type()) + .send() + .wrap_err_with(|| { + eyre!(format!( + "Error fetching {}/{}", + self.base_url, EXPORT_MAR_URL + )) + })?; + match r.status() { + StatusCode::OK => Ok(ExportGetResponse::Data { + delete_token: DeleteToken( + r.headers() + .iter() + .find(|h| h.0.as_str() == "etag") + .ok_or(eyre!("No ETag header included on the response")) + .map(|etag| etag.1.to_str())?? + .trim_matches('"') + .to_owned(), + ), + data: Box::new(r), + }), + StatusCode::NO_CONTENT => Ok(ExportGetResponse::NoData), + StatusCode::NOT_ACCEPTABLE => Err(eyre!("Requested format not supported")), + _ => Err(eyre!("Unexpected status code {}", r.status().as_u16())), + } + } + + pub fn export_delete(&self, delete_token: DeleteToken) -> Result { + let r = self + .client + .delete(format!("{}{}", self.base_url, EXPORT_MAR_URL)) + .header("If-Match", delete_token.0) + .send()?; + match r.status() { + StatusCode::NO_CONTENT => Ok(ExportDeleteResponse::Ok), + StatusCode::PRECONDITION_FAILED => Ok(ExportDeleteResponse::ErrorWrongDeleteToken), + StatusCode::NOT_FOUND => Ok(ExportDeleteResponse::Error404), + _ => Err(eyre!(format!( + "Unexpected status code {}", + r.status().as_u16() + ))), + } + } + + pub fn add_battery_reading(&self, battery_reading_string: &str) -> Result<()> { + let r = self + .client + .post(format!("{}{}", self.base_url, "/v1/battery/add_reading")) + .body(battery_reading_string.to_string()) + .send()?; + match r.status() { + StatusCode::OK => Ok(()), + _ => Err(eyre!( + "Unexpected status code {}: {}", + r.status().as_u16(), + from_utf8(&r.bytes()?)? + )), + } + } + + pub fn notify_crash(&self) -> Result<()> { + self.client + .post(format!("{}{}", self.base_url, "/v1/crash/report")) + .send()?; + Ok(()) + } + + pub fn report_sync(&self, success: bool) -> Result<()> { + let path = if success { + "/v1/sync/success" + } else { + "/v1/sync/failure" + }; + let r = self + .client + .post(format!("{}{}", self.base_url, path)) + .send()?; + match r.status() { + StatusCode::OK => Ok(()), + _ => Err(eyre!( + "Unexpected status code {}: {}", + r.status().as_u16(), + from_utf8(&r.bytes()?)? + )), + } + } + + pub fn start_session( + &self, + session_name: SessionName, + readings: Vec, + ) -> Result<()> { + let body = if readings.is_empty() { + session_name.to_string() + } else { + serde_json::to_string(&SessionRequest::new(session_name, readings))? + }; + let r = self + .client + .post(format!("{}/v1/session/start", self.base_url)) + .body(body) + .send()?; + match r.status() { + StatusCode::OK => Ok(()), + _ => Err(eyre!( + "Unexpected status code {}: {}", + r.status().as_u16(), + from_utf8(&r.bytes()?)? + )), + } + } + + pub fn end_session( + &self, + session_name: SessionName, + readings: Vec, + ) -> Result<()> { + let body = if readings.is_empty() { + session_name.to_string() + } else { + serde_json::to_string(&SessionRequest::new(session_name, readings))? + }; + let r = self + .client + .post(format!("{}/v1/session/end", self.base_url)) + .body(body) + .send()?; + match r.status() { + StatusCode::OK => Ok(()), + _ => Err(eyre!( + "Unexpected status code {}: {}", + r.status().as_u16(), + from_utf8(&r.bytes()?)? + )), + } + } + + #[cfg(feature = "logging")] + pub fn get_crash_logs(&self) -> Result>> { + use crate::logs::log_collector::{CrashLogs, CRASH_LOGS_URL}; + + let r = self + .client + .get(format!("{}{}", self.base_url, CRASH_LOGS_URL)) + .send()?; + + match r.status() { + StatusCode::OK => Ok(Some(r.json::()?.logs)), + _ => Err(eyre!("Unexpected status code {}", r.status().as_u16())), + } + } + + #[cfg(not(feature = "logging"))] + pub fn get_crash_logs(&self) -> Result>> { + Ok(None) + } +} diff --git a/memfaultd/src/cli/mod.rs b/memfaultd/src/cli/mod.rs new file mode 100644 index 0000000..83c6146 --- /dev/null +++ b/memfaultd/src/cli/mod.rs @@ -0,0 +1,70 @@ +#![allow(clippy::print_stdout, clippy::print_stderr)] +// +// Copyright (c) Memfault, Inc. +// See License.txt for details + +use eyre::eyre; +use log::LevelFilter; +use std::path::Path; +use stderrlog::{LogLevelNum, StdErrLog}; + +#[cfg(all(target_os = "linux", feature = "coredump"))] +mod memfault_core_handler; +#[cfg(feature = "mfw")] +mod memfault_watch; +mod memfaultctl; +mod memfaultd; +mod memfaultd_client; +mod show_settings; +mod version; + +pub use memfaultd_client::*; + +fn build_logger(level: LevelFilter) -> StdErrLog { + let mut log = stderrlog::new(); + + log.module("memfaultd"); + log.verbosity(LogLevelNum::from(level)); + + log +} + +fn init_logger(level: LevelFilter) { + build_logger(level).init().unwrap(); +} + +pub fn main() { + let arg0 = std::env::args().next().unwrap(); + let cmd_name = Path::new(&arg0) + .file_name() + .expect("") + .to_str() + .unwrap(); + + let result = match cmd_name { + #[cfg(all(target_os = "linux", feature = "coredump"))] + "memfault-core-handler" => memfault_core_handler::main(), + #[cfg(not(all(target_os = "linux", feature = "coredump")))] + "memfault-core-handler" => Err(eyre!( + "memfault-core-handler is not supported in this build" + )), + "memfaultctl" => memfaultctl::main(), + "memfaultd" => memfaultd::main(), + #[cfg(feature = "mfw")] + "mfw" => memfault_watch::main(), + #[cfg(not(feature = "mfw"))] + "mfw" => Err(eyre!("Memfault-watch is currently experimental. You must compile with the experimental flag enabled.")), + _ => Err(eyre!( + "Unknown command: {}. Should be memfaultd/memfaultctl/memfault-core-handler.", + cmd_name + )), + }; + + match result { + Ok(_) => (), + Err(e) => { + eprintln!("{:#}", e); + std::process::exit(-1); + } + } +} diff --git a/memfaultd/src/cli/show_settings.rs b/memfaultd/src/cli/show_settings.rs new file mode 100644 index 0000000..0919b0c --- /dev/null +++ b/memfaultd/src/cli/show_settings.rs @@ -0,0 +1,192 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::borrow::Cow; +use std::io::{stdout, Write}; +use std::path::Path; + +use eyre::Result; + +use crate::config::{DeviceInfo, DeviceInfoWarning, JsonConfigs, MemfaultdConfig}; +use crate::{ + build_info::{BUILD_ID, GIT_COMMIT, VERSION}, + config::Config, +}; + +fn dump_config( + writer: &mut impl Write, + configs: &JsonConfigs, + config_path: Option<&Path>, +) -> Result<()> { + let path_str = config_path + .map(Path::display) + .map(|d| Cow::Owned(d.to_string())) + .unwrap_or_else(|| Cow::Borrowed(Config::DEFAULT_CONFIG_PATH)); + writeln!(writer, "Base configuration ({}):", path_str)?; + writeln!(writer, "{}", serde_json::to_string_pretty(&configs.base)?)?; + writeln!(writer)?; + writeln!(writer, "Runtime configuration:")?; + writeln!( + writer, + "{}", + serde_json::to_string_pretty(&configs.runtime)? + )?; + Ok(()) +} + +type Device = (DeviceInfo, Vec); + +fn dump_device_info(writer: &mut impl Write, device: &Device) -> Result<()> { + let (device_info, _warnings) = device; + writeln!(writer, "Device configuration from memfault-device-info:")?; + writeln!(writer, " MEMFAULT_DEVICE_ID={}", device_info.device_id)?; + writeln!( + writer, + " MEMFAULT_HARDWARE_VERSION={}", + device_info.hardware_version + )?; + if let Some(sw_version) = device_info.software_version.as_ref() { + writeln!(writer, " MEMFAULT_SOFTWARE_VERSION={}", sw_version)?; + } + if let Some(sw_type) = device_info.software_type.as_ref() { + writeln!(writer, " MEMFAULT_SOFTWARE_TYPE={}", sw_type)?; + } + + Ok(()) +} + +struct Versions { + version: &'static str, + git_commit: &'static str, + build_id: &'static str, +} + +fn dump_version(writer: &mut impl Write, versions: &Versions) -> Result<()> { + writeln!(writer, "Memfault version:")?; + writeln!(writer, " VERSION={}", versions.version)?; + writeln!(writer, " GIT COMMIT={}", versions.git_commit)?; + writeln!(writer, " BUILD ID={}", versions.build_id)?; + Ok(()) +} + +fn dump_features(writer: &mut impl Write, features: &[&str]) -> Result<()> { + writeln!(writer, "Features enabled:")?; + for feature in features { + writeln!(writer, " {}", feature)?; + } + Ok(()) +} + +fn dump_settings( + writer: &mut impl Write, + configs: &JsonConfigs, + config_path: Option<&Path>, + device: &Device, + versions: &Versions, + features: &[&str], +) -> Result<()> { + dump_config(writer, configs, config_path)?; + writeln!(writer)?; + dump_device_info(writer, device)?; + writeln!(writer)?; + dump_version(writer, versions)?; + writeln!(writer)?; + dump_features(writer, features)?; + writeln!(writer)?; + Ok(()) +} + +pub fn show_settings(config_path: Option<&Path>) -> Result<()> { + let configs = MemfaultdConfig::parse_configs( + config_path.unwrap_or_else(|| Path::new(Config::DEFAULT_CONFIG_PATH)), + )?; + let versions = Versions { + version: VERSION, + git_commit: GIT_COMMIT, + build_id: BUILD_ID, + }; + + let enabled_features = [ + "reboot", + #[cfg(feature = "swupdate")] + "swupdate", + #[cfg(feature = "collectd")] + "collectd", + #[cfg(feature = "coredump")] + "coredump", + #[cfg(feature = "logging")] + "logging", + #[cfg(feature = "log-to-metrics")] + "log-to-metrics", + #[cfg(feature = "systemd")] + "systemd", + ]; + + dump_settings( + &mut stdout(), + &configs, + config_path, + &DeviceInfo::load()?, + &versions, + &enabled_features, + ) +} + +#[cfg(test)] +mod tests { + use std::io::Cursor; + use std::path::PathBuf; + + use insta::assert_snapshot; + use serde_json::json; + + use super::*; + use crate::config::MockDeviceInfoDefaults; + + #[test] + fn test() { + let configs = JsonConfigs { + base: json!({"project_key": "xyz"}), + runtime: json!({"enable_data_collection": true}), + }; + let config_path = PathBuf::from("/etc/memfaultd.conf"); + + let mut di_defaults = MockDeviceInfoDefaults::new(); + di_defaults.expect_software_type().returning(|| Ok(None)); + di_defaults.expect_software_version().returning(|| Ok(None)); + di_defaults + .expect_hardware_version() + .returning(|| Ok("Hardware".into())); + di_defaults + .expect_device_id() + .returning(|| Ok("123ABC".into())); + let device = DeviceInfo::parse( + Some(b"MEMFAULT_DEVICE_ID=X\nMEMFAULT_HARDWARE_VERSION=Y\nblahblahblah\n"), + di_defaults, + ) + .unwrap(); + + let versions = Versions { + version: "1.2.3", + git_commit: "abcdef", + build_id: "123456", + }; + + let enabled_features = ["reboot", "coredump"]; + + let output = Vec::new(); + let mut writer = Cursor::new(output); + dump_settings( + &mut writer, + &configs, + Some(&config_path), + &device, + &versions, + &enabled_features, + ) + .unwrap(); + + let output = String::from_utf8(writer.into_inner()).unwrap(); + assert_snapshot!(output); + } +} diff --git a/memfaultd/src/cli/snapshots/memfaultd__cli__show_settings__tests__test.snap b/memfaultd/src/cli/snapshots/memfaultd__cli__show_settings__tests__test.snap new file mode 100644 index 0000000..cface53 --- /dev/null +++ b/memfaultd/src/cli/snapshots/memfaultd__cli__show_settings__tests__test.snap @@ -0,0 +1,26 @@ +--- +source: memfaultd/src/cli/show_settings.rs +expression: output +--- +Base configuration (/etc/memfaultd.conf): +{ + "project_key": "xyz" +} + +Runtime configuration: +{ + "enable_data_collection": true +} + +Device configuration from memfault-device-info: + MEMFAULT_DEVICE_ID=X + MEMFAULT_HARDWARE_VERSION=Y + +Memfault version: + VERSION=1.2.3 + GIT COMMIT=abcdef + BUILD ID=123456 + +Features enabled: + reboot + coredump diff --git a/memfaultd/src/cli/version.rs b/memfaultd/src/cli/version.rs new file mode 100644 index 0000000..f808448 --- /dev/null +++ b/memfaultd/src/cli/version.rs @@ -0,0 +1,11 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::build_info::{BUILD_ID, GIT_COMMIT, VERSION}; + +pub fn format_version() -> String { + format!( + "VERSION={}\nGIT COMMIT={}\nBUILD ID={}", + VERSION, GIT_COMMIT, BUILD_ID + ) +} diff --git a/memfaultd/src/collectd/collectd_handler.rs b/memfaultd/src/collectd/collectd_handler.rs new file mode 100644 index 0000000..e36a309 --- /dev/null +++ b/memfaultd/src/collectd/collectd_handler.rs @@ -0,0 +1,216 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + io::Read, + sync::{Arc, Mutex}, +}; + +use eyre::{eyre, Result}; +use log::{debug, log_enabled, trace, warn}; +use tiny_http::{Method, Request, Response}; + +use crate::{ + collectd::payload::Payload, + http_server::{HttpHandler, HttpHandlerResult}, + metrics::{KeyedMetricReading, MetricReportManager, BUILTIN_SYSTEM_METRIC_NAMESPACES}, +}; + +/// A server that listens for collectd JSON pushes and stores them in memory. +#[derive(Clone)] +pub struct CollectdHandler { + data_collection_enabled: bool, + builtin_system_metric_collection_enabled: bool, + metrics_store: Arc>, + builtin_namespaces: Vec, +} + +impl CollectdHandler { + pub fn new( + data_collection_enabled: bool, + builtin_system_metric_collection_enabled: bool, + metrics_store: Arc>, + ) -> Self { + CollectdHandler { + data_collection_enabled, + builtin_system_metric_collection_enabled, + metrics_store, + builtin_namespaces: BUILTIN_SYSTEM_METRIC_NAMESPACES + .iter() + .map(|namespace| namespace.to_string() + "/") + .collect(), + } + } + + /// Convert a collectd JSON push (Payload[]) into a list of MetricReading. + fn parse_request(stream: &mut dyn Read) -> Result> { + let payloads: Vec = if log_enabled!(log::Level::Debug) { + let mut buf = vec![]; + stream.read_to_end(&mut buf)?; + let s = String::from_utf8_lossy(&buf); + trace!("Received JSON: {}", s); + match serde_json::from_slice(&buf) { + Ok(payloads) => payloads, + Err(e) => { + debug!("Error parsing JSON: {}\n{}", e, String::from_utf8(buf)?); + return Err(eyre!("Error parsing JSON: {}", e)); + } + } + } else { + serde_json::from_reader(stream)? + }; + Ok(payloads + .into_iter() + .flat_map(Vec::::from) + .collect()) + } +} + +impl HttpHandler for CollectdHandler { + fn handle_request(&self, request: &mut Request) -> HttpHandlerResult { + if request.url() != "/v1/collectd" || *request.method() != Method::Post { + return HttpHandlerResult::NotHandled; + } + if self.data_collection_enabled { + match Self::parse_request(request.as_reader()) { + Ok(readings) => { + let mut metrics_store = self.metrics_store.lock().unwrap(); + for reading in readings { + // If built-in metric collection IS enabled, we need to drop + // collectd metric readings who may have overlapping keys with + // memfaultd's built-in readings. To be safe, any reading whose + // metric key has the same top-level namespace as a built-in system + // metric will be dropped + // + // For example, since CPU metrics can be captured by memfaultd + // this conditional will cause us to drop all collectd + // metric readings whose keys start with "cpu/" when + // built-in system metric collection is enabled + if !self.builtin_system_metric_collection_enabled + || !self + .builtin_namespaces + .iter() + .any(|namespace| reading.name.as_str().starts_with(namespace)) + { + if let Err(e) = metrics_store.add_metric(reading) { + warn!("Invalid metric: {e}"); + } + } + } + } + Err(e) => { + warn!("Error parsing request: {}", e); + } + } + } + HttpHandlerResult::Response(Response::empty(200).boxed()) + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use insta::{assert_json_snapshot, assert_snapshot, with_settings}; + use rstest::{fixture, rstest}; + use tiny_http::{Method, TestRequest}; + + use crate::{ + http_server::{HttpHandler, HttpHandlerResult}, + metrics::MetricReportManager, + }; + + use super::CollectdHandler; + + #[rstest] + fn handle_push(handler: CollectdHandler) { + let r = TestRequest::new().with_method(Method::Post).with_path("/v1/collectd").with_body( + r#"[{"values":[0],"dstypes":["derive"],"dsnames":["value"],"time":1619712000.000,"interval":10.000,"host":"localhost","plugin":"cpu","plugin_instance":"0","type":"cpu","type_instance":"idle"}]"#, + ); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + let metrics = handler + .metrics_store + .lock() + .unwrap() + .take_heartbeat_metrics(); + assert_snapshot!(serde_json::to_string_pretty(&metrics) + .expect("heartbeat_manager should be serializable")); + } + + #[rstest] + fn ignores_data_when_data_collection_is_off() { + let handler = CollectdHandler::new( + false, + false, + Arc::new(Mutex::new(MetricReportManager::new())), + ); + let r = TestRequest::new().with_method(Method::Post).with_path("/v1/collectd").with_body( + r#"[{"values":[0],"dstypes":["derive"],"dsnames":["value"],"time":1619712000.000,"interval":10.000,"host":"localhost","plugin":"cpu","plugin_instance":"0","type":"cpu","type_instance":"idle"}]"#, + ); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + let metrics = handler + .metrics_store + .lock() + .unwrap() + .take_heartbeat_metrics(); + assert_snapshot!(serde_json::to_string_pretty(&metrics) + .expect("heartbeat_manager should be serializable")); + } + + #[rstest] + fn drops_cpu_metrics_when_builtin_system_metrics_are_enabled() { + let handler = + CollectdHandler::new(true, true, Arc::new(Mutex::new(MetricReportManager::new()))); + let r = TestRequest::new().with_method(Method::Post).with_path("/v1/collectd").with_body( + r#"[{"values":[0],"dstypes":["derive"],"dsnames":["value"],"time":1619712000.000,"interval":10.000,"host":"localhost","plugin":"cpu","plugin_instance":"0","type":"cpu","type_instance":"idle"}]"#, + ); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + // cpufreq should NOT be dropped as it's a different top-level namespace from "cpu" + let r = TestRequest::new().with_method(Method::Post).with_path("/v1/collectd").with_body( + r#"[{"values":[0],"dstypes":["derive"],"dsnames":["value"],"time":1619712000.000,"interval":10.000,"host":"localhost","plugin":"cpufreq","plugin_instance":"0","type":"cpu","type_instance":"idle"}]"#, + ); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + let r = TestRequest::new().with_method(Method::Post).with_path("/v1/collectd").with_body( + r#"[{"values":[0],"dstypes":["derive"],"dsnames":["value"],"time":1619712000.000,"interval":10.000,"host":"localhost","plugin":"mockplugin","plugin_instance":"0","type":"mock","type_instance":"test"}]"#, + ); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + let metrics = handler + .metrics_store + .lock() + .unwrap() + .take_heartbeat_metrics(); + + with_settings!({sort_maps => true}, { + assert_json_snapshot!(metrics); + }); + } + + #[fixture] + fn handler() -> CollectdHandler { + CollectdHandler::new( + true, + false, + Arc::new(Mutex::new(MetricReportManager::new())), + ) + } +} diff --git a/memfaultd/src/collectd/fixtures/sample-with-null.json b/memfaultd/src/collectd/fixtures/sample-with-null.json new file mode 100644 index 0000000..e06ed2a --- /dev/null +++ b/memfaultd/src/collectd/fixtures/sample-with-null.json @@ -0,0 +1,158 @@ +[ + { + "values": [973577], + "dstypes": ["gauge"], + "dsnames": ["value"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "uptime", + "plugin_instance": "", + "type": "uptime", + "type_instance": "" + }, + { + "values": [11160961024], + "dstypes": ["gauge"], + "dsnames": ["value"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "memory", + "plugin_instance": "", + "type": "memory", + "type_instance": "inactive" + }, + { + "values": [null, null], + "dstypes": ["derive", "derive"], + "dsnames": ["rx", "tx"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "interface", + "plugin_instance": "lo0", + "type": "if_octets", + "type_instance": "" + }, + { + "values": [null, null], + "dstypes": ["derive", "derive"], + "dsnames": ["rx", "tx"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "interface", + "plugin_instance": "lo0", + "type": "if_packets", + "type_instance": "" + }, + { + "values": [11797676032], + "dstypes": ["gauge"], + "dsnames": ["value"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "memory", + "plugin_instance": "", + "type": "memory", + "type_instance": "active" + }, + { + "values": [null, null], + "dstypes": ["derive", "derive"], + "dsnames": ["rx", "tx"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "interface", + "plugin_instance": "gif0", + "type": "if_packets", + "type_instance": "" + }, + { + "values": [null, null], + "dstypes": ["derive", "derive"], + "dsnames": ["rx", "tx"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "interface", + "plugin_instance": "gif0", + "type": "if_errors", + "type_instance": "" + }, + { + "values": [null, null], + "dstypes": ["derive", "derive"], + "dsnames": ["rx", "tx"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "interface", + "plugin_instance": "stf0", + "type": "if_octets", + "type_instance": "" + }, + { + "values": [3427991552], + "dstypes": ["gauge"], + "dsnames": ["value"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "memory", + "plugin_instance": "", + "type": "memory", + "type_instance": "wired" + }, + { + "values": [null, null], + "dstypes": ["derive", "derive"], + "dsnames": ["rx", "tx"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "interface", + "plugin_instance": "stf0", + "type": "if_errors", + "type_instance": "" + }, + { + "values": [null, null], + "dstypes": ["derive", "derive"], + "dsnames": ["rx", "tx"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "interface", + "plugin_instance": "anpi0", + "type": "if_octets", + "type_instance": "" + }, + { + "values": [null, null], + "dstypes": ["derive", "derive"], + "dsnames": ["rx", "tx"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "interface", + "plugin_instance": "anpi0", + "type": "if_packets", + "type_instance": "" + }, + { + "values": [null, null], + "dstypes": ["derive", "derive"], + "dsnames": ["rx", "tx"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "interface", + "plugin_instance": "anpi0", + "type": "if_errors", + "type_instance": "" + } +] diff --git a/memfaultd/src/collectd/fixtures/sample1.json b/memfaultd/src/collectd/fixtures/sample1.json new file mode 100644 index 0000000..1de8735 --- /dev/null +++ b/memfaultd/src/collectd/fixtures/sample1.json @@ -0,0 +1,146 @@ +[ + { + "dsnames": ["read", "write"], + "dstypes": ["derive", "derive"], + "host": "moon.local", + "interval": 10.0, + "plugin": "disk", + "plugin_instance": "1-0", + "time": 1684949147.351, + "type": "disk_octets", + "type_instance": "", + "values": [1887359.3321528, 1241166.17826548] + }, + { + "dsnames": ["read", "write"], + "dstypes": ["derive", "derive"], + "host": "moon.local", + "interval": 10.0, + "plugin": "disk", + "plugin_instance": "1-20", + "time": 1684949147.351, + "type": "disk_time", + "type_instance": "", + "values": [992441.95537229, 0] + }, + { + "dsnames": ["read", "write"], + "dstypes": ["derive", "derive"], + "host": "moon.local", + "interval": 10.0, + "plugin": "disk", + "plugin_instance": "1-0", + "time": 1684949147.351, + "type": "disk_time", + "type_instance": "", + "values": [17146.2298941808, 6375.40234502902] + }, + { + "dsnames": ["read", "write"], + "dstypes": ["derive", "derive"], + "host": "moon.local", + "interval": 10.0, + "plugin": "disk", + "plugin_instance": "1-0", + "time": 1684949147.351, + "type": "disk_ops", + "type_instance": "", + "values": [75.3295819258201, 101.940031849151] + }, + { + "dsnames": ["value"], + "dstypes": ["gauge"], + "host": "moon.local", + "interval": 10.0, + "plugin": "memory", + "plugin_instance": "", + "time": 1684949147.352, + "type": "memory", + "type_instance": "active", + "values": [8541454336] + }, + { + "dsnames": ["value"], + "dstypes": ["gauge"], + "host": "moon.local", + "interval": 10.0, + "plugin": "memory", + "plugin_instance": "", + "time": 1684949147.352, + "type": "memory", + "type_instance": "inactive", + "values": [8286470144] + }, + { + "dsnames": ["value"], + "dstypes": ["gauge"], + "host": "moon.local", + "interval": 10.0, + "plugin": "memory", + "plugin_instance": "", + "time": 1684949147.352, + "type": "memory", + "type_instance": "free", + "values": [502759424] + }, + { + "dsnames": ["value"], + "dstypes": ["gauge"], + "host": "moon.local", + "interval": 10.0, + "plugin": "statsd", + "plugin_instance": "", + "time": 1684949147.352, + "type": "latency", + "type_instance": "app.kombu_publish_duration,exchange=celeryev,routing_key=worker.heartbeat-average", + "values": [0.00399999972432852] + }, + { + "dsnames": ["value"], + "dstypes": ["gauge"], + "host": "moon.local", + "interval": 10.0, + "plugin": "memory", + "plugin_instance": "", + "time": 1684949147.352, + "type": "memory", + "type_instance": "wired", + "values": [3314139136] + }, + { + "dsnames": ["read", "write"], + "dstypes": ["derive", "derive"], + "host": "moon.local", + "interval": 10.0, + "plugin": "disk", + "plugin_instance": "1-20", + "time": 1684949147.351, + "type": "disk_ops", + "type_instance": "", + "values": [17.3067461702536, 0] + }, + { + "dsnames": ["read", "write"], + "dstypes": ["derive", "derive"], + "host": "moon.local", + "interval": 10.0, + "plugin": "disk", + "plugin_instance": "1-20", + "time": 1684949147.351, + "type": "disk_octets", + "type_instance": "", + "values": [70888.4535866987, 0] + }, + { + "dsnames": ["value"], + "dstypes": ["gauge"], + "host": "moon.local", + "interval": 10.0, + "plugin": "uptime", + "plugin_instance": "", + "time": 1684949147.353, + "type": "uptime", + "type_instance": "", + "values": [790156] + } +] diff --git a/memfaultd/src/collectd/fixtures/statsd-counter-first-seen.json b/memfaultd/src/collectd/fixtures/statsd-counter-first-seen.json new file mode 100644 index 0000000..3263295 --- /dev/null +++ b/memfaultd/src/collectd/fixtures/statsd-counter-first-seen.json @@ -0,0 +1,50 @@ +[ + { + "values": [null], + "dstypes": ["derive"], + "dsnames": ["value"], + "time": 1698360252.757, + "interval": 10.0, + "host": "linux-devbox", + "plugin": "statsd", + "plugin_instance": "", + "type": "derive", + "type_instance": "mypythonapp.counter1" + }, + { + "values": [2], + "dstypes": ["gauge"], + "dsnames": ["value"], + "time": 1698360252.757, + "interval": 10.0, + "host": "linux-devbox", + "plugin": "statsd", + "plugin_instance": "", + "type": "count", + "type_instance": "mypythonapp.counter1" + }, + { + "values": [null], + "dstypes": ["derive"], + "dsnames": ["value"], + "time": 1698360252.757, + "interval": 10.0, + "host": "linux-devbox", + "plugin": "statsd", + "plugin_instance": "", + "type": "derive", + "type_instance": "mypythonapp.counter2" + }, + { + "values": [20], + "dstypes": ["gauge"], + "dsnames": ["value"], + "time": 1698360252.757, + "interval": 10.0, + "host": "linux-devbox", + "plugin": "statsd", + "plugin_instance": "", + "type": "count", + "type_instance": "mypythonapp.counter2" + } +] diff --git a/memfaultd/src/collectd/fixtures/statsd-counter.json b/memfaultd/src/collectd/fixtures/statsd-counter.json new file mode 100644 index 0000000..b25b93d --- /dev/null +++ b/memfaultd/src/collectd/fixtures/statsd-counter.json @@ -0,0 +1,50 @@ +[ + { + "values": [1], + "dstypes": ["gauge"], + "dsnames": ["value"], + "time": 1698360892.757, + "interval": 10.0, + "host": "linux-devbox", + "plugin": "statsd", + "plugin_instance": "", + "type": "count", + "type_instance": "mypythonapp.counter1" + }, + { + "values": [0.999995505178062], + "dstypes": ["derive"], + "dsnames": ["value"], + "time": 1698360892.757, + "interval": 10.0, + "host": "linux-devbox", + "plugin": "statsd", + "plugin_instance": "", + "type": "derive", + "type_instance": "mypythonapp.counter2" + }, + { + "values": [10], + "dstypes": ["gauge"], + "dsnames": ["value"], + "time": 1698360892.757, + "interval": 10.0, + "host": "linux-devbox", + "plugin": "statsd", + "plugin_instance": "", + "type": "count", + "type_instance": "mypythonapp.counter2" + }, + { + "values": [0.0999993716964859], + "dstypes": ["derive"], + "dsnames": ["value"], + "time": 1698360892.757, + "interval": 10.0, + "host": "linux-devbox", + "plugin": "statsd", + "plugin_instance": "", + "type": "derive", + "type_instance": "mypythonapp.counter1" + } +] diff --git a/memfaultd/src/collectd/mod.rs b/memfaultd/src/collectd/mod.rs new file mode 100644 index 0000000..9bc7b99 --- /dev/null +++ b/memfaultd/src/collectd/mod.rs @@ -0,0 +1,7 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +mod payload; + +mod collectd_handler; +pub use collectd_handler::CollectdHandler; diff --git a/memfaultd/src/collectd/payload.rs b/memfaultd/src/collectd/payload.rs new file mode 100644 index 0000000..9b86711 --- /dev/null +++ b/memfaultd/src/collectd/payload.rs @@ -0,0 +1,210 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::str::FromStr; + +use chrono::{DateTime, Duration, Utc}; +use eyre::{eyre, Result}; +use itertools::izip; +use log::warn; +use serde::Deserialize; + +use crate::{ + metrics::{KeyedMetricReading, MetricReading, MetricStringKey}, + util::serialization::float_to_datetime, + util::serialization::float_to_duration, +}; + +/// https://collectd.org/wiki/index.php/Data_source +/// and https://git.octo.it/?p=collectd.git;a=blob;f=src/daemon/plugin.h;hb=master#l45 +#[derive(Debug, Deserialize)] +enum DataSourceType { + #[serde(rename = "gauge")] + Gauge, + #[serde(rename = "derive")] + Derive, + #[serde(rename = "counter")] + Counter, + #[serde(rename = "absolute")] + Absolute, + #[serde(rename = "unknown")] + Unknown, +} + +/// https://collectd.org/wiki/index.php/JSON +/// https://collectd.org/wiki/index.php/Value_list +#[derive(Debug, Deserialize)] +pub struct Payload { + dsnames: Vec, + dstypes: Vec, + #[allow(dead_code)] + host: String, + // CollectD encodes time and duration to a float before sending as JSON. + // https://github.com/collectd/collectd/blob/main/src/utils/format_json/format_json.c#L344-L345 + #[allow(dead_code)] + #[serde(with = "float_to_duration")] + interval: Duration, + plugin: String, + plugin_instance: Option, + #[serde(with = "float_to_datetime")] + time: DateTime, + #[serde(rename = "type")] + type_str: String, + type_instance: Option, + values: Vec>, +} + +impl Payload { + fn metric_name(&self, name: &String) -> Result { + let use_simple_reading_name = self.dsnames.len() == 1 && self.dsnames[0] == "value"; + let name_prefix = vec![ + Some(&self.plugin), + self.plugin_instance.as_ref(), + Some(&self.type_str), + self.type_instance.as_ref(), + ] + .into_iter() + .flatten() + .filter(|x| !x.is_empty()) + .map(|x| x.as_str()) + .collect::>() + .join("/"); + + let name = if use_simple_reading_name { + name_prefix + } else { + format!("{}/{}", name_prefix, name) + }; + MetricStringKey::from_str(&name).map_err(|e| eyre!("Invalid metric name: {e}")) + } +} + +impl From for Vec { + fn from(payload: Payload) -> Self { + izip!(&payload.dsnames, &payload.values, &payload.dstypes) + // Remove variables that have no value + .filter_map(|(name, value, dstype)| value.as_ref().map(|v| (name, v, dstype))) + // Remove variables with invalid names + .filter_map(|(name, value, dstype)| match payload.metric_name(name) { + Ok(key) => Some((key, value, dstype)), + _ => { + warn!("Ignoring metric with invalid name: {}", name); + None + } + }) + // Create the KeyedMetricValue + .map(|(key, value, dstype)| KeyedMetricReading { + name: key, + value: match dstype { + // Refer to https://github.com/collectd/collectd/wiki/Data-source + // for a general description of what CollectdD datasources are. + + // Statsd generated counter values. + // See https://github.com/collectd/collectd/blob/7c5ce9f250aafbb6ef89769d7543ea155618b2ad/src/statsd.c#L799-L810 + DataSourceType::Gauge if payload.type_str == "count" => { + MetricReading::Counter { + value: *value, + timestamp: payload.time, + } + } + DataSourceType::Gauge => MetricReading::Histogram { + value: *value, + timestamp: payload.time, + }, + DataSourceType::Derive => MetricReading::Histogram { + value: *value, + timestamp: payload.time, + }, + // A counter is a Derive (rate) that will never be negative. + DataSourceType::Counter => MetricReading::Histogram { + value: *value, + timestamp: payload.time, + }, + DataSourceType::Absolute => MetricReading::Histogram { + value: *value, + timestamp: payload.time, + }, + DataSourceType::Unknown => MetricReading::Histogram { + value: *value, + timestamp: payload.time, + }, + }, + }) + .collect::>() + } +} + +#[cfg(test)] +mod tests { + use chrono::Duration; + use std::fs::read_to_string; + use std::path::PathBuf; + + use rstest::rstest; + + use crate::metrics::KeyedMetricReading; + + use super::DataSourceType; + use super::Payload; + + #[rstest] + #[case("A", Some("B"), "C", Some("D"), "E", "A/B/C/D/E")] + #[case("A", None, "C", Some("D"), "E", "A/C/D/E")] + #[case("A", Some("B"), "C", None, "E", "A/B/C/E")] + #[case("A", None, "C", None, "E", "A/C/E")] + #[case("A", Some(""), "C", Some(""), "E", "A/C/E")] + #[case("A", Some("B"), "C", Some("D"), "value", "A/B/C/D")] + fn convert_collectd_to_metric_name( + #[case] plugin: &str, + #[case] plugin_instance: Option<&str>, + #[case] type_s: &str, + #[case] type_instance: Option<&str>, + #[case] dsname: &str, + #[case] expected: &str, + ) { + let p = Payload { + dsnames: vec![dsname.to_string()], + dstypes: vec![DataSourceType::Gauge], + host: "".to_string(), + interval: Duration::seconds(10), + plugin: plugin.to_string(), + plugin_instance: plugin_instance.map(|x| x.to_owned()), + time: chrono::Utc::now(), + type_str: type_s.to_string(), + type_instance: type_instance.map(|x| x.to_owned()), + values: vec![Some(42.0)], + }; + + let readings = Vec::::from(p); + assert_eq!(readings.len(), 1); + let KeyedMetricReading { name, .. } = &readings[0]; + assert_eq!(name.as_str(), expected) + } + + #[rstest] + // Note: sample1 contains multiple payloads. Some have equal timestamps (and need to be consolidated), some have "simple values". + #[case("sample1")] + #[case("sample-with-null")] + #[case("statsd-counter-first-seen")] + #[case("statsd-counter")] + fn convert_collectd_payload_into_heartbeat_metadata(#[case] name: &str) { + let input_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/collectd/fixtures") + .join(name) + .with_extension("json"); + + // Read multiple payloads from a single file (this is how we receive from CollectD) + let payloads = + serde_json::from_str::>(&read_to_string(&input_path).unwrap()) + .unwrap(); + + // Convert payload into metric-readings + let metadatas = payloads + .into_iter() + .flat_map(Vec::::from) + .collect::>(); + + // Check results + insta::assert_json_snapshot!(format!("{name}"), metadatas); + } +} diff --git a/memfaultd/src/collectd/snapshots/memfaultd__collectd__collectd_handler__tests__drops_cpu_metrics_when_builtin_system_metrics_are_enabled.snap b/memfaultd/src/collectd/snapshots/memfaultd__collectd__collectd_handler__tests__drops_cpu_metrics_when_builtin_system_metrics_are_enabled.snap new file mode 100644 index 0000000..c90ceff --- /dev/null +++ b/memfaultd/src/collectd/snapshots/memfaultd__collectd__collectd_handler__tests__drops_cpu_metrics_when_builtin_system_metrics_are_enabled.snap @@ -0,0 +1,8 @@ +--- +source: memfaultd/src/collectd/collectd_handler.rs +expression: metrics +--- +{ + "cpufreq/0/cpu/idle": 0.0, + "mockplugin/0/mock/test": 0.0 +} diff --git a/memfaultd/src/collectd/snapshots/memfaultd__collectd__collectd_handler__tests__handle_push.snap b/memfaultd/src/collectd/snapshots/memfaultd__collectd__collectd_handler__tests__handle_push.snap new file mode 100644 index 0000000..958fce1 --- /dev/null +++ b/memfaultd/src/collectd/snapshots/memfaultd__collectd__collectd_handler__tests__handle_push.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/collectd/collectd_handler.rs +expression: "serde_json::to_string_pretty(&metrics).expect(\"metric_store should be serializable\")" +--- +{ + "cpu/0/cpu/idle": 0.0 +} diff --git a/memfaultd/src/collectd/snapshots/memfaultd__collectd__collectd_handler__tests__ignores_data_when_data_collection_is_off.snap b/memfaultd/src/collectd/snapshots/memfaultd__collectd__collectd_handler__tests__ignores_data_when_data_collection_is_off.snap new file mode 100644 index 0000000..3f150d8 --- /dev/null +++ b/memfaultd/src/collectd/snapshots/memfaultd__collectd__collectd_handler__tests__ignores_data_when_data_collection_is_off.snap @@ -0,0 +1,5 @@ +--- +source: memfaultd/src/collectd/collectd_handler.rs +expression: "serde_json::to_string_pretty(&metrics).expect(\"metric_store should be serializable\")" +--- +{} diff --git a/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__sample-with-null.snap b/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__sample-with-null.snap new file mode 100644 index 0000000..27cd690 --- /dev/null +++ b/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__sample-with-null.snap @@ -0,0 +1,42 @@ +--- +source: memfaultd/src/collectd/payload.rs +expression: metadatas +--- +[ + { + "name": "uptime/uptime", + "value": { + "Histogram": { + "value": 973577.0, + "timestamp": "2023-06-06T22:04:02.785Z" + } + } + }, + { + "name": "memory/memory/inactive", + "value": { + "Histogram": { + "value": 11160961024.0, + "timestamp": "2023-06-06T22:04:02.785Z" + } + } + }, + { + "name": "memory/memory/active", + "value": { + "Histogram": { + "value": 11797676032.0, + "timestamp": "2023-06-06T22:04:02.785Z" + } + } + }, + { + "name": "memory/memory/wired", + "value": { + "Histogram": { + "value": 3427991552.0, + "timestamp": "2023-06-06T22:04:02.785Z" + } + } + } +] diff --git a/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__sample1.snap b/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__sample1.snap new file mode 100644 index 0000000..1108cb3 --- /dev/null +++ b/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__sample1.snap @@ -0,0 +1,168 @@ +--- +source: memfaultd/src/collectd/payload.rs +expression: metadatas +--- +[ + { + "name": "disk/1-0/disk_octets/read", + "value": { + "Histogram": { + "value": 1887359.3321528, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "disk/1-0/disk_octets/write", + "value": { + "Histogram": { + "value": 1241166.17826548, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "disk/1-20/disk_time/read", + "value": { + "Histogram": { + "value": 992441.95537229, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "disk/1-20/disk_time/write", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "disk/1-0/disk_time/read", + "value": { + "Histogram": { + "value": 17146.2298941808, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "disk/1-0/disk_time/write", + "value": { + "Histogram": { + "value": 6375.40234502902, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "disk/1-0/disk_ops/read", + "value": { + "Histogram": { + "value": 75.3295819258201, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "disk/1-0/disk_ops/write", + "value": { + "Histogram": { + "value": 101.940031849151, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "memory/memory/active", + "value": { + "Histogram": { + "value": 8541454336.0, + "timestamp": "2023-05-24T17:25:47.352Z" + } + } + }, + { + "name": "memory/memory/inactive", + "value": { + "Histogram": { + "value": 8286470144.0, + "timestamp": "2023-05-24T17:25:47.352Z" + } + } + }, + { + "name": "memory/memory/free", + "value": { + "Histogram": { + "value": 502759424.0, + "timestamp": "2023-05-24T17:25:47.352Z" + } + } + }, + { + "name": "statsd/latency/app.kombu_publish_duration,exchange=celeryev,routing_key=worker.heartbeat-average", + "value": { + "Histogram": { + "value": 0.00399999972432852, + "timestamp": "2023-05-24T17:25:47.352Z" + } + } + }, + { + "name": "memory/memory/wired", + "value": { + "Histogram": { + "value": 3314139136.0, + "timestamp": "2023-05-24T17:25:47.352Z" + } + } + }, + { + "name": "disk/1-20/disk_ops/read", + "value": { + "Histogram": { + "value": 17.3067461702536, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "disk/1-20/disk_ops/write", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "disk/1-20/disk_octets/read", + "value": { + "Histogram": { + "value": 70888.4535866987, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "disk/1-20/disk_octets/write", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "uptime/uptime", + "value": { + "Histogram": { + "value": 790156.0, + "timestamp": "2023-05-24T17:25:47.353Z" + } + } + } +] diff --git a/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__statsd-counter-first-seen.snap b/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__statsd-counter-first-seen.snap new file mode 100644 index 0000000..9b38f4d --- /dev/null +++ b/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__statsd-counter-first-seen.snap @@ -0,0 +1,24 @@ +--- +source: memfaultd/src/collectd/payload.rs +expression: metadatas +--- +[ + { + "name": "statsd/count/mypythonapp.counter1", + "value": { + "Counter": { + "value": 2.0, + "timestamp": "2023-10-26T22:44:12.757Z" + } + } + }, + { + "name": "statsd/count/mypythonapp.counter2", + "value": { + "Counter": { + "value": 20.0, + "timestamp": "2023-10-26T22:44:12.757Z" + } + } + } +] diff --git a/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__statsd-counter.snap b/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__statsd-counter.snap new file mode 100644 index 0000000..32f5b8c --- /dev/null +++ b/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__statsd-counter.snap @@ -0,0 +1,42 @@ +--- +source: memfaultd/src/collectd/payload.rs +expression: metadatas +--- +[ + { + "name": "statsd/count/mypythonapp.counter1", + "value": { + "Counter": { + "value": 1.0, + "timestamp": "2023-10-26T22:54:52.757Z" + } + } + }, + { + "name": "statsd/derive/mypythonapp.counter2", + "value": { + "Histogram": { + "value": 0.999995505178062, + "timestamp": "2023-10-26T22:54:52.757Z" + } + } + }, + { + "name": "statsd/count/mypythonapp.counter2", + "value": { + "Counter": { + "value": 10.0, + "timestamp": "2023-10-26T22:54:52.757Z" + } + } + }, + { + "name": "statsd/derive/mypythonapp.counter1", + "value": { + "Histogram": { + "value": 0.0999993716964859, + "timestamp": "2023-10-26T22:54:52.757Z" + } + } + } +] diff --git a/memfaultd/src/config/config_file.rs b/memfaultd/src/config/config_file.rs new file mode 100644 index 0000000..a6ceb02 --- /dev/null +++ b/memfaultd/src/config/config_file.rs @@ -0,0 +1,521 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::{eyre, Context}; +use serde::{Deserialize, Serialize}; +use std::net::IpAddr; +use std::time::Duration; +use std::{ + collections::{HashMap, HashSet}, + num::NonZeroU32, +}; +use std::{net::SocketAddr, path::PathBuf}; + +use crate::metrics::{MetricStringKey, SessionName}; +use crate::util::*; +use crate::util::{path::AbsolutePath, serialization::*}; + +#[derive(Serialize, Deserialize, Debug)] +pub struct MemfaultdConfig { + pub persist_dir: AbsolutePath, + pub tmp_dir: Option, + #[serde(rename = "tmp_dir_min_headroom_kib", with = "kib_to_usize")] + pub tmp_dir_min_headroom: usize, + pub tmp_dir_min_inodes: usize, + #[serde(rename = "tmp_dir_max_usage_kib", with = "kib_to_usize")] + pub tmp_dir_max_usage: usize, + #[serde(rename = "upload_interval_seconds", with = "seconds_to_duration")] + pub upload_interval: Duration, + #[serde(rename = "heartbeat_interval_seconds", with = "seconds_to_duration")] + pub heartbeat_interval: Duration, + pub enable_data_collection: bool, + pub enable_dev_mode: bool, + pub software_version: Option, + pub software_type: Option, + pub project_key: String, + pub base_url: String, + pub swupdate: SwUpdateConfig, + pub reboot: RebootConfig, + pub coredump: CoredumpConfig, + #[serde(rename = "fluent-bit")] + pub fluent_bit: FluentBitConfig, + pub logs: LogsConfig, + pub mar: MarConfig, + pub http_server: HttpServerConfig, + pub battery_monitor: Option, + pub connectivity_monitor: Option, + pub sessions: Option>, + pub metrics: MetricReportConfig, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct SwUpdateConfig { + pub input_file: PathBuf, + pub output_file: PathBuf, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct RebootConfig { + pub last_reboot_reason_file: PathBuf, +} + +#[derive(Serialize, Deserialize, Debug, Copy, Clone)] +pub enum CoredumpCompression { + #[serde(rename = "gzip")] + Gzip, + #[serde(rename = "none")] + None, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy)] +#[serde(tag = "type")] +pub enum CoredumpCaptureStrategy { + #[serde(rename = "threads")] + /// Only capture the stacks of the threads that were running at the time of the crash. + Threads { + #[serde(rename = "max_thread_size_kib", with = "kib_to_usize")] + max_thread_size: usize, + }, + #[serde(rename = "kernel_selection")] + /// Keep in the coredump what the kernel selected to be included in the coredump. + /// See https://man7.org/linux/man-pages/man5/core.5.html for more details. + KernelSelection, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct CoredumpConfig { + pub compression: CoredumpCompression, + #[serde(rename = "coredump_max_size_kib", with = "kib_to_usize")] + pub coredump_max_size: usize, + pub rate_limit_count: u32, + #[serde(rename = "rate_limit_duration_seconds", with = "seconds_to_duration")] + pub rate_limit_duration: Duration, + pub capture_strategy: CoredumpCaptureStrategy, + pub log_lines: usize, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct FluentBitConfig { + pub extra_fluentd_attributes: Vec, + pub bind_address: SocketAddr, + pub max_buffered_lines: usize, + pub max_connections: usize, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct HttpServerConfig { + pub bind_address: SocketAddr, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy)] +pub enum LogSource { + #[serde(rename = "fluent-bit")] + FluentBit, + #[cfg(feature = "systemd")] + #[serde(rename = "journald")] + Journald, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct LogsConfig { + #[serde(rename = "rotate_size_kib", with = "kib_to_usize")] + pub rotate_size: usize, + + #[serde(rename = "rotate_after_seconds", with = "seconds_to_duration")] + pub rotate_after: Duration, + + #[serde(with = "number_to_compression")] + pub compression_level: Compression, + + pub max_lines_per_minute: NonZeroU32, + + pub log_to_metrics: Option, + + pub storage: StorageConfig, + + pub source: LogSource, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy)] +pub enum StorageConfig { + #[serde(rename = "disabled")] + Disabled, + #[serde(rename = "persist")] + Persist, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct LogToMetricsConfig { + pub rules: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(tag = "type")] +pub enum LogToMetricRule { + #[serde(rename = "count_matching")] + CountMatching { + /// Regex applied on the MESSAGE field + pattern: String, + metric_name: String, + /// List of key-value that must exactly match before the regexp is applied + #[serde(default)] + filter: HashMap, + }, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct MarConfig { + #[serde(rename = "mar_file_max_size_kib", with = "kib_to_usize")] + pub mar_file_max_size: usize, + #[serde(rename = "mar_entry_max_age_seconds", with = "seconds_to_duration")] + pub mar_entry_max_age: Duration, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct BatteryMonitorConfig { + pub battery_info_command: String, + #[serde(with = "seconds_to_duration")] + pub interval_seconds: Duration, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct ConnectivityMonitorConfig { + #[serde(with = "seconds_to_duration")] + pub interval_seconds: Duration, + pub targets: Vec, + #[serde( + with = "seconds_to_duration", + default = "default_connection_check_timeout" + )] + pub timeout_seconds: Duration, +} +fn default_connection_check_timeout() -> Duration { + Duration::from_secs(10) +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct MetricReportConfig { + pub enable_daily_heartbeats: bool, + pub system_metric_collection: SystemMetricConfig, + pub statsd_server: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ConnectivityMonitorTarget { + #[serde(default = "default_connection_check_protocol")] + pub protocol: ConnectionCheckProtocol, + pub host: IpAddr, + pub port: u16, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "lowercase")] +pub enum ConnectionCheckProtocol { + Tcp, +} + +fn default_connection_check_protocol() -> ConnectionCheckProtocol { + ConnectionCheckProtocol::Tcp +} + +#[derive(Serialize, Clone, Deserialize, Debug)] +pub struct SessionConfig { + pub name: SessionName, + pub captured_metrics: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct StatsDServerConfig { + pub bind_address: SocketAddr, +} + +#[derive(Clone, Serialize, Deserialize, Debug)] +pub struct SystemMetricConfig { + pub enable: bool, + #[serde(with = "seconds_to_duration")] + pub poll_interval_seconds: Duration, + pub processes: Option>, + pub disk_space: Option>, + pub network_interfaces: Option>, +} + +use flate2::Compression; +use serde_json::Value; +use std::fs; +use std::path::Path; + +use crate::config::utils::{software_type_is_valid, software_version_is_valid}; + +pub struct JsonConfigs { + /// Built-in configuration and System configuration + pub base: Value, + /// Runtime configuration + pub runtime: Value, +} + +impl MemfaultdConfig { + pub fn load(config_path: &Path) -> eyre::Result { + let JsonConfigs { + base: mut config_json, + runtime, + } = Self::parse_configs(config_path)?; + Self::merge_into(&mut config_json, runtime); + + // Transform the JSON object into a typed structure. + let config: MemfaultdConfig = serde_json::from_value(config_json)?; + + let mut validation_errors = vec![]; + if let Some(software_version) = &config.software_version { + if let Err(e) = software_version_is_valid(software_version) { + validation_errors.push(format!(" Invalid value for \"software_version\": {}", e)); + } + } + if let Some(software_type) = &config.software_type { + if let Err(e) = software_type_is_valid(software_type) { + validation_errors.push(format!(" Invalid value for \"software_type\": {}", e)); + } + } + + match validation_errors.is_empty() { + true => Ok(config), + false => Err(eyre::eyre!("\n{}", validation_errors.join("\n"))), + } + } + + /// Parse config file from given path and returns (builtin+system config, runtime config). + pub fn parse_configs(config_path: &Path) -> eyre::Result { + // Initialize with the builtin config file. + let mut base: Value = Self::parse(include_str!("../../builtin.conf")) + .wrap_err("Error parsing built-in configuration file")?; + + // Read and parse the user config file. + let user_config = Self::parse(std::fs::read_to_string(config_path)?.as_str()) + .wrap_err(eyre!("Error reading {}", config_path.display()))?; + + // Merge the two JSON objects together + Self::merge_into(&mut base, user_config); + + // Load the runtime config but only if the file exists. (Missing runtime config is not an error.) + let runtime_config_path = Self::runtime_config_path_from_json(&base)?; + let runtime = if runtime_config_path.exists() { + Self::parse(fs::read_to_string(&runtime_config_path)?.as_str()).wrap_err(eyre!( + "Error reading runtime configuration {}", + runtime_config_path.display() + ))? + } else { + Value::Object(serde_json::Map::new()) + }; + + Ok(JsonConfigs { base, runtime }) + } + + /// Set and write boolean in runtime config. + pub fn set_and_write_bool_to_runtime_config(&self, key: &str, value: bool) -> eyre::Result<()> { + let config_string = match fs::read_to_string(self.runtime_config_path()) { + Ok(config_string) => config_string, + Err(e) => { + if e.kind() == std::io::ErrorKind::NotFound { + "{}".to_string() + } else { + return Err(eyre::eyre!("Failed to read runtime config: {}", e)); + } + } + }; + + let mut config_val = Self::parse(&config_string)?; + config_val[key] = Value::Bool(value); + + self.write_value_to_runtime_config(config_val) + } + + /// Write config to runtime config file. + /// + /// This is used to write the config to a file that can be read by the memfaultd process. + fn write_value_to_runtime_config(&self, value: Value) -> eyre::Result<()> { + let runtime_config_path = self.runtime_config_path(); + fs::write(runtime_config_path, value.to_string())?; + + Ok(()) + } + + pub fn runtime_config_path(&self) -> PathBuf { + PathBuf::from(self.persist_dir.clone()).join("runtime.conf") + } + + // Parse a Memfaultd JSON configuration file (with optional C-style comments) and return a serde_json::Value object. + fn parse(config_string: &str) -> eyre::Result { + let json_text = string::remove_comments(config_string); + let json: Value = serde_json::from_str(json_text.as_str())?; + if !json.is_object() { + return Err(eyre::eyre!("Configuration should be a JSON object.")); + } + Ok(json) + } + + /// Merge two JSON objects together. The values from the second one will override values in the first one. + fn merge_into(dest: &mut Value, src: Value) { + assert!(dest.is_object() && src.is_object()); + if let Value::Object(dest_map) = src { + for (key, value) in dest_map { + if let Some(obj) = dest.get_mut(&key) { + if obj.is_object() { + MemfaultdConfig::merge_into(obj, value); + continue; + } + } + dest[&key] = value; + } + } + } + + pub fn generate_tmp_filename(&self, filename: &str) -> PathBuf { + // Fall back to persist dir if tmp_dir is not set. + let tmp_dir = self.tmp_dir.as_ref().unwrap_or(&self.persist_dir); + PathBuf::from(tmp_dir.clone()).join(filename) + } + + pub fn generate_persist_filename(&self, filename: &str) -> PathBuf { + PathBuf::from(self.persist_dir.clone()).join(filename) + } + + /// Generate the path to the runtime config file from a serde_json::Value object. This should include the "persist_dir" field. + fn runtime_config_path_from_json(config: &Value) -> eyre::Result { + let mut persist_dir = PathBuf::from( + config["persist_dir"] + .as_str() + .ok_or(eyre::eyre!("Config['persist_dir'] must be a string."))?, + ); + persist_dir.push("runtime.conf"); + Ok(persist_dir) + } +} + +#[cfg(test)] +impl MemfaultdConfig { + pub fn test_fixture() -> Self { + use std::fs::write; + use tempfile::tempdir; + + let tmp = tempdir().unwrap(); + let config_path = tmp.path().join("memfaultd.conf"); + write(&config_path, "{}").unwrap(); + MemfaultdConfig::load(&config_path).unwrap() + } +} + +#[cfg(test)] +mod test { + use insta::{assert_json_snapshot, with_settings}; + use rstest::rstest; + + use super::*; + + use crate::test_utils::set_snapshot_suffix; + + #[test] + fn test_merge() { + let mut c = + serde_json::from_str(r#"{ "node": { "value": true, "valueB": false } }"#).unwrap(); + let j = serde_json::from_str(r#"{ "node2": "xxx" }"#).unwrap(); + + MemfaultdConfig::merge_into(&mut c, j); + + assert_eq!( + serde_json::to_string(&c).unwrap(), + r#"{"node":{"value":true,"valueB":false},"node2":"xxx"}"# + ); + } + + #[test] + fn test_merge_overwrite() { + let mut c = + serde_json::from_str(r#"{ "node": { "value": true, "valueB": false } }"#).unwrap(); + let j = serde_json::from_str(r#"{ "node": { "value": false }}"#).unwrap(); + + MemfaultdConfig::merge_into(&mut c, j); + + assert_eq!( + serde_json::to_string(&c).unwrap(), + r#"{"node":{"value":false,"valueB":false}}"# + ); + } + + #[test] + fn test_merge_overwrite_nested() { + let mut c = serde_json::from_str( + r#"{ "node": { "value": true, "valueB": false, "valueC": { "a": 1, "b": 2 } } }"#, + ) + .unwrap(); + let j = serde_json::from_str(r#"{ "node": { "valueC": { "b": 42 } }}"#).unwrap(); + + MemfaultdConfig::merge_into(&mut c, j); + + assert_eq!( + serde_json::to_string(&c).unwrap(), + r#"{"node":{"value":true,"valueB":false,"valueC":{"a":1,"b":42}}}"# + ); + } + + #[rstest] + #[case("empty_object")] + #[case("with_partial_logs")] + #[case("without_coredump_compression")] + #[case("with_coredump_capture_strategy_threads")] + #[case("with_log_to_metrics_rules")] + #[case("with_connectivity_monitor")] + #[case("with_sessions")] + #[case("metrics_config")] + fn can_parse_test_files(#[case] name: &str) { + let input_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/config/test-config") + .join(name) + .with_extension("json"); + // Verifies that the file is parsable + let content = MemfaultdConfig::load(&input_path).unwrap(); + // And that the configuration generated is what we expect. + // Use `cargo insta review` to quickly approve changes. + with_settings!({sort_maps => true}, { + assert_json_snapshot!(name, content); + }); + } + + #[rstest] + #[case("with_invalid_path")] + #[case("with_invalid_swt_swv")] + #[case("with_sessions_invalid_metric_name")] + #[case("with_sessions_invalid_session_name")] + fn will_reject_bad_config(#[case] name: &str) { + let input_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/config/test-config") + .join(name) + .with_extension("json"); + let result = MemfaultdConfig::load(&input_path); + assert!(result.is_err()); + } + + #[rstest] + #[case("no_file", None)] + #[case("empty_object", Some("{}"))] + #[case("other_key", Some(r#"{"key2":false}"#))] + fn test_set_and_write_bool_to_runtime_config( + #[case] test_name: &str, + #[case] config_string: Option<&str>, + ) { + let mut config = MemfaultdConfig::test_fixture(); + let temp_data_dir = tempfile::tempdir().unwrap(); + config.persist_dir = AbsolutePath::try_from(temp_data_dir.path().to_path_buf()).unwrap(); + + if let Some(config_string) = config_string { + std::fs::write(config.runtime_config_path(), config_string).unwrap(); + } + + config + .set_and_write_bool_to_runtime_config("key", true) + .unwrap(); + + let disk_config_string = std::fs::read_to_string(config.runtime_config_path()).unwrap(); + + set_snapshot_suffix!("{}", test_name); + insta::assert_json_snapshot!(disk_config_string); + } +} diff --git a/memfaultd/src/config/device_config.rs b/memfaultd/src/config/device_config.rs new file mode 100644 index 0000000..e10c0c5 --- /dev/null +++ b/memfaultd/src/config/device_config.rs @@ -0,0 +1,93 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use serde::{Deserialize, Serialize}; + +use crate::network::{DeviceConfigResponse, DeviceConfigResponseResolution, DeviceConfigRevision}; + +#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)] +pub enum Resolution { + Off, + Low, + Normal, + High, +} + +impl From for Resolution { + fn from(resolution: DeviceConfigResponseResolution) -> Self { + match resolution { + DeviceConfigResponseResolution::Off => Resolution::Off, + DeviceConfigResponseResolution::Low => Resolution::Low, + DeviceConfigResponseResolution::Normal => Resolution::Normal, + DeviceConfigResponseResolution::High => Resolution::High, + } + } +} + +#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq)] +pub struct Sampling { + pub debugging_resolution: Resolution, + pub logging_resolution: Resolution, + pub monitoring_resolution: Resolution, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +/// DeviceConfig is configuration provided by Memfault backend. +pub struct DeviceConfig { + pub revision: Option, + pub sampling: Sampling, +} + +impl From for DeviceConfig { + fn from(response: DeviceConfigResponse) -> Self { + Self { + revision: Some(response.data.revision), + sampling: Sampling { + debugging_resolution: response + .data + .config + .memfault + .sampling + .debugging_resolution + .into(), + logging_resolution: response + .data + .config + .memfault + .sampling + .logging_resolution + .into(), + monitoring_resolution: response + .data + .config + .memfault + .sampling + .monitoring_resolution + .into(), + }, + } + } +} + +impl Default for DeviceConfig { + fn default() -> Self { + Self { + revision: None, + sampling: Sampling { + debugging_resolution: Resolution::Off, + logging_resolution: Resolution::Off, + monitoring_resolution: Resolution::Off, + }, + } + } +} + +impl Sampling { + pub fn development() -> Self { + Self { + debugging_resolution: Resolution::High, + logging_resolution: Resolution::High, + monitoring_resolution: Resolution::High, + } + } +} diff --git a/memfaultd/src/config/device_info.rs b/memfaultd/src/config/device_info.rs new file mode 100644 index 0000000..f566e69 --- /dev/null +++ b/memfaultd/src/config/device_info.rs @@ -0,0 +1,545 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::fmt::{self, Display}; +use std::fs::read_to_string; +use std::process::Command; + +use eyre::{eyre, Result}; + +use crate::config::utils::{ + device_id_is_valid, hardware_version_is_valid, software_type_is_valid, + software_version_is_valid, +}; +use crate::util::etc_os_release::EtcOsRelease; + +const DEVICE_ID_PATH: &str = "/etc/machine-id"; +const HARDWARE_VERSION_COMMAND: &str = "uname"; +const HARDWARE_VERSION_ARGS: &[&str] = &["-n"]; + +#[cfg_attr(test, mockall::automock)] +/// Trait for providing default values for device info. +/// +/// This is mostly a convenience for testing, as the default implementation +/// reads the software version from /etc/os-release. +pub trait DeviceInfoDefaults { + /// Get the software version from the system. + fn software_version(&self) -> Result>; + + /// Get the device ID from the system. + fn device_id(&self) -> Result; + + /// Get the hardware version from the system. + fn hardware_version(&self) -> Result; + + /// Get the software type from the system. + fn software_type(&self) -> Result>; +} + +/// Default implementation of DeviceInfoDefaults. +pub struct DeviceInfoDefaultsImpl { + os_release: Option, +} + +impl DeviceInfoDefaultsImpl { + fn new(os_release: Option) -> Self { + Self { os_release } + } +} + +impl DeviceInfoDefaults for DeviceInfoDefaultsImpl { + fn software_version(&self) -> Result> { + Ok(self.os_release.as_ref().and_then(|os| os.version_id())) + } + + fn device_id(&self) -> Result { + let device_id = read_to_string(DEVICE_ID_PATH)?; + + Ok(device_id) + } + + fn hardware_version(&self) -> Result { + let output = Command::new(HARDWARE_VERSION_COMMAND) + .args(HARDWARE_VERSION_ARGS) + .output()?; + + Ok(String::from_utf8(output.stdout)?) + } + + fn software_type(&self) -> Result> { + Ok(self.os_release.as_ref().and_then(|os| os.id())) + } +} + +#[derive(Debug, PartialEq, Eq)] +pub enum DeviceInfoValue { + Configured(String), + Default(String), +} + +impl AsRef for DeviceInfoValue { + fn as_ref(&self) -> &str { + match self { + DeviceInfoValue::Configured(s) => s.as_ref(), + DeviceInfoValue::Default(s) => s.as_ref(), + } + } +} + +impl Display for DeviceInfoValue { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.as_ref()) + } +} + +#[derive(Debug)] +pub struct DeviceInfo { + pub device_id: String, + pub hardware_version: String, + pub software_version: Option, + pub software_type: Option, +} + +#[derive(PartialEq, Eq, Debug)] +pub struct DeviceInfoWarning { + line: String, + message: String, +} + +impl std::fmt::Display for DeviceInfoWarning { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Skipped line: '{}' ({})", self.line, self.message) + } +} + +impl DeviceInfo { + pub fn parse( + output: Option<&[u8]>, + defaults: T, + ) -> Result<(DeviceInfo, Vec)> { + let mut warnings = vec![]; + + let mut software_version = defaults + .software_version() + .unwrap_or_else(|_| { + warnings.push(DeviceInfoWarning { + line: "".into(), + message: "Failed to get default software version.".to_string(), + }); + None + }) + .map(DeviceInfoValue::Default); + let mut device_id = defaults.device_id().map_or_else( + |_| { + warnings.push(DeviceInfoWarning { + line: "".into(), + message: format!("Failed to open {}", DEVICE_ID_PATH), + }); + None + }, + |id| Some(id.trim().to_string()), + ); + let mut hardware_version = defaults.hardware_version().map_or_else( + |_| { + warnings.push(DeviceInfoWarning { + line: "".into(), + message: format!( + "Failed to to get hardware version from: '{}'", + HARDWARE_VERSION_COMMAND + ), + }); + None + }, + |hwv| Some(hwv.trim().to_string()), + ); + let mut software_type = defaults + .software_type() + .unwrap_or_else(|_| { + warnings.push(DeviceInfoWarning { + line: "".into(), + message: "Failed to get default software_type.".to_string(), + }); + None + }) + .map(DeviceInfoValue::Default); + + match output { + Some(output) => { + for line in std::str::from_utf8(output)?.lines() { + if let Some((key, value)) = line.split_once('=') { + match key { + "MEMFAULT_DEVICE_ID" => device_id = Some(value.into()), + "MEMFAULT_HARDWARE_VERSION" => hardware_version = Some(value.into()), + "MEMFAULT_SOFTWARE_VERSION" => { + software_version = Some(DeviceInfoValue::Configured(value.into())) + } + "MEMFAULT_SOFTWARE_TYPE" => { + software_type = Some(DeviceInfoValue::Configured(value.into())) + } + _ => warnings.push(DeviceInfoWarning { + line: line.into(), + message: "Unknown variable.".to_string(), + }), + } + } else { + warnings.push(DeviceInfoWarning { + line: line.into(), + message: "Expect '=' separated key/value pairs.".to_string(), + }) + } + } + } + None => { + warnings.push(DeviceInfoWarning { + line: "".into(), + message: "No output from memfault-device-info.".to_string(), + }); + } + } + + let di = DeviceInfo { + device_id: device_id.ok_or(eyre!("No device id supplied"))?, + hardware_version: hardware_version.ok_or(eyre!("No hardware version supplied"))?, + software_version, + software_type, + }; + + // Create vector of keys whose values have invalid characters + let validation_errors: Vec = [ + ( + "MEMFAULT_HARDWARE_VERSION", + hardware_version_is_valid(&di.hardware_version), + ), + ( + "MEMFAULT_SOFTWARE_VERSION", + di.software_version + .as_ref() + .map_or(Ok(()), |swv| software_version_is_valid(swv.as_ref())), + ), + ( + "MEMFAULT_SOFTWARE_TYPE", + di.software_type + .as_ref() + .map_or(Ok(()), |swt| software_type_is_valid(swt.as_ref())), + ), + ("MEMFAULT_DEVICE_ID", device_id_is_valid(&di.device_id)), + ] + .iter() + .filter_map(|(key, result)| match result { + Err(e) => Some(format!(" Invalid {}: {}", key, e)), + _ => None, + }) + .collect(); + + match validation_errors.is_empty() { + true => Ok((di, warnings)), + false => Err(eyre::eyre!("\n{}", validation_errors.join("\n"))), + } + } + + pub fn load() -> eyre::Result<(DeviceInfo, Vec)> { + let user_output = Command::new("memfault-device-info").output().ok(); + let stdout = user_output.as_ref().map(|o| o.stdout.as_slice()); + + let os_release = EtcOsRelease::load().ok(); + let di_defaults = DeviceInfoDefaultsImpl::new(os_release); + Self::parse(stdout, di_defaults) + } +} + +#[cfg(test)] +impl DeviceInfo { + pub fn test_fixture() -> Self { + DeviceInfo { + device_id: "001".to_owned(), + hardware_version: "DVT".to_owned(), + software_version: None, + software_type: None, + } + } + + pub fn test_fixture_with_overrides(software_version: &str, software_type: &str) -> Self { + DeviceInfo { + device_id: "001".to_owned(), + hardware_version: "DVT".to_owned(), + software_version: Some(DeviceInfoValue::Configured(software_version.into())), + software_type: Some(DeviceInfoValue::Configured(software_type.into())), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + #[test] + fn test_empty() { + let mut di_defaults = MockDeviceInfoDefaults::new(); + di_defaults + .expect_software_type() + .returning(|| Err(eyre!(""))); + di_defaults.expect_software_version().returning(|| Ok(None)); + di_defaults + .expect_hardware_version() + .returning(|| Err(eyre!(""))); + di_defaults + .expect_device_id() + .returning(|| Ok("123ABC".into())); + let r = DeviceInfo::parse(Some(b""), di_defaults); + assert!(r.is_err()) + } + + #[test] + fn test_with_warnings() { + let mut di_defaults = MockDeviceInfoDefaults::new(); + di_defaults.expect_software_type().returning(|| Ok(None)); + di_defaults.expect_software_version().returning(|| Ok(None)); + di_defaults + .expect_device_id() + .returning(|| Ok("123ABC".into())); + di_defaults + .expect_hardware_version() + .returning(|| Ok("Hardware".into())); + let r = DeviceInfo::parse( + Some(b"MEMFAULT_DEVICE_ID=X\nMEMFAULT_HARDWARE_VERSION=Y\nblahblahblah\n"), + di_defaults, + ); + assert!(r.is_ok()); + + let (di, warnings) = r.unwrap(); + assert_eq!(di.device_id, "X"); + assert_eq!(di.hardware_version, "Y"); + assert_eq!(warnings.len(), 1); + assert_eq!( + warnings[0], + DeviceInfoWarning { + line: "blahblahblah".into(), + message: "Expect '=' separated key/value pairs.".to_string() + } + ); + } + + #[rstest] + // Override software version + #[case(b"MEMFAULT_DEVICE_ID=123ABC\nMEMFAULT_HARDWARE_VERSION=1.0.0\nMEMFAULT_SOFTWARE_VERSION=1.2.3\n", Some("1.2.3".into()), None)] + // Override software type + #[case(b"MEMFAULT_DEVICE_ID=123ABC\nMEMFAULT_HARDWARE_VERSION=1.0.0\nMEMFAULT_SOFTWARE_TYPE=test\n", None, Some("test".into()))] + // Override both software version and type + #[case(b"MEMFAULT_DEVICE_ID=123ABC\nMEMFAULT_HARDWARE_VERSION=1.0.0\nMEMFAULT_SOFTWARE_VERSION=1.2.3\nMEMFAULT_SOFTWARE_TYPE=test\n", Some("1.2.3".into()), Some("test".into()))] + fn test_with_sw_version_and_type( + #[case] output: &[u8], + #[case] sw_version: Option, + #[case] sw_type: Option, + ) { + let mut di_defaults = MockDeviceInfoDefaults::new(); + di_defaults.expect_software_type().returning(|| Ok(None)); + di_defaults.expect_software_version().returning(|| Ok(None)); + di_defaults + .expect_hardware_version() + .returning(|| Ok("Hardware".into())); + di_defaults + .expect_device_id() + .returning(|| Ok("123ABC".into())); + let r = DeviceInfo::parse(Some(output), di_defaults); + assert!(r.is_ok()); + + let (di, warnings) = r.unwrap(); + assert_eq!(di.device_id, "123ABC"); + assert_eq!(di.hardware_version, "1.0.0"); + assert_eq!( + di.software_version, + sw_version.map(DeviceInfoValue::Configured) + ); + assert_eq!(di.software_type, sw_type.map(DeviceInfoValue::Configured)); + + assert_eq!(warnings.len(), 0); + } + + #[rstest] + #[case::default_with_no_response( + Some("1.2.3".to_string()), + Some(DeviceInfoValue::Default("1.2.3".to_string())), + b"" + )] + #[case::default_with_response( + Some("1.2.3".to_string()), + Some(DeviceInfoValue::Configured("1.2.4".to_string())), + b"MEMFAULT_SOFTWARE_VERSION=1.2.4" + )] + #[case::no_default_with_response( + None, + Some(DeviceInfoValue::Configured("1.2.4".to_string())), + b"MEMFAULT_SOFTWARE_VERSION=1.2.4" + )] + #[case::no_default_no_response(None, None, b"")] + fn test_with_default_swv( + #[case] software_version_default: Option, + #[case] expected: Option, + #[case] output: &[u8], + ) { + // Required device info parameters that will cause a panic if not present + let mut output_required = + b"MEMFAULT_DEVICE_ID=DEVICE\nMEMFAULT_HARDWARE_VERSION=HARDWARE\n".to_vec(); + output_required.extend(output); + + let mut di_defaults = MockDeviceInfoDefaults::new(); + di_defaults + .expect_software_type() + .returning(|| Err(eyre!(""))); + di_defaults + .expect_software_version() + .returning(move || Ok(software_version_default.clone())); + di_defaults + .expect_hardware_version() + .returning(|| Err(eyre!(""))); + di_defaults.expect_device_id().returning(|| Err(eyre!(""))); + + let (di, _warnings) = DeviceInfo::parse(Some(&output_required), di_defaults).unwrap(); + assert_eq!(di.software_version, expected); + } + + #[rstest] + #[case::default_with_no_response(Some("123ABC".to_string()), Some(DeviceInfoValue::Default("123ABC".to_string())), b"")] + #[case::default_with_response(Some("123ABC".to_string()), Some(DeviceInfoValue::Configured("main".to_string())), b"MEMFAULT_SOFTWARE_TYPE=main")] + #[case::no_default_with_response(None, Some(DeviceInfoValue::Configured("main".to_string())), b"MEMFAULT_SOFTWARE_TYPE=main")] + #[case::no_default_no_response(None, None, b"")] + fn test_with_default_sw_type( + #[case] software_type_default: Option, + #[case] expected: Option, + #[case] output: &[u8], + ) { + // Required device info parameters that will cause a panic if not present + let mut output_required = + b"MEMFAULT_DEVICE_ID=DEVICE\nMEMFAULT_HARDWARE_VERSION=HARDWARE\n".to_vec(); + output_required.extend(output); + + let mut di_defaults = MockDeviceInfoDefaults::new(); + di_defaults + .expect_software_version() + .returning(|| Err(eyre!(""))); + di_defaults + .expect_hardware_version() + .returning(|| Err(eyre!(""))); + di_defaults.expect_device_id().returning(|| Err(eyre!(""))); + di_defaults + .expect_software_type() + .returning(move || Ok(software_type_default.clone())); + + let (di, _warnings) = DeviceInfo::parse(Some(&output_required), di_defaults).unwrap(); + assert_eq!(di.software_type, expected); + } + + #[rstest] + #[case::default_with_no_response(Some("123ABC".to_string()), Some("123ABC".to_string()), b"")] + #[case::default_with_whitespace(Some("123ABC\n".to_string()), Some("123ABC".to_string()), b"")] + #[case::default_with_response(Some("123ABC".to_string()), Some("DEVICE".to_string()), b"MEMFAULT_DEVICE_ID=DEVICE")] + #[case::no_default_with_response(None, Some("DEVICE".to_string()), b"MEMFAULT_DEVICE_ID=DEVICE")] + #[case::no_default_no_response(None, None, b"")] + fn test_with_default_device_id( + #[case] device_id_default: Option, + #[case] expected: Option, + #[case] output: &[u8], + ) { + // Required device info parameters that will cause a panic if not present + let mut output_required = b"MEMFAULT_HARDWARE_VERSION=HARDWARE\n".to_vec(); + output_required.extend(output); + + let mut di_defaults = MockDeviceInfoDefaults::new(); + di_defaults + .expect_software_type() + .returning(|| Err(eyre!(""))); + di_defaults + .expect_software_version() + .returning(|| Err(eyre!(""))); + di_defaults + .expect_hardware_version() + .returning(|| Err(eyre!(""))); + di_defaults + .expect_device_id() + .returning(move || device_id_default.clone().ok_or(eyre!(""))); + + let ret = DeviceInfo::parse(Some(&output_required), di_defaults); + if let Some(expected) = expected { + let (di, _warnings) = ret.unwrap(); + assert_eq!(di.device_id, expected); + } else { + assert!(ret.is_err()); + } + } + + #[rstest] + #[case::default_with_no_response(Some("123ABC".to_string()), Some("123ABC".to_string()), b"")] + #[case::default_with_whitespace(Some("123ABC\n".to_string()), Some("123ABC".to_string()), b"")] + #[case::default_with_response(Some("123ABC".to_string()), Some("HARDWARE".to_string()), b"MEMFAULT_HARDWARE_VERSION=HARDWARE")] + #[case::no_default_with_response(None, Some("HARDWARE".to_string()), b"MEMFAULT_HARDWARE_VERSION=HARDWARE")] + #[case::no_default_no_response(None, None, b"")] + fn test_with_default_hardware_version( + #[case] hardware_version_default: Option, + #[case] expected: Option, + #[case] output: &[u8], + ) { + // Required device info parameters that will cause a panic if not present + let mut output_required = b"MEMFAULT_DEVICE_ID=DEVICE\n".to_vec(); + output_required.extend(output); + + let mut di_defaults = MockDeviceInfoDefaults::new(); + di_defaults + .expect_software_type() + .returning(|| Err(eyre!(""))); + di_defaults + .expect_software_version() + .returning(|| Err(eyre!(""))); + di_defaults.expect_device_id().returning(|| Err(eyre!(""))); + di_defaults + .expect_hardware_version() + .returning(move || hardware_version_default.clone().ok_or(eyre!(""))); + + let ret = DeviceInfo::parse(Some(&output_required), di_defaults); + if let Some(expected) = expected { + let (di, _warnings) = ret.unwrap(); + assert_eq!(di.hardware_version, expected); + } else { + assert!(ret.is_err()); + } + } + + #[rstest] + fn test_with_no_device_info() { + let expected_software_type = "SOFTWARE_TYPE".to_string(); + let expected_software_version = "SOFTWARE_VERSION".to_string(); + let expected_hardware_version = "HARDWARE_VERSION".to_string(); + let expected_device_id = "DEVICE_ID".to_string(); + + let mut di_defaults = MockDeviceInfoDefaults::new(); + let default_software_type = expected_software_type.clone(); + di_defaults + .expect_software_type() + .returning(move || Ok(Some(default_software_type.clone()))); + let default_software_version = expected_software_version.clone(); + di_defaults + .expect_software_version() + .returning(move || Ok(Some(default_software_version.clone()))); + let default_hardware_version = expected_hardware_version.clone(); + di_defaults + .expect_hardware_version() + .returning(move || Ok(default_hardware_version.clone())); + let default_device_id = expected_device_id.clone(); + di_defaults + .expect_device_id() + .returning(move || Ok(default_device_id.clone())); + + let r = DeviceInfo::parse(None, di_defaults).unwrap(); + + assert_eq!( + r.0.software_type, + Some(DeviceInfoValue::Default(expected_software_type)) + ); + assert_eq!( + r.0.software_version, + Some(DeviceInfoValue::Default(expected_software_version)) + ); + assert_eq!(r.0.hardware_version, expected_hardware_version); + assert_eq!(r.0.device_id, expected_device_id); + } +} diff --git a/memfaultd/src/config/mod.rs b/memfaultd/src/config/mod.rs new file mode 100644 index 0000000..c5c6e83 --- /dev/null +++ b/memfaultd/src/config/mod.rs @@ -0,0 +1,457 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::eyre; +use std::net::SocketAddr; +use std::time::Duration; +use std::{ + path::{Path, PathBuf}, + sync::{Arc, RwLock}, +}; + +use crate::{ + network::{NetworkClient, NetworkConfig}, + util::{DiskBacked, UnwrapOrDie, UpdateStatus}, +}; + +use crate::util::disk_size::DiskSize; + +#[cfg(test)] +pub use self::config_file::ConnectionCheckProtocol; + +#[cfg(target_os = "linux")] +pub use self::config_file::{CoredumpCaptureStrategy, CoredumpCompression}; + +use self::device_info::DeviceInfoValue; +#[cfg(test)] +pub use self::device_info::MockDeviceInfoDefaults; +pub use self::{ + config_file::{ + ConnectivityMonitorConfig, ConnectivityMonitorTarget, JsonConfigs, LogSource, + LogToMetricRule, MemfaultdConfig, SessionConfig, StorageConfig, SystemMetricConfig, + }, + device_config::{DeviceConfig, Resolution, Sampling}, + device_info::{DeviceInfo, DeviceInfoDefaultsImpl, DeviceInfoWarning}, +}; + +use crate::mar::MarEntryBuilder; +use crate::mar::Metadata; +use eyre::{Context, Result}; + +mod config_file; +mod device_config; +mod device_info; +mod utils; + +const FALLBACK_SOFTWARE_VERSION: &str = "0.0.0-memfault-unknown"; +const FALLBACK_SOFTWARE_TYPE: &str = "memfault-unknown"; + +/// Container of the entire memfaultd configuration. +/// Implement `From` trait to initialize module specific configuration (see `NetworkConfig` for example). +pub struct Config { + pub device_info: DeviceInfo, + pub config_file: MemfaultdConfig, + pub config_file_path: PathBuf, + cached_device_config: Arc>>, +} + +const LOGS_SUBDIRECTORY: &str = "logs"; +const MAR_STAGING_SUBDIRECTORY: &str = "mar"; +const DEVICE_CONFIG_FILE: &str = "device_config.json"; +const COREDUMP_RATE_LIMITER_FILENAME: &str = "coredump_rate_limit"; + +impl Config { + pub const DEFAULT_CONFIG_PATH: &'static str = "/etc/memfaultd.conf"; + + pub fn read_from_system(user_config: Option<&Path>) -> Result { + // Select config file to read + let config_file = user_config.unwrap_or_else(|| Path::new(Self::DEFAULT_CONFIG_PATH)); + + let config = MemfaultdConfig::load(config_file).wrap_err(eyre!( + "Unable to read config file {}", + &config_file.display() + ))?; + + let (device_info, warnings) = + DeviceInfo::load().wrap_err(eyre!("Unable to load device info"))?; + #[allow(clippy::print_stderr)] + warnings.iter().for_each(|w| eprintln!("{}", w)); + + let device_config = DiskBacked::from_path(&Self::device_config_path_from_config(&config)); + + Ok(Self { + device_info, + config_file: config, + config_file_path: config_file.to_owned(), + cached_device_config: Arc::new(RwLock::new(device_config)), + }) + } + + pub fn refresh_device_config(&self, client: &impl NetworkClient) -> Result { + let response = client.fetch_device_config()?; + + // Let the server know that we have applied the new version if it still + // believes we have an older one. + let confirm_version = match response.data.completed { + Some(v) if v == response.data.revision => None, + _ => Some(response.data.revision), + }; + + // Always write the config to our cache. + let new_config: DeviceConfig = response.into(); + let update_status = self + .cached_device_config + .write() + .unwrap_or_die() + .set(new_config)?; + + // After saving, create the device-config confirmation MAR entry + if let Some(revision) = confirm_version { + let mar_staging = self.mar_staging_path(); + MarEntryBuilder::new(&mar_staging)? + .set_metadata(Metadata::new_device_config(revision)) + .save(&NetworkConfig::from(self))?; + } + Ok(update_status) + } + + pub fn tmp_dir(&self) -> PathBuf { + match self.config_file.tmp_dir { + Some(ref tmp_dir) => tmp_dir.clone(), + None => self.config_file.persist_dir.clone(), + } + .into() + } + + pub fn tmp_dir_max_size(&self) -> DiskSize { + DiskSize::new_capacity(self.config_file.tmp_dir_max_usage as u64) + } + + pub fn tmp_dir_min_headroom(&self) -> DiskSize { + DiskSize { + bytes: self.config_file.tmp_dir_min_headroom as u64, + inodes: self.config_file.tmp_dir_min_inodes as u64, + } + } + + pub fn coredump_rate_limiter_file_path(&self) -> PathBuf { + self.tmp_dir().join(COREDUMP_RATE_LIMITER_FILENAME) + } + + pub fn logs_path(&self) -> PathBuf { + self.tmp_dir().join(LOGS_SUBDIRECTORY) + } + + pub fn mar_staging_path(&self) -> PathBuf { + self.tmp_dir().join(MAR_STAGING_SUBDIRECTORY) + } + + fn device_config_path_from_config(config_file: &MemfaultdConfig) -> PathBuf { + config_file.persist_dir.join(DEVICE_CONFIG_FILE) + } + pub fn device_config_path(&self) -> PathBuf { + Self::device_config_path_from_config(&self.config_file) + } + + pub fn sampling(&self) -> Sampling { + if self.config_file.enable_dev_mode { + Sampling::development() + } else { + self.device_config().sampling + } + } + + /// Returns the device_config at the time of the call. If the device_config is updated + /// after this call, the returned value will not be updated. + /// This can block for a small moment if another thread is currently updating the device_config. + fn device_config(&self) -> DeviceConfig { + self.cached_device_config + .read() + // If another thread crashed while holding the mutex we want to crash the program + .unwrap_or_die() + // If we were not able to load from local-cache then return the defaults. + .get() + .clone() + } + + /// Returns the software version for the device. + /// + /// The precedence is as follows: + /// 1. Configured software version in device_info + /// 2. Configured software version in config_file + /// 3. Default software version in device_info + /// 4. Fallback software version + pub fn software_version(&self) -> &str { + match ( + &self.device_info.software_version, + &self.config_file.software_version, + ) { + (Some(DeviceInfoValue::Configured(sw_version)), _) => sw_version.as_ref(), + (None, Some(sw_version)) => sw_version.as_ref(), + (Some(DeviceInfoValue::Default(_)), Some(sw_version)) => sw_version.as_ref(), + (Some(DeviceInfoValue::Default(sw_version)), None) => sw_version.as_ref(), + (None, None) => FALLBACK_SOFTWARE_VERSION, + } + } + + /// Returns the software type for the device. + /// + /// The precedence is as follows: + /// 1. Configured software type in device_info + /// 2. Configured software type in config_file + /// 3. Default software type in device_info + /// 4. Fallback software type + pub fn software_type(&self) -> &str { + match ( + &self.device_info.software_type, + &self.config_file.software_type, + ) { + (Some(DeviceInfoValue::Configured(software_type)), _) => software_type.as_ref(), + (None, Some(software_type)) => software_type.as_ref(), + (Some(DeviceInfoValue::Default(_)), Some(software_type)) => software_type.as_ref(), + (Some(DeviceInfoValue::Default(software_type)), None) => software_type.as_ref(), + (None, None) => FALLBACK_SOFTWARE_TYPE, + } + } + + pub fn mar_entry_max_age(&self) -> Duration { + self.config_file.mar.mar_entry_max_age + } + + pub fn battery_monitor_periodic_update_enabled(&self) -> bool { + self.config_file.battery_monitor.is_some() + } + + pub fn battery_monitor_battery_info_command(&self) -> &str { + match self.config_file.battery_monitor.as_ref() { + Some(battery_config) => battery_config.battery_info_command.as_ref(), + None => "", + } + } + + pub fn battery_monitor_interval(&self) -> Duration { + match self.config_file.battery_monitor.as_ref() { + Some(battery_config) => battery_config.interval_seconds, + None => Duration::from_secs(0), + } + } + + pub fn connectivity_monitor_config(&self) -> Option<&ConnectivityMonitorConfig> { + self.config_file.connectivity_monitor.as_ref() + } + + pub fn session_configs(&self) -> Option<&Vec> { + self.config_file.sessions.as_ref() + } + + pub fn statsd_server_enabled(&self) -> bool { + self.config_file.metrics.statsd_server.is_some() + } + + pub fn statsd_server_address(&self) -> Result { + match &self.config_file.metrics.statsd_server { + Some(statsd_server_config) => Ok(statsd_server_config.bind_address), + None => Err(eyre!("No StatsD server bind_address configured!")), + } + } + + pub fn builtin_system_metric_collection_enabled(&self) -> bool { + self.config_file.metrics.system_metric_collection.enable + } + + pub fn system_metric_poll_interval(&self) -> Duration { + self.config_file + .metrics + .system_metric_collection + .poll_interval_seconds + } + + pub fn system_metric_config(&self) -> SystemMetricConfig { + self.config_file.metrics.system_metric_collection.clone() + } +} + +#[cfg(test)] +impl Config { + pub fn test_fixture() -> Self { + Config { + device_info: DeviceInfo::test_fixture(), + config_file: MemfaultdConfig::test_fixture(), + config_file_path: PathBuf::from("test_fixture.conf"), + cached_device_config: Arc::new(RwLock::new(DiskBacked::from_path(&PathBuf::from( + "/dev/null", + )))), + } + } + + pub fn test_fixture_with_info_overrides(software_version: &str, software_type: &str) -> Self { + Config { + device_info: DeviceInfo::test_fixture_with_overrides(software_version, software_type), + config_file: MemfaultdConfig::test_fixture(), + config_file_path: PathBuf::from("test_fixture.conf"), + cached_device_config: Arc::new(RwLock::new(DiskBacked::from_path(&PathBuf::from( + "/dev/null", + )))), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use std::{fs::create_dir_all, path::PathBuf}; + + use rstest::{fixture, rstest}; + + use crate::{ + config::{device_info::DeviceInfoValue, Config}, + mar::MarEntry, + network::{ + DeviceConfigResponse, DeviceConfigResponseConfig, DeviceConfigResponseData, + DeviceConfigResponseResolution, MockNetworkClient, + }, + util::path::AbsolutePath, + }; + + #[test] + fn tmp_dir_defaults_to_persist_dir() { + let config = Config::test_fixture(); + + assert_eq!(config.tmp_dir(), config.config_file.persist_dir); + } + + #[test] + fn tmp_folder_set() { + let mut config = Config::test_fixture(); + let abs_path = PathBuf::from("/my/abs/path"); + config.config_file.tmp_dir = Some(AbsolutePath::try_from(abs_path.clone()).unwrap()); + + assert_eq!(config.tmp_dir(), abs_path); + } + + #[test] + fn test_info_overrides_file() { + let config = + Config::test_fixture_with_info_overrides("1.0.0-overridden", "overridden-type"); + + assert_eq!(config.software_version(), "1.0.0-overridden"); + assert_eq!(config.software_type(), "overridden-type"); + } + + #[rstest] + fn generate_mar_device_config_confirmation_when_needed(mut fixture: Fixture) { + fixture + .client + .expect_fetch_device_config() + .return_once(|| Ok(DEVICE_CONFIG_SAMPLE)); + fixture + .config + .refresh_device_config(&fixture.client) + .unwrap(); + + assert_eq!(fixture.count_mar_entries(), 1); + } + + #[rstest] + fn do_not_generate_mar_device_config_if_not_needed(mut fixture: Fixture) { + let mut device_config = DEVICE_CONFIG_SAMPLE; + device_config.data.completed = Some(device_config.data.revision); + + fixture + .client + .expect_fetch_device_config() + .return_once(move || Ok(device_config)); + fixture + .config + .refresh_device_config(&fixture.client) + .unwrap(); + + assert_eq!(fixture.count_mar_entries(), 0); + } + + #[rstest] + #[case(Some(DeviceInfoValue::Configured("1.0.0".into())), None, "1.0.0")] + #[case(Some(DeviceInfoValue::Default("1.0.0".into())), None, "1.0.0")] + #[case(Some(DeviceInfoValue::Configured("1.0.0".into())), Some("2.0.0"), "1.0.0")] + #[case(Some(DeviceInfoValue::Default("1.0.0".into())), Some("2.0.0"), "2.0.0")] + #[case(None, Some("2.0.0"), "2.0.0")] + #[case(None, None, FALLBACK_SOFTWARE_VERSION)] + fn software_version_precedence( + #[case] device_info_swv: Option, + #[case] config_swv: Option<&str>, + #[case] expected: &str, + ) { + let mut config = Config::test_fixture(); + config.device_info.software_version = device_info_swv; + config.config_file.software_version = config_swv.map(String::from); + + assert_eq!(config.software_version(), expected); + } + + #[rstest] + #[case(Some(DeviceInfoValue::Configured("test".into())), None, "test")] + #[case(Some(DeviceInfoValue::Default("test".into())), None, "test")] + #[case(Some(DeviceInfoValue::Configured("test".into())), Some("prod"), "test")] + #[case(Some(DeviceInfoValue::Default("test".into())), Some("prod"), "prod")] + #[case(None, Some("prod"), "prod")] + #[case(None, None, FALLBACK_SOFTWARE_TYPE)] + fn software_type_precedence( + #[case] device_info_swv: Option, + #[case] config_swv: Option<&str>, + #[case] expected: &str, + ) { + let mut config = Config::test_fixture(); + config.device_info.software_type = device_info_swv; + config.config_file.software_type = config_swv.map(String::from); + + assert_eq!(config.software_type(), expected); + } + + struct Fixture { + config: Config, + _tmp_dir: tempfile::TempDir, + client: MockNetworkClient, + } + + #[fixture] + fn fixture() -> Fixture { + Fixture::new() + } + + impl Fixture { + fn new() -> Self { + let tmp_dir = tempfile::tempdir().unwrap(); + let mut config = Config::test_fixture(); + config.config_file.persist_dir = tmp_dir.path().to_path_buf().try_into().unwrap(); + create_dir_all(config.mar_staging_path()).unwrap(); + Self { + config, + _tmp_dir: tmp_dir, + client: MockNetworkClient::new(), + } + } + + fn count_mar_entries(self) -> usize { + MarEntry::iterate_from_container(&self.config.mar_staging_path()) + .unwrap() + .count() + } + } + + const DEVICE_CONFIG_SAMPLE: DeviceConfigResponse = DeviceConfigResponse { + data: DeviceConfigResponseData { + completed: None, + revision: 42, + config: DeviceConfigResponseConfig { + memfault: crate::network::DeviceConfigResponseMemfault { + sampling: crate::network::DeviceConfigResponseSampling { + debugging_resolution: DeviceConfigResponseResolution::High, + logging_resolution: DeviceConfigResponseResolution::High, + monitoring_resolution: DeviceConfigResponseResolution::High, + }, + }, + }, + }, + }; +} diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__empty_object.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__empty_object.snap new file mode 100644 index 0000000..e207028 --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__empty_object.snap @@ -0,0 +1,73 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: content +--- +{ + "persist_dir": "/media/memfault", + "tmp_dir": null, + "tmp_dir_min_headroom_kib": 10240, + "tmp_dir_min_inodes": 100, + "tmp_dir_max_usage_kib": 102400, + "upload_interval_seconds": 3600, + "heartbeat_interval_seconds": 3600, + "enable_data_collection": false, + "enable_dev_mode": false, + "software_version": null, + "software_type": null, + "project_key": "", + "base_url": "https://device.memfault.com", + "swupdate": { + "input_file": "/etc/swupdate.cfg", + "output_file": "/tmp/swupdate.cfg" + }, + "reboot": { + "last_reboot_reason_file": "/media/last_reboot_reason" + }, + "coredump": { + "compression": "gzip", + "coredump_max_size_kib": 96000, + "rate_limit_count": 5, + "rate_limit_duration_seconds": 3600, + "capture_strategy": { + "type": "threads", + "max_thread_size_kib": 32 + }, + "log_lines": 100 + }, + "fluent-bit": { + "extra_fluentd_attributes": [], + "bind_address": "127.0.0.1:5170", + "max_buffered_lines": 1000, + "max_connections": 4 + }, + "logs": { + "rotate_size_kib": 10240, + "rotate_after_seconds": 3600, + "compression_level": 1, + "max_lines_per_minute": 500, + "log_to_metrics": null, + "storage": "persist", + "source": "fluent-bit" + }, + "mar": { + "mar_file_max_size_kib": 10240, + "mar_entry_max_age_seconds": 604800 + }, + "http_server": { + "bind_address": "127.0.0.1:8787" + }, + "battery_monitor": null, + "connectivity_monitor": null, + "sessions": null, + "metrics": { + "enable_daily_heartbeats": false, + "system_metric_collection": { + "enable": false, + "poll_interval_seconds": 10, + "processes": null, + "disk_space": null, + "network_interfaces": null + }, + "statsd_server": null + } +} diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__metrics_config.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__metrics_config.snap new file mode 100644 index 0000000..96e7659 --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__metrics_config.snap @@ -0,0 +1,79 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: content +--- +{ + "persist_dir": "/media/memfault", + "tmp_dir": null, + "tmp_dir_min_headroom_kib": 10240, + "tmp_dir_min_inodes": 100, + "tmp_dir_max_usage_kib": 102400, + "upload_interval_seconds": 3600, + "heartbeat_interval_seconds": 3600, + "enable_data_collection": false, + "enable_dev_mode": false, + "software_version": null, + "software_type": null, + "project_key": "", + "base_url": "https://device.memfault.com", + "swupdate": { + "input_file": "/etc/swupdate.cfg", + "output_file": "/tmp/swupdate.cfg" + }, + "reboot": { + "last_reboot_reason_file": "/media/last_reboot_reason" + }, + "coredump": { + "compression": "gzip", + "coredump_max_size_kib": 96000, + "rate_limit_count": 5, + "rate_limit_duration_seconds": 3600, + "capture_strategy": { + "type": "threads", + "max_thread_size_kib": 32 + }, + "log_lines": 100 + }, + "fluent-bit": { + "extra_fluentd_attributes": [], + "bind_address": "127.0.0.1:5170", + "max_buffered_lines": 1000, + "max_connections": 4 + }, + "logs": { + "rotate_size_kib": 10240, + "rotate_after_seconds": 3600, + "compression_level": 1, + "max_lines_per_minute": 500, + "log_to_metrics": null, + "storage": "persist", + "source": "fluent-bit" + }, + "mar": { + "mar_file_max_size_kib": 10240, + "mar_entry_max_age_seconds": 604800 + }, + "http_server": { + "bind_address": "127.0.0.1:8787" + }, + "battery_monitor": null, + "connectivity_monitor": null, + "sessions": null, + "metrics": { + "enable_daily_heartbeats": false, + "system_metric_collection": { + "enable": false, + "poll_interval_seconds": 10, + "processes": [ + "wefaultd" + ], + "disk_space": [ + "/dev/sda1" + ], + "network_interfaces": [ + "eth1" + ] + }, + "statsd_server": null + } +} diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__set_and_write_bool_to_runtime_config@empty_object.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__set_and_write_bool_to_runtime_config@empty_object.snap new file mode 100644 index 0000000..c350b14 --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__set_and_write_bool_to_runtime_config@empty_object.snap @@ -0,0 +1,5 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: disk_config_string +--- +"{\"key\":true}" diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__set_and_write_bool_to_runtime_config@no_file.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__set_and_write_bool_to_runtime_config@no_file.snap new file mode 100644 index 0000000..c350b14 --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__set_and_write_bool_to_runtime_config@no_file.snap @@ -0,0 +1,5 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: disk_config_string +--- +"{\"key\":true}" diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__set_and_write_bool_to_runtime_config@other_key.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__set_and_write_bool_to_runtime_config@other_key.snap new file mode 100644 index 0000000..f7a8775 --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__set_and_write_bool_to_runtime_config@other_key.snap @@ -0,0 +1,5 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: disk_config_string +--- +"{\"key\":true,\"key2\":false}" diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_connectivity_monitor.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_connectivity_monitor.snap new file mode 100644 index 0000000..9fda658 --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_connectivity_monitor.snap @@ -0,0 +1,83 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: content +--- +{ + "persist_dir": "/media/memfault", + "tmp_dir": null, + "tmp_dir_min_headroom_kib": 10240, + "tmp_dir_min_inodes": 100, + "tmp_dir_max_usage_kib": 102400, + "upload_interval_seconds": 3600, + "heartbeat_interval_seconds": 3600, + "enable_data_collection": false, + "enable_dev_mode": false, + "software_version": null, + "software_type": null, + "project_key": "", + "base_url": "https://device.memfault.com", + "swupdate": { + "input_file": "/etc/swupdate.cfg", + "output_file": "/tmp/swupdate.cfg" + }, + "reboot": { + "last_reboot_reason_file": "/media/last_reboot_reason" + }, + "coredump": { + "compression": "gzip", + "coredump_max_size_kib": 96000, + "rate_limit_count": 5, + "rate_limit_duration_seconds": 3600, + "capture_strategy": { + "type": "threads", + "max_thread_size_kib": 32 + }, + "log_lines": 100 + }, + "fluent-bit": { + "extra_fluentd_attributes": [], + "bind_address": "127.0.0.1:5170", + "max_buffered_lines": 1000, + "max_connections": 4 + }, + "logs": { + "rotate_size_kib": 10240, + "rotate_after_seconds": 3600, + "compression_level": 1, + "max_lines_per_minute": 500, + "log_to_metrics": null, + "storage": "persist", + "source": "fluent-bit" + }, + "mar": { + "mar_file_max_size_kib": 10240, + "mar_entry_max_age_seconds": 604800 + }, + "http_server": { + "bind_address": "127.0.0.1:8787" + }, + "battery_monitor": null, + "connectivity_monitor": { + "interval_seconds": 30, + "targets": [ + { + "protocol": "tcp", + "host": "8.8.8.8", + "port": 443 + } + ], + "timeout_seconds": 10 + }, + "sessions": null, + "metrics": { + "enable_daily_heartbeats": false, + "system_metric_collection": { + "enable": false, + "poll_interval_seconds": 10, + "processes": null, + "disk_space": null, + "network_interfaces": null + }, + "statsd_server": null + } +} diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_coredump_capture_strategy_threads.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_coredump_capture_strategy_threads.snap new file mode 100644 index 0000000..e207028 --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_coredump_capture_strategy_threads.snap @@ -0,0 +1,73 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: content +--- +{ + "persist_dir": "/media/memfault", + "tmp_dir": null, + "tmp_dir_min_headroom_kib": 10240, + "tmp_dir_min_inodes": 100, + "tmp_dir_max_usage_kib": 102400, + "upload_interval_seconds": 3600, + "heartbeat_interval_seconds": 3600, + "enable_data_collection": false, + "enable_dev_mode": false, + "software_version": null, + "software_type": null, + "project_key": "", + "base_url": "https://device.memfault.com", + "swupdate": { + "input_file": "/etc/swupdate.cfg", + "output_file": "/tmp/swupdate.cfg" + }, + "reboot": { + "last_reboot_reason_file": "/media/last_reboot_reason" + }, + "coredump": { + "compression": "gzip", + "coredump_max_size_kib": 96000, + "rate_limit_count": 5, + "rate_limit_duration_seconds": 3600, + "capture_strategy": { + "type": "threads", + "max_thread_size_kib": 32 + }, + "log_lines": 100 + }, + "fluent-bit": { + "extra_fluentd_attributes": [], + "bind_address": "127.0.0.1:5170", + "max_buffered_lines": 1000, + "max_connections": 4 + }, + "logs": { + "rotate_size_kib": 10240, + "rotate_after_seconds": 3600, + "compression_level": 1, + "max_lines_per_minute": 500, + "log_to_metrics": null, + "storage": "persist", + "source": "fluent-bit" + }, + "mar": { + "mar_file_max_size_kib": 10240, + "mar_entry_max_age_seconds": 604800 + }, + "http_server": { + "bind_address": "127.0.0.1:8787" + }, + "battery_monitor": null, + "connectivity_monitor": null, + "sessions": null, + "metrics": { + "enable_daily_heartbeats": false, + "system_metric_collection": { + "enable": false, + "poll_interval_seconds": 10, + "processes": null, + "disk_space": null, + "network_interfaces": null + }, + "statsd_server": null + } +} diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_log_to_metrics_rules.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_log_to_metrics_rules.snap new file mode 100644 index 0000000..286b22e --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_log_to_metrics_rules.snap @@ -0,0 +1,90 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: content +--- +{ + "persist_dir": "/media/memfault", + "tmp_dir": null, + "tmp_dir_min_headroom_kib": 10240, + "tmp_dir_min_inodes": 100, + "tmp_dir_max_usage_kib": 102400, + "upload_interval_seconds": 3600, + "heartbeat_interval_seconds": 3600, + "enable_data_collection": false, + "enable_dev_mode": false, + "software_version": null, + "software_type": null, + "project_key": "", + "base_url": "https://device.memfault.com", + "swupdate": { + "input_file": "/etc/swupdate.cfg", + "output_file": "/tmp/swupdate.cfg" + }, + "reboot": { + "last_reboot_reason_file": "/media/last_reboot_reason" + }, + "coredump": { + "compression": "gzip", + "coredump_max_size_kib": 96000, + "rate_limit_count": 5, + "rate_limit_duration_seconds": 3600, + "capture_strategy": { + "type": "threads", + "max_thread_size_kib": 32 + }, + "log_lines": 100 + }, + "fluent-bit": { + "extra_fluentd_attributes": [], + "bind_address": "127.0.0.1:5170", + "max_buffered_lines": 1000, + "max_connections": 4 + }, + "logs": { + "rotate_size_kib": 10240, + "rotate_after_seconds": 3600, + "compression_level": 1, + "max_lines_per_minute": 500, + "log_to_metrics": { + "rules": [ + { + "type": "count_matching", + "pattern": "ssh", + "metric_name": "ssh_logins", + "filter": {} + }, + { + "type": "count_matching", + "pattern": "ssh", + "metric_name": "ssh_logins", + "filter": { + "_SYSTEMD_UNIT": "sshd.service" + } + } + ] + }, + "storage": "persist", + "source": "fluent-bit" + }, + "mar": { + "mar_file_max_size_kib": 10240, + "mar_entry_max_age_seconds": 604800 + }, + "http_server": { + "bind_address": "127.0.0.1:8787" + }, + "battery_monitor": null, + "connectivity_monitor": null, + "sessions": null, + "metrics": { + "enable_daily_heartbeats": false, + "system_metric_collection": { + "enable": false, + "poll_interval_seconds": 10, + "processes": null, + "disk_space": null, + "network_interfaces": null + }, + "statsd_server": null + } +} diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_partial_logs.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_partial_logs.snap new file mode 100644 index 0000000..e207028 --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_partial_logs.snap @@ -0,0 +1,73 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: content +--- +{ + "persist_dir": "/media/memfault", + "tmp_dir": null, + "tmp_dir_min_headroom_kib": 10240, + "tmp_dir_min_inodes": 100, + "tmp_dir_max_usage_kib": 102400, + "upload_interval_seconds": 3600, + "heartbeat_interval_seconds": 3600, + "enable_data_collection": false, + "enable_dev_mode": false, + "software_version": null, + "software_type": null, + "project_key": "", + "base_url": "https://device.memfault.com", + "swupdate": { + "input_file": "/etc/swupdate.cfg", + "output_file": "/tmp/swupdate.cfg" + }, + "reboot": { + "last_reboot_reason_file": "/media/last_reboot_reason" + }, + "coredump": { + "compression": "gzip", + "coredump_max_size_kib": 96000, + "rate_limit_count": 5, + "rate_limit_duration_seconds": 3600, + "capture_strategy": { + "type": "threads", + "max_thread_size_kib": 32 + }, + "log_lines": 100 + }, + "fluent-bit": { + "extra_fluentd_attributes": [], + "bind_address": "127.0.0.1:5170", + "max_buffered_lines": 1000, + "max_connections": 4 + }, + "logs": { + "rotate_size_kib": 10240, + "rotate_after_seconds": 3600, + "compression_level": 1, + "max_lines_per_minute": 500, + "log_to_metrics": null, + "storage": "persist", + "source": "fluent-bit" + }, + "mar": { + "mar_file_max_size_kib": 10240, + "mar_entry_max_age_seconds": 604800 + }, + "http_server": { + "bind_address": "127.0.0.1:8787" + }, + "battery_monitor": null, + "connectivity_monitor": null, + "sessions": null, + "metrics": { + "enable_daily_heartbeats": false, + "system_metric_collection": { + "enable": false, + "poll_interval_seconds": 10, + "processes": null, + "disk_space": null, + "network_interfaces": null + }, + "statsd_server": null + } +} diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_sessions.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_sessions.snap new file mode 100644 index 0000000..898a424 --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_sessions.snap @@ -0,0 +1,81 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: content +--- +{ + "persist_dir": "/media/memfault", + "tmp_dir": null, + "tmp_dir_min_headroom_kib": 10240, + "tmp_dir_min_inodes": 100, + "tmp_dir_max_usage_kib": 102400, + "upload_interval_seconds": 3600, + "heartbeat_interval_seconds": 3600, + "enable_data_collection": false, + "enable_dev_mode": false, + "software_version": null, + "software_type": null, + "project_key": "", + "base_url": "https://device.memfault.com", + "swupdate": { + "input_file": "/etc/swupdate.cfg", + "output_file": "/tmp/swupdate.cfg" + }, + "reboot": { + "last_reboot_reason_file": "/media/last_reboot_reason" + }, + "coredump": { + "compression": "gzip", + "coredump_max_size_kib": 96000, + "rate_limit_count": 5, + "rate_limit_duration_seconds": 3600, + "capture_strategy": { + "type": "threads", + "max_thread_size_kib": 32 + }, + "log_lines": 100 + }, + "fluent-bit": { + "extra_fluentd_attributes": [], + "bind_address": "127.0.0.1:5170", + "max_buffered_lines": 1000, + "max_connections": 4 + }, + "logs": { + "rotate_size_kib": 10240, + "rotate_after_seconds": 3600, + "compression_level": 1, + "max_lines_per_minute": 500, + "log_to_metrics": null, + "storage": "persist", + "source": "fluent-bit" + }, + "mar": { + "mar_file_max_size_kib": 10240, + "mar_entry_max_age_seconds": 604800 + }, + "http_server": { + "bind_address": "127.0.0.1:8787" + }, + "battery_monitor": null, + "connectivity_monitor": null, + "sessions": [ + { + "name": "test-session", + "captured_metrics": [ + "cpu.total", + "statsd.wefaultd.blendpower" + ] + } + ], + "metrics": { + "enable_daily_heartbeats": false, + "system_metric_collection": { + "enable": false, + "poll_interval_seconds": 10, + "processes": null, + "disk_space": null, + "network_interfaces": null + }, + "statsd_server": null + } +} diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__without_coredump_compression.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__without_coredump_compression.snap new file mode 100644 index 0000000..c22b015 --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__without_coredump_compression.snap @@ -0,0 +1,73 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: content +--- +{ + "persist_dir": "/media/memfault", + "tmp_dir": null, + "tmp_dir_min_headroom_kib": 10240, + "tmp_dir_min_inodes": 100, + "tmp_dir_max_usage_kib": 102400, + "upload_interval_seconds": 3600, + "heartbeat_interval_seconds": 3600, + "enable_data_collection": false, + "enable_dev_mode": false, + "software_version": null, + "software_type": null, + "project_key": "", + "base_url": "https://device.memfault.com", + "swupdate": { + "input_file": "/etc/swupdate.cfg", + "output_file": "/tmp/swupdate.cfg" + }, + "reboot": { + "last_reboot_reason_file": "/media/last_reboot_reason" + }, + "coredump": { + "compression": "none", + "coredump_max_size_kib": 96000, + "rate_limit_count": 5, + "rate_limit_duration_seconds": 3600, + "capture_strategy": { + "type": "threads", + "max_thread_size_kib": 32 + }, + "log_lines": 100 + }, + "fluent-bit": { + "extra_fluentd_attributes": [], + "bind_address": "127.0.0.1:5170", + "max_buffered_lines": 1000, + "max_connections": 4 + }, + "logs": { + "rotate_size_kib": 10240, + "rotate_after_seconds": 3600, + "compression_level": 1, + "max_lines_per_minute": 500, + "log_to_metrics": null, + "storage": "persist", + "source": "fluent-bit" + }, + "mar": { + "mar_file_max_size_kib": 10240, + "mar_entry_max_age_seconds": 604800 + }, + "http_server": { + "bind_address": "127.0.0.1:8787" + }, + "battery_monitor": null, + "connectivity_monitor": null, + "sessions": null, + "metrics": { + "enable_daily_heartbeats": false, + "system_metric_collection": { + "enable": false, + "poll_interval_seconds": 10, + "processes": null, + "disk_space": null, + "network_interfaces": null + }, + "statsd_server": null + } +} diff --git a/memfaultd/src/config/test-config/empty_object.json b/memfaultd/src/config/test-config/empty_object.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/memfaultd/src/config/test-config/empty_object.json @@ -0,0 +1 @@ +{} diff --git a/memfaultd/src/config/test-config/metrics_config.json b/memfaultd/src/config/test-config/metrics_config.json new file mode 100644 index 0000000..b3f1ad7 --- /dev/null +++ b/memfaultd/src/config/test-config/metrics_config.json @@ -0,0 +1,12 @@ +{ + "metrics": { + "enable_daily_heartbeats": false, + "system_metric_collection": { + "enable": false, + "poll_interval_seconds": 10, + "processes": ["wefaultd"], + "disk_space": ["/dev/sda1"], + "network_interfaces": ["eth1"] + } + } +} diff --git a/memfaultd/src/config/test-config/with_connectivity_monitor.json b/memfaultd/src/config/test-config/with_connectivity_monitor.json new file mode 100644 index 0000000..620a03d --- /dev/null +++ b/memfaultd/src/config/test-config/with_connectivity_monitor.json @@ -0,0 +1,13 @@ +{ + "connectivity_monitor": { + "interval_seconds": 30, + "timeout_seconds": 10, + "targets": [ + { + "host": "8.8.8.8", + "port": 443, + "protocol": "tcp" + } + ] + } +} diff --git a/memfaultd/src/config/test-config/with_coredump_capture_strategy_threads.json b/memfaultd/src/config/test-config/with_coredump_capture_strategy_threads.json new file mode 100644 index 0000000..6167d96 --- /dev/null +++ b/memfaultd/src/config/test-config/with_coredump_capture_strategy_threads.json @@ -0,0 +1,8 @@ +{ + "coredump": { + "capture_strategy": { + "type": "threads", + "max_thread_size_kib": 32 + } + } +} diff --git a/memfaultd/src/config/test-config/with_invalid_path.json b/memfaultd/src/config/test-config/with_invalid_path.json new file mode 100644 index 0000000..e7fc77c --- /dev/null +++ b/memfaultd/src/config/test-config/with_invalid_path.json @@ -0,0 +1,3 @@ +{ + "tmp_dir": "tmp" +} diff --git a/memfaultd/src/config/test-config/with_invalid_swt_swv.json b/memfaultd/src/config/test-config/with_invalid_swt_swv.json new file mode 100644 index 0000000..41d0b3f --- /dev/null +++ b/memfaultd/src/config/test-config/with_invalid_swt_swv.json @@ -0,0 +1,4 @@ +{ + "software_version": "1.0.0?", + "software_type": "test program" +} diff --git a/memfaultd/src/config/test-config/with_log_to_metrics_rules.json b/memfaultd/src/config/test-config/with_log_to_metrics_rules.json new file mode 100644 index 0000000..f590b8d --- /dev/null +++ b/memfaultd/src/config/test-config/with_log_to_metrics_rules.json @@ -0,0 +1,21 @@ +{ + "logs": { + "log_to_metrics": { + "rules": [ + { + "type": "count_matching", + "pattern": "ssh", + "metric_name": "ssh_logins" + }, + { + "type": "count_matching", + "pattern": "ssh", + "metric_name": "ssh_logins", + "filter": { + "_SYSTEMD_UNIT": "sshd.service" + } + } + ] + } + } +} diff --git a/memfaultd/src/config/test-config/with_partial_logs.json b/memfaultd/src/config/test-config/with_partial_logs.json new file mode 100644 index 0000000..7302b45 --- /dev/null +++ b/memfaultd/src/config/test-config/with_partial_logs.json @@ -0,0 +1,5 @@ +{ + "logs": { + "bind_address": "0.0.0.0:4242" + } +} diff --git a/memfaultd/src/config/test-config/with_sessions.json b/memfaultd/src/config/test-config/with_sessions.json new file mode 100644 index 0000000..8599462 --- /dev/null +++ b/memfaultd/src/config/test-config/with_sessions.json @@ -0,0 +1,8 @@ +{ + "sessions": [ + { + "name": "test-session", + "captured_metrics": ["cpu.total", "statsd.wefaultd.blendpower"] + } + ] +} diff --git a/memfaultd/src/config/test-config/with_sessions_invalid_metric_name.json b/memfaultd/src/config/test-config/with_sessions_invalid_metric_name.json new file mode 100644 index 0000000..849a219 --- /dev/null +++ b/memfaultd/src/config/test-config/with_sessions_invalid_metric_name.json @@ -0,0 +1,11 @@ +{ + "sessions": [ + { + "name": "test-session", + "captured_metrics": [ + "string-with-non-ascii-\u{1F4A9}", + "statsd.wefaultd.blendpower" + ] + } + ] +} diff --git a/memfaultd/src/config/test-config/with_sessions_invalid_session_name.json b/memfaultd/src/config/test-config/with_sessions_invalid_session_name.json new file mode 100644 index 0000000..e6c0a72 --- /dev/null +++ b/memfaultd/src/config/test-config/with_sessions_invalid_session_name.json @@ -0,0 +1,8 @@ +{ + "sessions": [ + { + "name": "123-test-session", + "captured_metrics": ["cpu.total", "statsd.wefaultd.blendpower"] + } + ] +} diff --git a/memfaultd/src/config/test-config/without_coredump_compression.json b/memfaultd/src/config/test-config/without_coredump_compression.json new file mode 100644 index 0000000..03e6be4 --- /dev/null +++ b/memfaultd/src/config/test-config/without_coredump_compression.json @@ -0,0 +1,5 @@ +{ + "coredump": { + "compression": "none" + } +} diff --git a/memfaultd/src/config/utils.rs b/memfaultd/src/config/utils.rs new file mode 100644 index 0000000..0d371b0 --- /dev/null +++ b/memfaultd/src/config/utils.rs @@ -0,0 +1,48 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::util::patterns::{ + alphanum_slug_dots_colon_is_valid, alphanum_slug_dots_colon_spaces_parens_slash_is_valid, + alphanum_slug_is_valid, +}; + +pub fn software_type_is_valid(s: &str) -> eyre::Result<()> { + alphanum_slug_dots_colon_is_valid(s, 128) +} + +pub fn software_version_is_valid(s: &str) -> eyre::Result<()> { + alphanum_slug_dots_colon_spaces_parens_slash_is_valid(s, 128) +} + +pub fn hardware_version_is_valid(s: &str) -> eyre::Result<()> { + alphanum_slug_dots_colon_is_valid(s, 128) +} + +pub fn device_id_is_valid(id: &str) -> eyre::Result<()> { + alphanum_slug_is_valid(id, 128) +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + #[rstest] + // Minimum 1 character + #[case("A", true)] + // Allowed characters + #[case( + "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyz_-", + true + )] + // Disallowed characters + #[case("DEMO.1234", false)] + #[case("DEMO 1234", false)] + // Too short (0 characters) + #[case("", false)] + // Too long (129 characters) + #[case("012345679012345679012345679012345679012345679012345679012345679012345679012345679012345679012345679012345679012345678901234567890", false)] + fn device_id_is_valid_works(#[case] device_id: &str, #[case] expected: bool) { + assert_eq!(device_id_is_valid(device_id).is_ok(), expected); + } +} diff --git a/memfaultd/src/coredump/mod.rs b/memfaultd/src/coredump/mod.rs new file mode 100644 index 0000000..e57d58d --- /dev/null +++ b/memfaultd/src/coredump/mod.rs @@ -0,0 +1,30 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::Result; +use std::path::Path; + +#[cfg_attr(not(target_os = "linux"), allow(unused_variables))] +pub fn coredump_configure_kernel(config_path: &Path) -> Result<()> { + #[cfg(target_os = "linux")] + { + use eyre::Context; + use std::fs::write; + + write( + "/proc/sys/kernel/core_pattern", + format!( + "|/usr/sbin/memfault-core-handler -c {} %P", + config_path.display() + ), + ) + .wrap_err("Unable to write coredump pattern") + } + #[cfg(not(target_os = "linux"))] + { + use log::warn; + + warn!("Skipping coredump setting on non-Linux systems."); + Ok(()) + } +} diff --git a/memfaultd/src/fluent_bit/decode_time.rs b/memfaultd/src/fluent_bit/decode_time.rs new file mode 100644 index 0000000..70da2e9 --- /dev/null +++ b/memfaultd/src/fluent_bit/decode_time.rs @@ -0,0 +1,233 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! This module implements a custom deserializer for FluentBit timestamps. +//! +//! From +//! +//! > EventTime uses msgpack extension format of type 0 to carry nanosecond precision of time. +//! > +//! > Client MAY send EventTime instead of plain integer representation of second since unix epoch. +//! > Server SHOULD accept both formats of integer and EventTime. +//! > Binary representation of EventTime may be fixext or ext(with length 8). + +use core::fmt; +use std::{collections::HashMap, fmt::Formatter}; + +use chrono::{DateTime, TimeZone, Utc}; +use serde::{ + de::{Error, Visitor}, + Deserialize, Deserializer, Serialize, Serializer, +}; +use serde_bytes::ByteBuf; + +/// Deserialize a FluentBit time which can be a u32 timestamp or an EventTime +pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let tv = TimeVisitor {}; + deserializer.deserialize_any(tv) +} + +/// Serialize a FluentBit time as an EventTime +pub fn serialize(time: &DateTime, serializer: S) -> Result +where + S: Serializer, +{ + // From https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#eventtime-ext-format + let mut buf = ByteBuf::with_capacity(8); + buf.extend_from_slice(&i32::to_be_bytes(time.timestamp() as i32)); + buf.extend_from_slice(&i32::to_be_bytes(time.timestamp_subsec_nanos() as i32)); + + FluentdTimeExtType((FLUENTD_TIME_EXT_TYPE, buf)).serialize(serializer) +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename = "_ExtStruct")] +/// This is how Msgpack Ext type is represented by rmp_serde: +/// See: https://docs.rs/rmp-serde/latest/rmp_serde/constant.MSGPACK_EXT_STRUCT_NAME.html +/// And https://docs.racket-lang.org/msgpack/index.html#%28part._.Message.Pack_extension_type%29 +struct FluentdTimeExtType((i8, ByteBuf)); +const FLUENTD_TIME_EXT_TYPE: i8 = 0; + +/// Visit a FluentBit time which can be a u32 timestamp or an EventTime +/// (extended type with nanoseconds precision). +struct TimeVisitor {} + +impl<'de> Visitor<'de> for TimeVisitor { + type Value = DateTime; + + fn expecting(&self, formatter: &mut Formatter) -> fmt::Result { + write!(formatter, "an integer or a Ext/FixExt with length 8") + } + + // Called when the time is provided as an unsigned 32 bit value. + fn visit_u32(self, v: u32) -> Result + where + E: serde::de::Error, + { + Utc.timestamp_opt(v as i64, 0) + .single() + .ok_or_else(|| Error::custom("Invalid timestamp")) + } + + // Called when the time is provided as an EventTime. + fn visit_newtype_struct(self, deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_any(EventTimeVisitor {}) + } + + // Since fluent-bit 2.1, the timestamp is provided in a seq followed by a map of optional metadata. + // See https://github.com/fluent/fluent-bit/issues/6666 + // Before: + // (ExtType(code=0, data=b'd\xc0\xee\x7f7a\xe4\x89'), {'rand_value': 13625873794586244841}) + // After: + // ((ExtType(code=0, data=b'd\xc0\xeeT79\xc5g'), {}), {'rand_value': 1235066654796201019}) + // + // We currently just ignore the map of metadata. + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let a: FluentdTimeExtType = seq.next_element()?.ok_or_else(|| { + Error::custom("Invalid timestamp - expected an extension type with tag 0") + })?; + if a.0 .0 != FLUENTD_TIME_EXT_TYPE { + return Err(Error::custom("Invalid timestamp tag")); + } + + let ts = bytes_to_timestamp(a.0 .1)?; + let _ignored_metadata = seq.next_element::>()?; + Ok(ts) + } +} + +/// Visit a FluentBit EventTime (an extended type with nanosecond precision). +struct EventTimeVisitor {} + +impl<'de> Visitor<'de> for EventTimeVisitor { + type Value = DateTime; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(formatter, " a Ext/FixExt with length 8") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let tag = seq.next_element::()?; + let buf = seq.next_element::()?; + + // Validate the tag value is 0 for a timestamp. + match (tag, buf) { + (Some(FLUENTD_TIME_EXT_TYPE), Some(bytes)) => bytes_to_timestamp(bytes), + (Some(tag), _) => Err(serde::de::Error::custom(format!( + "Invalid tag {} - expected 0.", + tag + ))), + _ => Err(serde::de::Error::custom("Invalid event tag.")), + } + } +} + +/// Convert a byte buffer to a timestamp. +fn bytes_to_timestamp(bytes: ByteBuf) -> Result, E> +where + E: serde::de::Error, +{ + if bytes.len() == 8 { + // We verified that bytes is long enough so bytes[0..4] will + // never fail. It will return a [u8] of length 4. + // We still need `.try_into()` to convert [u8] into [u8; 4] + // because the compiler cannot verify that the length is 4 at + // compile time. #failproof™ + let seconds_bytes: [u8; 4] = bytes[0..4].try_into().unwrap(); + let nanoseconds_bytes: [u8; 4] = bytes[4..].try_into().unwrap(); + Utc.timestamp_opt( + u32::from_be_bytes(seconds_bytes) as i64, + u32::from_be_bytes(nanoseconds_bytes), + ) + .single() + .ok_or_else(|| Error::custom("Invalid timestamp")) + } else { + Err(E::custom("Invalid buffer length.")) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::deserialize; + + // This test make sure we are able to deserialize the three documentated + // variants of time encoding (the third argument specifies the variant to + // use). + #[rstest] + #[case(0, 0, serialize_fixext8)] + #[case(1675709515, 276*1_000_000, serialize_fixext8)] + #[case(1675709515, 276*1_000_000, serialize_varext8)] + #[case(1675709515, 0, serialize_integer)] + fn decode_encoded_time( + #[case] seconds: i32, + #[case] nanoseconds: i32, + #[case] serialize: fn(i32, i32) -> Vec, + ) { + let buf = serialize(seconds, nanoseconds); + let mut deserializer = rmp_serde::Deserializer::new(&buf[..]); + let t = deserialize(&mut deserializer).expect("should be deserializable"); + + assert_eq!(t.timestamp(), seconds as i64); + assert_eq!( + t.timestamp_nanos() - t.timestamp() * 1_000_000_000, + nanoseconds as i64 + ); + } + + #[rstest] + fn decode_ext_buffer_too_small() { + let buf = serialize_fixext8(1675709515, 0); + let mut deserializer = rmp_serde::Deserializer::new(&buf[..(buf.len() - 2)]); + + let e = deserialize(&mut deserializer).err().unwrap(); + assert!(e.to_string().contains("unexpected end of file"),); + } + + #[rstest] + fn decode_ext_invalid_tag() { + let mut buf = serialize_fixext8(1675709515, 0); + buf[1] = 0x42; + let mut deserializer = rmp_serde::Deserializer::new(&buf[..]); + + let e = deserialize(&mut deserializer).err().unwrap(); + assert!(e.to_string().contains("Invalid tag"),); + } + + fn serialize_fixext8(seconds: i32, nanoseconds: i32) -> Vec { + // From https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#eventtime-ext-format + let mut buf = vec![0xd7, 0x00]; + buf.extend_from_slice(&i32::to_be_bytes(seconds)); + buf.extend_from_slice(&i32::to_be_bytes(nanoseconds)); + buf + } + + fn serialize_varext8(seconds: i32, nanoseconds: i32) -> Vec { + // From https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#eventtime-ext-format + let mut buf = vec![0xC7, 0x08, 0x00]; + buf.extend_from_slice(&i32::to_be_bytes(seconds)); + buf.extend_from_slice(&i32::to_be_bytes(nanoseconds)); + buf + } + + fn serialize_integer(seconds: i32, _nanoseconds: i32) -> Vec { + // Fluentd spec says we should support time encoded as an integer + // https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#eventtime-ext-format + // Integers look like this: https://github.com/msgpack/msgpack/blob/master/spec.md#int-format-family + let mut buf = vec![0xCE]; + buf.extend_from_slice(&i32::to_be_bytes(seconds)); + buf + } +} diff --git a/memfaultd/src/fluent_bit/mod.rs b/memfaultd/src/fluent_bit/mod.rs new file mode 100644 index 0000000..d636089 --- /dev/null +++ b/memfaultd/src/fluent_bit/mod.rs @@ -0,0 +1,380 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! fluent-bit +//! +//! Provides FluentBitConnectionHandler to handle to TCP connections from +//! fluent-bit. A threadpool is used to limit the number of active connections at +//! a given time. +//! +//! The start() function returns a multi-producer single-consumer channel in +//! which the messages will be delivered. +//! +//! Messages are deserialized into FluentdMessage instances. +//! +//! We set a limit on the number of messages in the channel. If messages are not +//! consumed, the FluentBitReceiver will start to apply backpressure on +//! fluent-bitbit server. +//! +use std::net::TcpStream; +use std::sync::mpsc::{Receiver, SyncSender}; +use std::{collections::HashMap, net::SocketAddr}; + +use chrono::{DateTime, Utc}; +use eyre::Result; +use log::warn; +use rmp_serde::Deserializer; +use serde::{Deserialize, Serialize}; + +use crate::{config::Config, logs::log_entry::LogEntry}; +use crate::{ + logs::log_entry::LogValue, + util::tcp_server::{TcpConnectionHandler, TcpNullConnectionHandler, ThreadedTcpServer}, +}; + +mod decode_time; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum FluentdValue { + String(String), + Float(f64), +} + +impl From for LogValue { + fn from(value: FluentdValue) -> Self { + match value { + FluentdValue::String(s) => LogValue::String(s), + FluentdValue::Float(f) => LogValue::Float(f), + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct FluentdMessage( + #[serde(with = "decode_time")] pub DateTime, + pub HashMap, +); + +impl From for LogEntry { + fn from(value: FluentdMessage) -> Self { + let data = value.1.into_iter().map(|(k, v)| (k, v.into())).collect(); + LogEntry { ts: value.0, data } + } +} + +#[derive(Clone)] +pub struct FluentBitConnectionHandler { + sender: SyncSender, +} + +impl FluentBitConnectionHandler { + /// Starts the fluent-bit server with a handler delivers parsed messages to a receiver channel. + pub fn start(config: FluentBitConfig) -> Result<(ThreadedTcpServer, Receiver)> { + let (sender, receiver) = std::sync::mpsc::sync_channel(config.max_buffered_lines); + let server = ThreadedTcpServer::start( + config.bind_address, + config.max_connections, + FluentBitConnectionHandler { sender }, + )?; + Ok((server, receiver)) + } + + /// Starts the fluent-bit server with a handler that drops all data. + /// This is used in case data collection is disabled. We want to keep servicing fluent-bit in + /// this scenario, to avoid it retrying and buffering up data. + pub fn start_null(config: FluentBitConfig) -> Result { + ThreadedTcpServer::start( + config.bind_address, + config.max_connections, + TcpNullConnectionHandler {}, + ) + } +} + +impl TcpConnectionHandler for FluentBitConnectionHandler { + fn handle_connection(&self, stream: TcpStream) -> Result<()> { + let mut de = Deserializer::new(stream); + + loop { + match FluentdMessage::deserialize(&mut de) { + Ok(msg) => { + if self.sender.send(msg).is_err() { + // An error indicates that the channel has been closed, we should + // kill this thread. + break; + } + } + Err(e) => { + match e { + rmp_serde::decode::Error::InvalidMarkerRead(e) + if e.kind() == std::io::ErrorKind::UnexpectedEof => + { + // silently ignore end of stream + } + _ => warn!("FluentD decoding error: {:?}", e), + } + // After any deserialization error, we want to kill the connection. + break; + } + } + } + Ok(()) + } +} + +pub struct FluentBitConfig { + bind_address: SocketAddr, + max_buffered_lines: usize, + max_connections: usize, +} + +impl From<&Config> for FluentBitConfig { + fn from(config: &Config) -> Self { + Self { + bind_address: config.config_file.fluent_bit.bind_address, + max_buffered_lines: config.config_file.fluent_bit.max_buffered_lines, + max_connections: config.config_file.fluent_bit.max_connections, + } + } +} + +#[cfg(test)] +mod tests { + use std::net::TcpListener; + use std::{ + io::Write, net::Shutdown, sync::mpsc::sync_channel, thread, thread::JoinHandle, + time::Duration, + }; + + use rstest::{fixture, rstest}; + + use crate::test_utils::setup_logger; + + use super::*; + + #[rstest] + #[cfg_attr(not(target_os = "linux"), allow(unused_variables, unused_mut))] + fn deserialize_bogus_message(_setup_logger: (), mut connection: FluentBitFixture) { + // this test can flake on macOS. Run it only on Linux to avoid flaking Mac CI + #[cfg(target_os = "linux")] + { + connection.client.write_all("bogus".as_bytes()).unwrap(); + connection.client.shutdown(Shutdown::Both).unwrap(); + + // Make sure there is nothing received + let received = connection.receiver.recv(); + assert!(received.is_err()); + + // The handler should return without an error + assert!(connection.thread.join().is_ok()); + } + } + + #[rstest] + fn deserialize_one_message( + _setup_logger: (), + mut connection: FluentBitFixture, + message: FluentBitMessageFixture, + ) { + connection.client.write_all(&message.bytes).unwrap(); + connection.client.shutdown(Shutdown::Both).unwrap(); + + // Make sure message is received + let received = connection.receiver.recv().unwrap(); + assert_eq!(received.0, message.msg.0); + assert_eq!( + serde_json::to_string(&received.1).unwrap(), + serde_json::to_string(&message.msg.1).unwrap() + ); + + // The handler should return without an error + assert!(connection.thread.join().is_ok()); + } + + #[rstest] + fn deserialize_one_message_received_in_two_parts( + _setup_logger: (), + mut connection: FluentBitFixture, + message: FluentBitMessageFixture, + ) { + let buf1 = &message.bytes[0..10]; + let buf2 = &message.bytes[10..]; + + connection.client.write_all(buf1).unwrap(); + connection.client.flush().unwrap(); + // Make sure the other thread has time to do something + thread::sleep(Duration::from_millis(5)); + connection.client.write_all(buf2).unwrap(); + connection.client.shutdown(Shutdown::Both).unwrap(); + + // Make sure message is received + let received = connection.receiver.recv().unwrap(); + assert_eq!(received.0, message.msg.0); + assert_eq!( + serde_json::to_string(&received.1).unwrap(), + serde_json::to_string(&message.msg.1).unwrap() + ); + + // The handler should return without an error + assert!(connection.thread.join().is_ok()); + } + + #[rstest] + fn deserialize_two_concatenated_messages( + _setup_logger: (), + mut connection: FluentBitFixture, + message: FluentBitMessageFixture, + #[from(message)] message2: FluentBitMessageFixture, + ) { + let mut buf = message.bytes.clone(); + buf.extend(message2.bytes); + connection.client.write_all(&buf).unwrap(); + connection.client.shutdown(Shutdown::Both).unwrap(); + + // Make sure two messages are received + let received1 = connection.receiver.recv().unwrap(); + let received2 = connection.receiver.recv().unwrap(); + assert_eq!(received1.0, message.msg.0); + assert_eq!( + serde_json::to_string(&received1.1).unwrap(), + serde_json::to_string(&message.msg.1).unwrap() + ); + + assert_eq!(received2.0, message2.msg.0); + assert_eq!( + serde_json::to_string(&received2.1).unwrap(), + serde_json::to_string(&message2.msg.1).unwrap() + ); + + // The handler should return without an error + assert!(connection.thread.join().is_ok()); + } + + #[rstest] + /// Test the new data format with metadata associated to the timestamp (fluent-bit 2.1+) + fn deserialize_timestamp_with_metadata(_setup_logger: (), mut connection: FluentBitFixture) { + let buf = [ + 0xDD, 0x00, 0x00, 0x00, 0x02, 0xDD, 0x00, 0x00, 0x00, 0x02, 0xD7, 0x00, 0x64, 0x1B, + 0xD5, 0xF9, 0x1E, 0x4A, 0xFD, 0x58, 0xDF, 0x00, 0x00, 0x00, 0x00, 0xDF, 0x00, 0x00, + 0x00, 0x15, 0xAA, 0x5F, 0x54, 0x52, 0x41, 0x4E, 0x53, 0x50, 0x4F, 0x52, 0x54, 0xA6, + 0x73, 0x74, 0x64, 0x6F, 0x75, 0x74, 0xAA, 0x5F, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4D, + 0x5F, 0x49, 0x44, 0xD9, 0x20, 0x65, 0x62, 0x33, 0x37, 0x39, 0x30, 0x37, 0x62, 0x63, + 0x33, 0x34, 0x61, 0x34, 0x31, 0x64, 0x63, 0x62, 0x34, 0x61, 0x37, 0x65, 0x36, 0x63, + 0x37, 0x33, 0x61, 0x30, 0x32, 0x61, 0x66, 0x30, 0x32, 0xA8, 0x50, 0x52, 0x49, 0x4F, + 0x52, 0x49, 0x54, 0x59, 0xA1, 0x36, 0xAF, 0x53, 0x59, 0x53, 0x4C, 0x4F, 0x47, 0x5F, + 0x46, 0x41, 0x43, 0x49, 0x4C, 0x49, 0x54, 0x59, 0xA1, 0x33, 0xB1, 0x53, 0x59, 0x53, + 0x4C, 0x4F, 0x47, 0x5F, 0x49, 0x44, 0x45, 0x4E, 0x54, 0x49, 0x46, 0x49, 0x45, 0x52, + 0xA4, 0x67, 0x65, 0x74, 0x68, 0xA4, 0x5F, 0x50, 0x49, 0x44, 0xA3, 0x37, 0x37, 0x38, + 0xA4, 0x5F, 0x55, 0x49, 0x44, 0xA4, 0x31, 0x30, 0x30, 0x35, 0xA4, 0x5F, 0x47, 0x49, + 0x44, 0xA4, 0x31, 0x30, 0x30, 0x35, 0xA5, 0x5F, 0x43, 0x4F, 0x4D, 0x4D, 0xA4, 0x67, + 0x65, 0x74, 0x68, 0xA4, 0x5F, 0x45, 0x58, 0x45, 0xB3, 0x2F, 0x75, 0x73, 0x72, 0x2F, + 0x6C, 0x6F, 0x63, 0x61, 0x6C, 0x2F, 0x62, 0x69, 0x6E, 0x2F, 0x67, 0x65, 0x74, 0x68, + 0xA8, 0x5F, 0x43, 0x4D, 0x44, 0x4C, 0x49, 0x4E, 0x45, 0xD9, 0x9D, 0x2F, 0x75, 0x73, + 0x72, 0x2F, 0x6C, 0x6F, 0x63, 0x61, 0x6C, 0x2F, 0x62, 0x69, 0x6E, 0x2F, 0x67, 0x65, + 0x74, 0x68, 0x20, 0x2D, 0x2D, 0x6D, 0x61, 0x69, 0x6E, 0x6E, 0x65, 0x74, 0x20, 0x2D, + 0x2D, 0x64, 0x61, 0x74, 0x61, 0x64, 0x69, 0x72, 0x20, 0x2F, 0x64, 0x61, 0x74, 0x61, + 0x2D, 0x65, 0x74, 0x68, 0x2F, 0x67, 0x65, 0x74, 0x68, 0x20, 0x2D, 0x2D, 0x61, 0x75, + 0x74, 0x68, 0x72, 0x70, 0x63, 0x2E, 0x6A, 0x77, 0x74, 0x73, 0x65, 0x63, 0x72, 0x65, + 0x74, 0x20, 0x2F, 0x64, 0x61, 0x74, 0x61, 0x2D, 0x65, 0x74, 0x68, 0x2F, 0x6A, 0x77, + 0x74, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x2F, 0x6A, 0x77, 0x74, 0x2E, 0x68, 0x65, + 0x78, 0x20, 0x2D, 0x2D, 0x6D, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x20, 0x2D, 0x2D, + 0x6D, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2E, 0x61, 0x64, 0x64, 0x72, 0x20, 0x31, + 0x32, 0x37, 0x2E, 0x30, 0x2E, 0x30, 0x2E, 0x31, 0x20, 0x2D, 0x2D, 0x6D, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x73, 0x2E, 0x70, 0x6F, 0x72, 0x74, 0x20, 0x36, 0x30, 0x36, 0x31, + 0xAE, 0x5F, 0x43, 0x41, 0x50, 0x5F, 0x45, 0x46, 0x46, 0x45, 0x43, 0x54, 0x49, 0x56, + 0x45, 0xA1, 0x30, 0xB0, 0x5F, 0x53, 0x45, 0x4C, 0x49, 0x4E, 0x55, 0x58, 0x5F, 0x43, + 0x4F, 0x4E, 0x54, 0x45, 0x58, 0x54, 0xAB, 0x75, 0x6E, 0x63, 0x6F, 0x6E, 0x66, 0x69, + 0x6E, 0x65, 0x64, 0x0A, 0xAF, 0x5F, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4D, 0x44, 0x5F, + 0x43, 0x47, 0x52, 0x4F, 0x55, 0x50, 0xD9, 0x22, 0x2F, 0x73, 0x79, 0x73, 0x74, 0x65, + 0x6D, 0x2E, 0x73, 0x6C, 0x69, 0x63, 0x65, 0x2F, 0x67, 0x65, 0x74, 0x68, 0x2D, 0x6D, + 0x61, 0x69, 0x6E, 0x6E, 0x65, 0x74, 0x2E, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0xAD, 0x5F, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4D, 0x44, 0x5F, 0x55, 0x4E, 0x49, 0x54, + 0xB4, 0x67, 0x65, 0x74, 0x68, 0x2D, 0x6D, 0x61, 0x69, 0x6E, 0x6E, 0x65, 0x74, 0x2E, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0xAE, 0x5F, 0x53, 0x59, 0x53, 0x54, 0x45, + 0x4D, 0x44, 0x5F, 0x53, 0x4C, 0x49, 0x43, 0x45, 0xAC, 0x73, 0x79, 0x73, 0x74, 0x65, + 0x6D, 0x2E, 0x73, 0x6C, 0x69, 0x63, 0x65, 0xB6, 0x5F, 0x53, 0x59, 0x53, 0x54, 0x45, + 0x4D, 0x44, 0x5F, 0x49, 0x4E, 0x56, 0x4F, 0x43, 0x41, 0x54, 0x49, 0x4F, 0x4E, 0x5F, + 0x49, 0x44, 0xD9, 0x20, 0x35, 0x33, 0x30, 0x33, 0x35, 0x36, 0x66, 0x36, 0x33, 0x63, + 0x64, 0x31, 0x34, 0x64, 0x64, 0x61, 0x62, 0x30, 0x66, 0x65, 0x64, 0x31, 0x65, 0x38, + 0x30, 0x64, 0x36, 0x61, 0x30, 0x35, 0x65, 0x63, 0xA8, 0x5F, 0x42, 0x4F, 0x4F, 0x54, + 0x5F, 0x49, 0x44, 0xD9, 0x20, 0x62, 0x36, 0x36, 0x36, 0x31, 0x61, 0x66, 0x39, 0x36, + 0x33, 0x64, 0x38, 0x34, 0x61, 0x62, 0x37, 0x39, 0x62, 0x33, 0x38, 0x32, 0x33, 0x63, + 0x62, 0x65, 0x66, 0x35, 0x39, 0x66, 0x37, 0x33, 0x64, 0xAB, 0x5F, 0x4D, 0x41, 0x43, + 0x48, 0x49, 0x4E, 0x45, 0x5F, 0x49, 0x44, 0xD9, 0x20, 0x35, 0x61, 0x32, 0x63, 0x39, + 0x38, 0x66, 0x38, 0x35, 0x31, 0x64, 0x32, 0x34, 0x33, 0x64, 0x33, 0x38, 0x36, 0x61, + 0x62, 0x62, 0x35, 0x64, 0x37, 0x62, 0x39, 0x31, 0x32, 0x64, 0x64, 0x31, 0x66, 0xA9, + 0x5F, 0x48, 0x4F, 0x53, 0x54, 0x4E, 0x41, 0x4D, 0x45, 0xA7, 0x66, 0x72, 0x61, 0x63, + 0x74, 0x61, 0x6C, 0xA7, 0x4D, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0xD9, 0xC2, 0x49, + 0x4E, 0x46, 0x4F, 0x20, 0x5B, 0x30, 0x33, 0x2D, 0x32, 0x33, 0x7C, 0x30, 0x34, 0x3A, + 0x33, 0x30, 0x3A, 0x34, 0x39, 0x2E, 0x35, 0x30, 0x38, 0x5D, 0x20, 0x49, 0x6D, 0x70, + 0x6F, 0x72, 0x74, 0x65, 0x64, 0x20, 0x6E, 0x65, 0x77, 0x20, 0x70, 0x6F, 0x74, 0x65, + 0x6E, 0x74, 0x69, 0x61, 0x6C, 0x20, 0x63, 0x68, 0x61, 0x69, 0x6E, 0x20, 0x73, 0x65, + 0x67, 0x6D, 0x65, 0x6E, 0x74, 0x20, 0x20, 0x20, 0x20, 0x20, 0x62, 0x6C, 0x6F, 0x63, + 0x6B, 0x73, 0x3D, 0x31, 0x20, 0x20, 0x20, 0x20, 0x74, 0x78, 0x73, 0x3D, 0x36, 0x30, + 0x30, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6D, 0x67, 0x61, 0x73, 0x3D, 0x32, 0x34, 0x2E, + 0x36, 0x39, 0x37, 0x20, 0x20, 0x65, 0x6C, 0x61, 0x70, 0x73, 0x65, 0x64, 0x3D, 0x33, + 0x31, 0x36, 0x2E, 0x35, 0x39, 0x34, 0x6D, 0x73, 0x20, 0x20, 0x20, 0x20, 0x6D, 0x67, + 0x61, 0x73, 0x70, 0x73, 0x3D, 0x37, 0x38, 0x2E, 0x30, 0x30, 0x37, 0x20, 0x20, 0x6E, + 0x75, 0x6D, 0x62, 0x65, 0x72, 0x3D, 0x31, 0x36, 0x2C, 0x38, 0x38, 0x37, 0x2C, 0x38, + 0x38, 0x36, 0x20, 0x68, 0x61, 0x73, 0x68, 0x3D, 0x30, 0x37, 0x63, 0x33, 0x65, 0x36, + 0x2E, 0x2E, 0x63, 0x34, 0x36, 0x37, 0x33, 0x39, 0x20, 0x64, 0x69, 0x72, 0x74, 0x79, + 0x3D, 0x31, 0x30, 0x32, 0x32, 0x2E, 0x34, 0x30, 0x4D, 0x69, 0x42, + ]; + + connection.client.write_all(&buf).unwrap(); + + // Make sure one messages is received + let _received1 = connection + .receiver + .recv_timeout(Duration::from_millis(10)) + .unwrap(); + + // The handler should return without an error + connection.client.shutdown(Shutdown::Both).unwrap(); + assert!(connection.thread.join().is_ok()); + } + + struct FluentBitFixture { + client: TcpStream, + thread: JoinHandle>, + receiver: Receiver, + } + + #[fixture] + fn connection() -> FluentBitFixture { + let (sender, receiver) = sync_channel(1); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let local_address = listener.local_addr().unwrap(); + + let client = TcpStream::connect(local_address).unwrap(); + let (server, _) = listener.accept().unwrap(); + + let handler = FluentBitConnectionHandler { sender }; + let thread = thread::spawn(move || handler.handle_connection(server)); + + FluentBitFixture { + client, + thread, + receiver, + } + } + + struct FluentBitMessageFixture { + msg: FluentdMessage, + bytes: Vec, + } + + #[fixture] + fn message() -> FluentBitMessageFixture { + let msg = FluentdMessage( + Utc::now(), + HashMap::from([( + "MESSAGE".to_owned(), + FluentdValue::String("something happened on the way to the moon".into()), + )]), + ); + let bytes = rmp_serde::to_vec(&msg).unwrap(); + FluentBitMessageFixture { msg, bytes } + } +} diff --git a/memfaultd/src/http_server/handler.rs b/memfaultd/src/http_server/handler.rs new file mode 100644 index 0000000..43e07b8 --- /dev/null +++ b/memfaultd/src/http_server/handler.rs @@ -0,0 +1,94 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::Result; +use tiny_http::{Request, ResponseBox}; + +/// Return type of the HttpHandler +pub enum HttpHandlerResult { + /// Request was processed and response is proposed. + Response(ResponseBox), + /// An error occurred while processing the request (will return 500). + Error(String), + /// Request not handled by this handler. Continue to next handler. + NotHandled, +} + +/// This little helper makes it possible to use the ? operator in handlers when +/// you have already checked method and path and know that they should handle +/// the request, possibly failing while doing so. +/// +/// ``` +/// # use eyre::Result; +/// use tiny_http::{Request, Response, ResponseBox}; +/// use memfaultd::http_server::{HttpHandler, HttpHandlerResult}; +/// +/// struct CounterHandler { +/// counter: u32, +/// }; +/// +/// impl CounterHandler { +/// fn handle_read(&self) -> Result { +/// Ok(Response::from_string("42").boxed()) +/// } +/// } +/// +/// impl HttpHandler for CounterHandler { +/// fn handle_request(&self, r: &mut Request) -> HttpHandlerResult { +/// if r.url() == "/count" { +/// self.handle_read().into() +/// } +/// else { +/// HttpHandlerResult::NotHandled +/// } +/// } +/// } +/// ``` +impl From> for HttpHandlerResult { + fn from(r: Result) -> Self { + match r { + Ok(response) => HttpHandlerResult::Response(response), + Err(e) => HttpHandlerResult::Error(e.to_string()), + } + } +} + +/// An HttpHandler can handle a request and send a response. +pub trait HttpHandler: Send { + /// Handle a request and prepares the response. + /// + /// ``` + /// # use eyre::Result; + /// use tiny_http::{ Request, Response, ResponseBox }; + /// use memfaultd::http_server::{HttpHandler, HttpHandlerResult}; + /// + /// struct CounterHandler { + /// counter: u32, + /// }; + /// + /// impl HttpHandler for CounterHandler { + /// fn handle_request(&self, r: &mut Request) -> HttpHandlerResult { + /// HttpHandlerResult::Response(Response::empty(200).boxed()) + /// } + /// } + /// + /// ``` + fn handle_request(&self, request: &mut Request) -> HttpHandlerResult; +} + +#[cfg(test)] +mod tests { + use tiny_http::ResponseBox; + + use super::HttpHandlerResult; + + impl HttpHandlerResult { + pub fn expect(self, m: &'static str) -> ResponseBox { + match self { + HttpHandlerResult::Response(response) => response, + HttpHandlerResult::Error(e) => panic!("{}: HttpHandlerResult::Error({})", m, e), + HttpHandlerResult::NotHandled => panic!("{}: HttpHandlerResult::Nothandled", m), + } + } + } +} diff --git a/memfaultd/src/http_server/mod.rs b/memfaultd/src/http_server/mod.rs new file mode 100644 index 0000000..d8a46d2 --- /dev/null +++ b/memfaultd/src/http_server/mod.rs @@ -0,0 +1,19 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Memfaultd HTTP server +//! Used on device for communication with other programs and `memfaultctl`. +//! +//! Typically binds to 127.0.0.1 and only available locally. +//! +mod handler; +mod request_bodies; +mod server; +mod utils; + +pub use handler::{HttpHandler, HttpHandlerResult}; +pub use server::HttpServer; + +pub use utils::ConvenientHeader; + +pub use request_bodies::SessionRequest; diff --git a/memfaultd/src/http_server/request_bodies.rs b/memfaultd/src/http_server/request_bodies.rs new file mode 100644 index 0000000..155f782 --- /dev/null +++ b/memfaultd/src/http_server/request_bodies.rs @@ -0,0 +1,28 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use serde::{Deserialize, Serialize}; + +use crate::metrics::{KeyedMetricReading, SessionName}; + +#[derive(Serialize, Deserialize)] +pub struct SessionRequest { + pub session_name: SessionName, + pub readings: Vec, +} + +impl SessionRequest { + pub fn new(session_name: SessionName, readings: Vec) -> Self { + Self { + session_name, + readings, + } + } + + pub fn new_without_readings(session_name: SessionName) -> Self { + Self { + session_name, + readings: vec![], + } + } +} diff --git a/memfaultd/src/http_server/server.rs b/memfaultd/src/http_server/server.rs new file mode 100644 index 0000000..5441b6e --- /dev/null +++ b/memfaultd/src/http_server/server.rs @@ -0,0 +1,73 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{net::SocketAddr, thread::spawn}; + +use eyre::{eyre, Result}; +use log::{debug, trace, warn}; +use tiny_http::{Request, Response, Server}; + +use crate::http_server::{HttpHandler, HttpHandlerResult}; + +/// A server that listens for collectd JSON pushes and stores them in memory. +pub struct HttpServer { + handlers: Option>>, +} + +impl HttpServer { + pub fn new(handlers: Vec>) -> Self { + HttpServer { + handlers: Some(handlers), + } + } + + pub fn start(&mut self, listening_address: SocketAddr) -> Result<()> { + let server = Server::http(listening_address).map_err(|e| { + eyre!("Error starting server: could not bind to {listening_address}: {e}") + })?; + + if let Some(handlers) = self.handlers.take() { + spawn(move || { + debug!("HTTP Server started on {listening_address}"); + + for request in server.incoming_requests() { + Self::handle_request(&handlers, request); + } + }); + Ok(()) + } else { + Err(eyre!("HTTP Server already started")) + } + } + + fn handle_request(handlers: &[Box], mut request: Request) { + trace!( + "HTTP request {:?} {:?}\n{:?}", + request.method(), + request.url(), + request.headers() + ); + + let method = request.method().to_owned(); + let url = request.url().to_owned(); + for handler in handlers.iter() { + match handler.handle_request(&mut request) { + HttpHandlerResult::Response(response) => { + if let Err(e) = request.respond(response) { + warn!("HTTP: Error sending response {} {}: {:?}", method, url, e); + } + return; + } + HttpHandlerResult::Error(e) => { + warn!("HTTP: Error processing request {} {}: {}", method, url, e); + let _r = request + .respond(Response::empty(500).with_data(e.as_bytes(), Some(e.len()))); + return; + } + HttpHandlerResult::NotHandled => { /* continue */ } + }; + } + debug!("HTTP[404] {} {}", method, url); + let _r = request.respond(Response::empty(404)); + } +} diff --git a/memfaultd/src/http_server/snapshots/memfaultd__http_server__response__tests__response_error.snap b/memfaultd/src/http_server/snapshots/memfaultd__http_server__response__tests__response_error.snap new file mode 100644 index 0000000..d2fa9eb --- /dev/null +++ b/memfaultd/src/http_server/snapshots/memfaultd__http_server__response__tests__response_error.snap @@ -0,0 +1,5 @@ +--- +source: memfaultd/src/http_server/response.rs +expression: "serde_json::to_string(&HttpResponse::Error(\"oops\".into())).unwrap()" +--- +{"status":"error","message":"oops"} diff --git a/memfaultd/src/http_server/snapshots/memfaultd__http_server__response__tests__response_not_found.snap b/memfaultd/src/http_server/snapshots/memfaultd__http_server__response__tests__response_not_found.snap new file mode 100644 index 0000000..abd36b3 --- /dev/null +++ b/memfaultd/src/http_server/snapshots/memfaultd__http_server__response__tests__response_not_found.snap @@ -0,0 +1,5 @@ +--- +source: memfaultd/src/http_server/response.rs +expression: "serde_json::to_string(&HttpResponse::NotFound).unwrap()" +--- +{"status":"not_found"} diff --git a/memfaultd/src/http_server/snapshots/memfaultd__http_server__response__tests__response_ok.snap b/memfaultd/src/http_server/snapshots/memfaultd__http_server__response__tests__response_ok.snap new file mode 100644 index 0000000..2b20e60 --- /dev/null +++ b/memfaultd/src/http_server/snapshots/memfaultd__http_server__response__tests__response_ok.snap @@ -0,0 +1,5 @@ +--- +source: memfaultd/src/http_server/response.rs +expression: "serde_json::to_string(&HttpResponse::Ok).unwrap()" +--- +{"status":"ok"} diff --git a/memfaultd/src/http_server/utils.rs b/memfaultd/src/http_server/utils.rs new file mode 100644 index 0000000..deffebb --- /dev/null +++ b/memfaultd/src/http_server/utils.rs @@ -0,0 +1,15 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::{eyre, Result}; +use tiny_http::Header; + +/// Wraps Header.from_bytes into something that returns a Result<> compatible with eyre::Result. +pub trait ConvenientHeader { + fn from_strings(name: &str, value: &str) -> Result
; +} +impl ConvenientHeader for Header { + fn from_strings(name: &str, value: &str) -> Result
{ + Header::from_bytes(name, value).map_err(|_e| eyre!("Invalid header ({}: {})", name, value)) + } +} diff --git a/memfaultd/src/lib.rs b/memfaultd/src/lib.rs new file mode 100644 index 0000000..f4d8bfb --- /dev/null +++ b/memfaultd/src/lib.rs @@ -0,0 +1,33 @@ +#![deny(clippy::print_stdout, clippy::print_stderr)] +// +// Copyright (c) Memfault, Inc. +// See License.txt for details + +pub mod cli; +#[cfg(feature = "collectd")] +mod collectd; +mod config; +#[cfg(feature = "coredump")] +mod coredump; +#[cfg(feature = "logging")] +mod fluent_bit; + +pub mod http_server; +#[cfg(feature = "logging")] +mod logs; +pub mod mar; +mod memfaultd; +pub mod metrics; +mod network; +mod reboot; +mod retriable_error; +mod service_manager; +#[cfg(feature = "swupdate")] +mod swupdate; +#[cfg(test)] +mod test_utils; +pub mod util; + +pub mod build_info { + include!(concat!(env!("OUT_DIR"), "/build_info.rs")); +} diff --git a/memfaultd/src/logs/completed_log.rs b/memfaultd/src/logs/completed_log.rs new file mode 100644 index 0000000..e3569eb --- /dev/null +++ b/memfaultd/src/logs/completed_log.rs @@ -0,0 +1,15 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::mar::CompressionAlgorithm; +use std::path::PathBuf; +use uuid::Uuid; + +/// CompletedLog represents a log that has been rotated and is ready to be moved into the MAR +/// staging area. +pub struct CompletedLog { + pub path: PathBuf, + pub cid: Uuid, + pub next_cid: Uuid, + pub compression: CompressionAlgorithm, +} diff --git a/memfaultd/src/logs/fluent_bit_adapter.rs b/memfaultd/src/logs/fluent_bit_adapter.rs new file mode 100644 index 0000000..c2249c1 --- /dev/null +++ b/memfaultd/src/logs/fluent_bit_adapter.rs @@ -0,0 +1,104 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! An adapter to connect FluentBit to our LogCollector. +//! +use std::sync::mpsc::Receiver; + +use log::warn; + +use crate::fluent_bit::FluentdMessage; +use crate::logs::log_entry::LogEntry; + +/// An iterator that can be used as a source of logs for LogCollector. +/// Will filter fluent-bit messages to keep only log messages and convert them +/// to a serde_json::Value ready to be written to disk. +pub struct FluentBitAdapter { + receiver: Receiver, + extra_fields: Vec, +} + +impl FluentBitAdapter { + pub fn new(receiver: Receiver, extra_fluent_bit_fields: &[String]) -> Self { + Self { + receiver, + extra_fields: extra_fluent_bit_fields.to_owned(), + } + } + + /// Convert a FluentdMessage into a serde_json::Value that we can log. + /// Returns None when this message should be filtered out. + fn convert_message(msg: FluentdMessage, extra_fields: &[String]) -> Option { + if !msg.1.contains_key("MESSAGE") { + // We are only interested in log messages. They will have a 'MESSAGE' key. + // Metrics do not have the MESSAGE key. + return None; + } + + let mut log_entry = LogEntry::from(msg); + log_entry.filter_fields(extra_fields); + + Some(log_entry) + } +} + +impl Iterator for FluentBitAdapter { + type Item = LogEntry; + /// Convert a FluentdMessage to a LogRecord for LogCollector. + /// Messages can be filtered out by returning None here. + fn next(&mut self) -> Option { + loop { + let msg_r = self.receiver.recv(); + match msg_r { + Ok(msg) => { + let value = FluentBitAdapter::convert_message(msg, &self.extra_fields); + match value { + v @ Some(_) => return v, + None => continue, + } + } + Err(e) => { + warn!("fluent-bit stopped receiving messages with error: {:?}", e); + return None; + } + } + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::mpsc::channel; + + use chrono::{DateTime, NaiveDateTime, Utc}; + use insta::{assert_json_snapshot, with_settings}; + + use super::*; + use crate::fluent_bit::{FluentdMessage, FluentdValue}; + + #[test] + fn test_fluent_bit_adapter() { + let (tx, rx) = channel(); + let mut adapter = FluentBitAdapter::new(rx, &[]); + + let mut map = HashMap::new(); + map.insert( + "MESSAGE".to_string(), + FluentdValue::String("test".to_string()), + ); + let msg = FluentdMessage(time(), map); + tx.send(msg).unwrap(); + + let log_entry = adapter.next().unwrap(); + + with_settings!({sort_maps => true}, { + assert_json_snapshot!(log_entry); + }); + } + + fn time() -> DateTime { + let naive = NaiveDateTime::from_timestamp_millis(1334250000000).unwrap(); + DateTime::::from_utc(naive, Utc) + } +} diff --git a/memfaultd/src/logs/headroom.rs b/memfaultd/src/logs/headroom.rs new file mode 100644 index 0000000..6302849 --- /dev/null +++ b/memfaultd/src/logs/headroom.rs @@ -0,0 +1,465 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::{ + logs::log_file::{LogFile, LogFileControl}, + util::disk_size::DiskSize, +}; +use eyre::Result; +use serde_json::Value; + +#[derive(Debug)] +enum Headroom { + Ok, + Shortage { + num_dropped_logs: usize, + has_rotated: bool, + }, +} + +pub trait HeadroomCheck { + fn check( + &mut self, + log_timestamp: Option<&Value>, + log_file_control: &mut impl LogFileControl, + ) -> Result; +} + +pub struct HeadroomLimiter { + state: Headroom, + /// Minimum amount of free space that must be kept available in the mount point in which + /// log_tmp_path resides. If there is not sufficient head room, logs will be dropped. + min_headroom: DiskSize, + get_available_space: Box Result + Send>, +} + +impl HeadroomLimiter { + pub fn new Result + Send + 'static>( + min_headroom: DiskSize, + get_available_space: S, + ) -> Self { + Self { + state: Headroom::Ok, + min_headroom, + get_available_space: Box::new(get_available_space), + } + } +} + +impl HeadroomCheck for HeadroomLimiter { + /// Checks whether there is enough headroom to continue writing logs. + /// If there is not enough headroom, this will flush the current log file and rotate at most + /// once when needed, until there is enough headroom again. When there's enough space again, it + /// will emit a log message mentioning the number of dropped logs. + /// Returns Ok(true) if there is enough headroom, Ok(false) if there is not enough headroom. + /// It only returns an error if there is an error writing the "Dropped N logs" message. + fn check( + &mut self, + log_timestamp: Option<&Value>, + log_file_control: &mut impl LogFileControl, + ) -> Result { + let available = (self.get_available_space)()?; + let has_headroom = available.exceeds(&self.min_headroom); + + self.state = match (has_headroom, &self.state) { + // Enter insufficient headroom state: + (false, Headroom::Ok) => { + // Best-effort warning log & flush. If this fails, just keep going. + let current_log = log_file_control.current_log()?; + let _ = current_log.write_log( + log_timestamp.cloned(), + match ( + available.bytes >= self.min_headroom.bytes, + available.inodes >= self.min_headroom.inodes, + ) { + (false, false) => "Low on disk space and inodes. Starting to drop logs...", + (false, true) => "Low on disk space. Starting to drop logs...", + (true, false) => "Low on inodes. Starting to drop logs...", + _ => unreachable!(), + }, + ); + let _ = current_log.flush(); + Headroom::Shortage { + has_rotated: log_file_control.rotate_if_needed().unwrap_or(false), + num_dropped_logs: 1, + } + } + // Already in insufficient headroom state: + ( + false, + Headroom::Shortage { + has_rotated, + num_dropped_logs, + }, + ) => { + // Rotate logs once only: + let num_dropped_logs = *num_dropped_logs + 1; + let has_rotated = + *has_rotated || log_file_control.rotate_if_needed().unwrap_or(false); + Headroom::Shortage { + num_dropped_logs, + has_rotated, + } + } + // Exit insufficient headroom state: + ( + true, + Headroom::Shortage { + num_dropped_logs, .. + }, + ) => { + let current_log = log_file_control.current_log()?; + current_log.write_log( + log_timestamp.cloned(), + format!( + "Recovered from low disk space. Dropped {} logs.", + num_dropped_logs + ), + )?; + Headroom::Ok + } + // Already in headroom OK state and staying in this state: + (true, Headroom::Ok) => Headroom::Ok, + }; + Ok(has_headroom) + } +} + +#[cfg(test)] +mod tests { + use crate::util::disk_size::DiskSize; + use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }; + + use super::*; + use eyre::eyre; + use rstest::{fixture, rstest}; + + #[rstest] + fn returns_true_if_headroom_ok_and_stays_ok(mut fixture: Fixture) { + let log_timestamp = Value::from(12345); + let mut log_file_control = FakeLogFileControl::default(); + fixture.set_available_space(MIN_HEADROOM); + + // Enough headroom: check() returns true and no calls to log_file_control are made: + assert!(fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap()); + assert_eq!(0, log_file_control.logs_written.len()); + assert_eq!(0, log_file_control.flush_count); + assert_eq!(0, log_file_control.rotation_count); + } + + #[rstest] + fn log_upon_enter_and_exit_headroom_space_shortage(mut fixture: Fixture) { + let log_timestamp = Value::from(12345); + let mut log_file_control = FakeLogFileControl::default(); + + // Enter headroom shortage: check() returns false: + fixture.set_available_space(MIN_HEADROOM - 1); + assert!(!fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap()); + + // Check that the warning log was written: + assert_eq!(1, log_file_control.logs_written.len()); + assert!(log_file_control.logs_written[0] + .contains("Low on disk space. Starting to drop logs...")); + // Check that the log was flushed: + assert_eq!(1, log_file_control.flush_count); + + // Still not enough headroom: check() returns false: + assert!(!fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap()); + + // Recover from headroom shortage: check() returns true again: + fixture.set_available_space(MIN_HEADROOM); + assert!(fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap()); + + // Check that the "recovered" log was written: + assert_eq!(2, log_file_control.logs_written.len()); + assert!(log_file_control.logs_written[1] + .contains("Recovered from low disk space. Dropped 2 logs.")); + } + + #[rstest] + fn log_upon_enter_and_exit_headroom_node_shortage(mut fixture: Fixture) { + let log_timestamp = Value::from(12345); + let mut log_file_control = FakeLogFileControl::default(); + + // Enter headroom shortage: check() returns false: + fixture.set_available_inodes(MIN_INODES - 1); + assert!(!fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap()); + + // Check that the warning log was written: + assert_eq!(1, log_file_control.logs_written.len()); + assert!( + log_file_control.logs_written[0].contains("Low on inodes. Starting to drop logs...") + ); + // Check that the log was flushed: + assert_eq!(1, log_file_control.flush_count); + + // Still not enough headroom: check() returns false: + assert!(!fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap()); + + // Recover from headroom shortage: check() returns true again: + fixture.set_available_inodes(MIN_INODES); + assert!(fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap()); + + // Check that the "recovered" log was written: + assert_eq!(2, log_file_control.logs_written.len()); + assert!(log_file_control.logs_written[1] + .contains("Recovered from low disk space. Dropped 2 logs.")); + } + + #[rstest] + fn rotate_once_only_entering_headroom_shortage(mut fixture: Fixture) { + let log_timestamp = Value::from(12345); + let mut log_file_control = FakeLogFileControl { + // Make log_file_control.rotate_if_needed() return Ok(true): + rotate_return: Some(true), + ..Default::default() + }; + + // Enter headroom shortage: + fixture.set_available_space(MIN_HEADROOM - 1); + fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap(); + assert_eq!(log_file_control.rotation_count, 1); + + // Check again. Rotation should not be attempted again: + fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap(); + assert_eq!(log_file_control.rotation_count, 1); + } + + #[rstest] + fn rotate_once_only_during_headroom_shortage(mut fixture: Fixture) { + let log_timestamp = Value::from(12345); + let mut log_file_control = FakeLogFileControl::default(); + + // Enter headroom shortage: + fixture.set_available_space(MIN_HEADROOM - 1); + fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap(); + assert_eq!(log_file_control.rotation_count, 0); + + // Make log_file_control.rotate_if_needed() return Ok(true): + log_file_control.rotate_return = Some(true); + + // Check again. Rotation should be attempted again: + fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap(); + assert_eq!(log_file_control.rotation_count, 1); + + // Check again. Rotation should not be attempted again: + fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap(); + assert_eq!(log_file_control.rotation_count, 1); + } + + #[rstest] + fn retry_rotate_after_failure(mut fixture: Fixture) { + let log_timestamp = Value::from(12345); + let mut log_file_control = FakeLogFileControl { + // Make log_file_control.rotate_if_needed() return Err(...): + rotate_return: None, + ..Default::default() + }; + + // Enter headroom shortage: + fixture.set_available_space(MIN_HEADROOM - 1); + fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap(); + assert_eq!(log_file_control.rotation_count, 0); + + // Check again. Rotation should be attempted again: + // Make log_file_control.rotate_if_needed() return Ok(true): + log_file_control.rotate_return = Some(true); + fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap(); + assert_eq!(log_file_control.rotation_count, 1); + } + + #[rstest] + fn write_error_of_initial_warning_message_is_ignored(mut fixture: Fixture) { + let log_timestamp = Value::from(12345); + let mut log_file_control = FakeLogFileControl::default(); + + fixture.set_available_space(MIN_HEADROOM - 1); + log_file_control.write_should_fail = true; + assert!(fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .is_ok()); + } + + #[rstest] + fn write_error_of_recovery_log_message_is_bubbled_up(mut fixture: Fixture) { + let log_timestamp = Value::from(12345); + let mut log_file_control = FakeLogFileControl::default(); + + fixture.set_available_space(MIN_HEADROOM - 1); + fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap(); + fixture.set_available_space(MIN_HEADROOM); + log_file_control.write_should_fail = true; + assert!(fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .is_err()); + } + + struct FakeLogFileControl { + logs_written: Vec, + write_should_fail: bool, + flush_count: usize, + flush_should_fail: bool, + /// This controls the result of rotate_if_needed(). + /// Some(...) is mapped to Ok(...) and None is mapped to Err(...). + rotate_return: Option, + /// Number of times actually rotated (rotate_if_needed() calls while rotate_return was Some(true)): + rotation_count: usize, + } + + impl Default for FakeLogFileControl { + fn default() -> Self { + FakeLogFileControl { + logs_written: Vec::new(), + flush_count: 0, + flush_should_fail: false, + write_should_fail: false, + rotate_return: Some(false), + rotation_count: 0, + } + } + } + + impl LogFile for FakeLogFileControl { + fn write_json_line(&mut self, json: Value) -> Result<()> { + if self.write_should_fail { + Err(eyre!("Write failed")) + } else { + self.logs_written.push(serde_json::to_string(&json)?); + Ok(()) + } + } + + fn flush(&mut self) -> Result<()> { + self.flush_count += 1; + if self.flush_should_fail { + Err(eyre!("Flush failed")) + } else { + Ok(()) + } + } + } + + impl LogFileControl for FakeLogFileControl { + fn rotate_if_needed(&mut self) -> Result { + match self.rotate_return { + Some(rv) => { + if rv { + self.rotation_count += 1; + } + Ok(rv) + } + None => Err(eyre!("Rotate failed")), + } + } + + fn rotate_unless_empty(&mut self) -> Result<()> { + unimplemented!(); + } + + fn current_log(&mut self) -> Result<&mut FakeLogFileControl> { + Ok(self) + } + + fn close(self) -> Result<()> { + Ok(()) + } + } + + struct Fixture { + available_space: Arc, + available_inodes: Arc, + limiter: HeadroomLimiter, + } + + impl Fixture { + fn set_available_space(&mut self, available_space: u64) { + self.available_space + .store(available_space, Ordering::Relaxed) + } + fn set_available_inodes(&mut self, available_inodes: u64) { + self.available_inodes + .store(available_inodes, Ordering::Relaxed) + } + } + + const MIN_HEADROOM: u64 = 1024; + const MIN_INODES: u64 = 10; + const INITIAL_AVAILABLE_SPACE: u64 = 1024 * 1024; + const INITIAL_AVAILABLE_INODES: u64 = 100; + + #[fixture] + fn fixture() -> Fixture { + let available_space = Arc::new(AtomicU64::new(INITIAL_AVAILABLE_SPACE)); + let available_inodes = Arc::new(AtomicU64::new(INITIAL_AVAILABLE_INODES)); + + let space = available_space.clone(); + let inodes = available_inodes.clone(); + + Fixture { + limiter: HeadroomLimiter::new( + DiskSize { + bytes: MIN_HEADROOM, + inodes: MIN_INODES, + }, + move || { + Ok(DiskSize { + bytes: space.load(Ordering::Relaxed), + inodes: inodes.load(Ordering::Relaxed), + }) + }, + ), + available_inodes, + available_space, + } + } +} diff --git a/memfaultd/src/logs/journald_parser.rs b/memfaultd/src/logs/journald_parser.rs new file mode 100644 index 0000000..2332a58 --- /dev/null +++ b/memfaultd/src/logs/journald_parser.rs @@ -0,0 +1,393 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use chrono::{DateTime, NaiveDateTime, Utc}; +use eyre::{eyre, Result}; +use libc::free; +use nix::poll::{poll, PollFd}; +use serde::Serialize; +use std::ffi::CString; +use std::fs::read_to_string; +use std::{collections::HashMap, path::PathBuf}; +use std::{ffi::c_char, mem::MaybeUninit}; + +use log::{debug, warn}; +use memfaultc_sys::systemd::{ + sd_journal, sd_journal_add_match, sd_journal_enumerate_data, sd_journal_get_cursor, + sd_journal_get_fd, sd_journal_get_realtime_usec, sd_journal_next, sd_journal_open, + sd_journal_process, sd_journal_seek_cursor, +}; + +use super::log_entry::{LogEntry, LogValue}; +use crate::util::system::read_system_boot_id; +/// A trait for reading journal entries from the systemd journal. +/// +/// This trait is used to abstract the raw journal entry reading logic from the rest of the codebase. +/// This allows for easier testing and mocking of the journal reading logic. +#[cfg_attr(test, mockall::automock)] +pub trait JournalRaw { + /// Check if the next journal entry is available. + /// + /// Returns `Ok(true)` if the next journal entry is available, `Ok(false)` if there are no more entries, + /// and an error if there was an issue reading the journal. + fn next_entry_available(&mut self) -> Result; + + /// Get the raw field data of the current journal entry. + /// + /// Returns the raw string representing the key-value pairs of the journal entry, or `None` if there are no more entries. + /// Returns an error if there was an issue reading the journal. + fn get_entry_field_data(&mut self) -> Result>; + + /// Waits for the next journal entry to be available. + /// + /// This method should block until the next journal entry is available. + fn wait_for_entry(&mut self) -> Result<()>; +} + +/// Raw journal entry data. +/// +/// This struct represents the raw data of a journal entry. It can be converted into a `JournalEntry` struct, +/// which contains the parsed key-value pairs of the journal entry. +#[derive(Debug)] +pub struct JournalEntryRaw { + pub ts: DateTime, + pub fields: Vec, +} + +impl JournalEntryRaw { + pub fn new(fields: Vec, ts: DateTime) -> Self { + Self { ts, fields } + } +} + +/// An implementation of the `JournalRaw` trait that reads journal entries from the systemd journal. +/// +/// This struct is used to read journal entries from the systemd journal. It relies on ffi calls into +/// libsystemd. +pub struct JournalRawImpl { + journal: *mut sd_journal, + wait_fd: PollFd, + cursor_file: PathBuf, +} + +impl JournalRawImpl { + /// Timeout journal polling after 1 minute. + const POLL_TIMEOUT_MS: i32 = 1000 * 60; + const JOURNAL_CURSOR_FILE: &str = "JOURNALD_CURSOR"; + + pub fn new(tmp_path: PathBuf) -> Self { + let mut journal = std::ptr::null_mut(); + let cursor_file = tmp_path.join(Self::JOURNAL_CURSOR_FILE); + let cursor_string = read_to_string(&cursor_file).ok(); + + unsafe { + sd_journal_open(&mut journal, 0); + } + + let fd = unsafe { sd_journal_get_fd(journal) }; + let wait_fd = PollFd::new(fd, nix::poll::PollFlags::POLLIN); + + if let Some(cursor) = cursor_string { + let cursor = cursor.trim(); + let ret = unsafe { sd_journal_seek_cursor(journal, cursor.as_ptr() as *const c_char) }; + if ret < 0 { + warn!("Failed to seek journal to cursor: {}", ret); + } + } else if let Err(e) = Self::seek_to_current_boot_start(journal) { + warn!("Couldn't seek journal to start of current boot: {}", e); + } + + Self { + journal, + wait_fd, + cursor_file, + } + } + + /// Seeks the journal to the start of the current boot's logs + /// + /// Returns OK if the journal is now set up to return the first log + /// from the current boot + /// Returns an error if the function could not confirm the next + /// entry will be the start of the current boot + fn seek_to_current_boot_start(journal: *mut sd_journal) -> Result<()> { + let boot_id = read_system_boot_id()?; + let boot_id_match = CString::new(format!("_BOOT_ID={}", boot_id.as_simple()))?; + + let ret = + unsafe { sd_journal_add_match(journal, boot_id_match.as_ptr() as *const c_char, 0) }; + match ret { + ret if ret < 0 => Err(eyre!( + "Failed to add match on current boot ID to journal: {}", + ret + )), + 0 => Ok(()), + _ => Ok(()), + } + } + + /// Save the current journal cursor to a file. + /// + /// This method saves the current journal cursor to a file so that the journal can be resumed from the + /// same point after a restart. + fn save_cursor(&self) { + let mut cursor: MaybeUninit<*const c_char> = MaybeUninit::uninit(); + let ret = unsafe { sd_journal_get_cursor(self.journal, cursor.as_mut_ptr()) }; + if ret < 0 { + warn!("Failed to get journal cursor: {}", ret); + } else { + let cursor = unsafe { cursor.assume_init() }; + let cursor_cstr = unsafe { std::ffi::CStr::from_ptr(cursor) }; + let cursor_str = cursor_cstr.to_str().unwrap_or_default(); + + let write_result = std::fs::write(&self.cursor_file, cursor_str); + unsafe { + free(cursor as *mut libc::c_void); + } + match write_result { + Ok(_) => (), + Err(e) => warn!( + "Failed to write journal cursor to {:?}: {}", + self.cursor_file, e + ), + } + } + } + + pub fn get_timestamp(&self) -> Result> { + let mut timestamp = 0u64; + let ret = unsafe { sd_journal_get_realtime_usec(self.journal, &mut timestamp) }; + if ret < 0 { + return Err(eyre!("Failed to get journal entry timestamp: {}", ret)); + } + + let datetime = NaiveDateTime::from_timestamp_micros(timestamp as i64) + .ok_or_else(|| eyre!("Failed to convert journal timestamp to DateTime"))?; + + Ok(DateTime::::from_utc(datetime, Utc)) + } +} + +impl Drop for JournalRawImpl { + fn drop(&mut self) { + self.save_cursor(); + unsafe { + libc::free(self.journal as *mut libc::c_void); + } + } +} + +impl JournalRaw for JournalRawImpl { + fn next_entry_available(&mut self) -> Result { + let ret = unsafe { sd_journal_next(self.journal) }; + match ret { + ret if ret < 0 => Err(eyre!("Failed to get next journal entry: {}", ret)), + 0 => Ok(false), + _ => Ok(true), + } + } + + fn get_entry_field_data(&mut self) -> Result> { + let mut data = MaybeUninit::uninit(); + let mut data_len = MaybeUninit::uninit(); + let mut fields = Vec::new(); + + let timestamp = self.get_timestamp().map_or_else( + |e| { + debug!( + "Failed to get journal entry timestamp, falling back to ingestion time: {}", + e + ); + Utc::now() + }, + |t| t, + ); + + let mut enum_ret = unsafe { + sd_journal_enumerate_data(self.journal, data.as_mut_ptr(), data_len.as_mut_ptr()) + }; + + while enum_ret > 0 { + let bytes = + unsafe { std::slice::from_raw_parts(data.assume_init(), data_len.assume_init()) }; + + let kv_string = String::from_utf8_lossy(bytes).to_string(); + fields.push(kv_string); + + enum_ret = unsafe { + sd_journal_enumerate_data(self.journal, data.as_mut_ptr(), data_len.as_mut_ptr()) + }; + } + + if enum_ret < 0 { + Err(eyre!("Failed to read journal entry data: {}", enum_ret)) + } else { + Ok(Some(JournalEntryRaw::new(fields, timestamp))) + } + } + + fn wait_for_entry(&mut self) -> Result<()> { + let mut fds = [self.wait_fd]; + let ret = poll(&mut fds, Self::POLL_TIMEOUT_MS)?; + if ret < 0 { + return Err(eyre!("Failed to poll for journal entry: {}", ret)); + } + + // This call clears the queue status of the poll fd + let ret = unsafe { sd_journal_process(self.journal) }; + if ret < 0 { + return Err(eyre!("Failed to process journal entry: {}", ret)); + } + + Ok(()) + } +} + +/// A fully parsed journal entry. +/// +/// This struct represents a fully parsed journal entry, with all key-value pairs parsed into a HashMap. +#[derive(Serialize, Debug)] +pub struct JournalEntry { + pub ts: DateTime, + pub fields: HashMap, +} + +impl From for JournalEntry { + fn from(raw: JournalEntryRaw) -> Self { + let fields = raw + .fields + .into_iter() + .fold(HashMap::new(), |mut acc, field| { + let kv: Vec<&str> = field.splitn(2, '=').collect(); + if kv.len() == 2 { + acc.insert(kv[0].to_string(), kv[1].to_string()); + } + acc + }); + + Self { ts: raw.ts, fields } + } +} + +impl From for LogEntry { + fn from(entry: JournalEntry) -> Self { + let ts = entry.ts; + let data = entry + .fields + .into_iter() + .fold(HashMap::new(), |mut acc, (k, v)| { + acc.insert(k, LogValue::String(v)); + acc + }); + + LogEntry { ts, data } + } +} + +/// A wrapper around the `JournalRaw` trait that provides an iterator over journal entries. +/// +/// This struct provides an iterator over journal entries, abstracting the raw journal reading logic. +pub struct Journal { + journal: J, +} + +impl Journal { + pub fn new(journal: J) -> Self { + Self { journal } + } + + /// Get the next journal entry. + /// + /// This function will return a `JournalEntry` if the next entry is available, or `None` if there are no more entries. + /// Returns an error if there was an issue reading the journal. + fn next_entry(&mut self) -> Result> { + match self.journal.next_entry_available()? { + false => Ok(None), + true => { + let raw = self.journal.get_entry_field_data()?; + Ok(raw.map(|raw| raw.into())) + } + } + } + + /// Get an iterator over all available journal entries. + pub fn iter(&mut self) -> impl Iterator + '_ { + std::iter::from_fn(move || match self.next_entry() { + Ok(entry) => entry, + Err(e) => { + warn!("Failed to get next journal entry: {}", e); + None + } + }) + } + + pub fn wait_for_entry(&mut self) -> Result<()> { + self.journal.wait_for_entry() + } +} + +#[cfg(test)] +mod test { + use super::*; + + use insta::{assert_json_snapshot, with_settings}; + use mockall::Sequence; + + #[test] + fn test_from_raw_journal_entry() { + let raw_entry = raw_journal_entry(); + let entry = JournalEntry::from(raw_entry); + + with_settings!({sort_maps => true}, { + assert_json_snapshot!(entry); + }); + } + + #[test] + fn test_journal_happy_path() { + let mut raw_journal = MockJournalRaw::new(); + let mut seq = Sequence::new(); + + raw_journal + .expect_next_entry_available() + .times(1) + .in_sequence(&mut seq) + .returning(|| Ok(true)); + + raw_journal + .expect_get_entry_field_data() + .times(1) + .in_sequence(&mut seq) + .returning(|| Ok(Some(raw_journal_entry()))); + + raw_journal + .expect_next_entry_available() + .times(1) + .in_sequence(&mut seq) + .returning(|| Ok(false)); + + let mut journal = Journal::new(raw_journal); + let mut journal_iter = journal.iter(); + let entry = journal_iter.next(); + assert!(entry.is_some()); + with_settings!({sort_maps => true}, { + assert_json_snapshot!(entry.unwrap()); + }); + let entry = journal_iter.next(); + assert!(entry.is_none()); + } + + fn timestamp() -> DateTime { + let timestamp = NaiveDateTime::from_timestamp_millis(1713462571).unwrap(); + DateTime::::from_utc(timestamp, Utc) + } + + fn raw_journal_entry() -> JournalEntryRaw { + let fields = [ + "_SYSTEMD_UNIT=user@1000.service", + "MESSAGE=audit: type=1400 audit(1713462571.968:7508): apparmor=\"DENIED\" operation=\"open\" class=\"file\" profile=\"snap.firefox.firefox\" name=\"/etc/fstab\" pid=10122 comm=\"firefox\" requested_mask=\"r\" denied_mask=\"r\" fsuid=1000 ouid=0", + ]; + + JournalEntryRaw::new(fields.iter().map(|s| s.to_string()).collect(), timestamp()) + } +} diff --git a/memfaultd/src/logs/journald_provider.rs b/memfaultd/src/logs/journald_provider.rs new file mode 100644 index 0000000..55b8713 --- /dev/null +++ b/memfaultd/src/logs/journald_provider.rs @@ -0,0 +1,173 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::logs::journald_parser::{Journal, JournalRaw, JournalRawImpl}; + +use eyre::{eyre, Result}; +use log::{error, warn}; + +use std::thread::spawn; +use std::{ + path::PathBuf, + sync::mpsc::{sync_channel, Receiver, SyncSender}, +}; + +use super::log_entry::LogEntry; + +const ENTRY_CHANNEL_SIZE: usize = 1024; + +/// A log provider that reads log entries from Journald and sends them to a receiver. +pub struct JournaldLogProvider { + journal: Journal, + entry_sender: SyncSender, +} + +impl JournaldLogProvider { + pub fn new(journal: J, entry_sender: SyncSender) -> Self { + Self { + journal: Journal::new(journal), + entry_sender, + } + } + + fn run_once(&mut self) -> Result<()> { + for entry in self.journal.iter() { + let mut log_entry = LogEntry::from(entry); + // TODO: Add support for filtering additional fields + log_entry.filter_fields(&[]); + + if let Err(e) = self.entry_sender.send(log_entry) { + return Err(eyre!("Journald channel dropped: {}", e)); + } + } + Ok(()) + } + + pub fn start(&mut self) -> Result<()> { + loop { + // Block until another entry is available + self.journal.wait_for_entry()?; + + self.run_once()?; + } + } +} + +/// A receiver for log entries from Journald. +pub struct JournaldLogReceiver { + entry_receiver: Receiver, +} + +impl JournaldLogReceiver { + pub fn new(entry_receiver: Receiver) -> Self { + Self { entry_receiver } + } +} + +impl Iterator for JournaldLogReceiver { + type Item = LogEntry; + + fn next(&mut self) -> Option { + match self.entry_receiver.recv() { + Ok(v) => Some(v), + Err(e) => { + warn!("Failed to receive entry: {}", e); + None + } + } + } +} + +/// Start a Journald log provider and return a receiver for the log entries. +/// +/// This function will start a new thread that reads log entries from Journald and sends them to the +/// returned receiver. It takes in the temporary storage path to use as the location of storing the +/// cursor file. +pub fn start_journald_provider(tmp_path: PathBuf) -> JournaldLogReceiver { + let (entry_sender, entry_receiver) = sync_channel(ENTRY_CHANNEL_SIZE); + + spawn(move || { + let journal_raw = JournalRawImpl::new(tmp_path); + let mut provider = JournaldLogProvider::new(journal_raw, entry_sender); + if let Err(e) = provider.start() { + error!("Journald provider failed: {}", e); + } + }); + + JournaldLogReceiver::new(entry_receiver) +} + +#[cfg(test)] +mod test { + use chrono::{DateTime, NaiveDateTime, Utc}; + use insta::{assert_json_snapshot, with_settings}; + use mockall::Sequence; + + use super::*; + + use crate::logs::journald_parser::{JournalEntryRaw, MockJournalRaw}; + + #[test] + fn test_happy_path() { + let mut journal_raw = MockJournalRaw::new(); + let mut seq = Sequence::new(); + + let (sender, receiver) = sync_channel(ENTRY_CHANNEL_SIZE); + + journal_raw + .expect_next_entry_available() + .times(1) + .in_sequence(&mut seq) + .returning(|| Ok(true)); + journal_raw + .expect_get_entry_field_data() + .returning(|| Ok(Some(raw_journal_entry()))); + journal_raw + .expect_next_entry_available() + .times(1) + .in_sequence(&mut seq) + .returning(|| Ok(false)); + + let mut provider = JournaldLogProvider::new(journal_raw, sender); + + assert!(provider.run_once().is_ok()); + let entry = receiver.try_recv().unwrap(); + with_settings!({sort_maps => true}, { + assert_json_snapshot!(entry); + }); + } + + #[test] + fn test_channel_dropped() { + let mut journal_raw = MockJournalRaw::new(); + let mut seq = Sequence::new(); + + let (sender, receiver) = sync_channel(1); + drop(receiver); + + journal_raw + .expect_next_entry_available() + .times(1) + .in_sequence(&mut seq) + .returning(|| Ok(true)); + journal_raw + .expect_get_entry_field_data() + .returning(|| Ok(Some(raw_journal_entry()))); + + let mut provider = JournaldLogProvider::new(journal_raw, sender); + + assert!(provider.run_once().is_err()); + } + + fn raw_journal_entry() -> JournalEntryRaw { + let fields = [ + "_SYSTEMD_UNIT=user@1000.service", + "MESSAGE=audit: type=1400 audit(1713462571.968:7508): apparmor=\"DENIED\" operation=\"open\" class=\"file\" profile=\"snap.firefox.firefox\" name=\"/etc/fstab\" pid=10122 comm=\"firefox\" requested_mask=\"r\" denied_mask=\"r\" fsuid=1000 ouid=0", + ]; + + let timestamp = NaiveDateTime::from_timestamp_millis(1337).unwrap(); + let timestamp = DateTime::::from_utc(timestamp, Utc); + + JournalEntryRaw::new(fields.iter().map(|s| s.to_string()).collect(), timestamp) + } +} diff --git a/memfaultd/src/logs/log_collector.rs b/memfaultd/src/logs/log_collector.rs new file mode 100644 index 0000000..b896353 --- /dev/null +++ b/memfaultd/src/logs/log_collector.rs @@ -0,0 +1,755 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Collect logs into log files and save them as MAR entries. +//! +use std::io::Cursor; +use std::num::NonZeroU32; +use std::sync::Arc; +use std::time::Duration; +use std::{fs, thread}; +use std::{path::PathBuf, sync::Mutex}; + +use eyre::{eyre, Context, Result}; +use flate2::Compression; +use log::{error, trace, warn}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tiny_http::{Header, Method, Request, Response, ResponseBox, StatusCode}; + +use crate::util::rate_limiter::RateLimiter; +use crate::{config::Config, metrics::MetricReportManager}; +use crate::{config::LogToMetricRule, logs::completed_log::CompletedLog}; +use crate::{config::StorageConfig, http_server::ConvenientHeader}; +use crate::{ + http_server::HttpHandler, + logs::log_file::{LogFile, LogFileControl, LogFileControlImpl}, +}; +use crate::{http_server::HttpHandlerResult, logs::recovery::recover_old_logs}; +use crate::{logs::headroom::HeadroomCheck, util::circular_queue::CircularQueue}; + +pub const CRASH_LOGS_URL: &str = "/api/v1/crash-logs"; + +use super::log_entry::LogEntry; +#[cfg(feature = "log-to-metrics")] +use super::log_to_metrics::LogToMetrics; + +pub struct LogCollector { + inner: Arc>>>, +} + +impl LogCollector { + /// This value is used to clamp the number of lines captured in a coredump. + /// + /// This is done to prevent the coredump from becoming too large. The value was chosen + /// arbitrarily to be large enough to capture a reasonable amount of logs, but small enough + /// to prevent the coredump from becoming too large. The current default is 100 lines. + const MAX_IN_MEMORY_LINES: usize = 500; + + /// Create a new log collector and open a new log file for writing. + /// The on_log_completion callback will be called when a log file is completed. + /// This callback must move (or delete) the log file! + pub fn open Result<()> + Send + 'static>( + log_config: LogCollectorConfig, + mut on_log_completion: R, + headroom_limiter: H, + #[cfg_attr(not(feature = "log-to-metrics"), allow(unused_variables))] + heartbeat_manager: Arc>, + ) -> Result { + fs::create_dir_all(&log_config.log_tmp_path).wrap_err_with(|| { + format!( + "Unable to create directory to store in-progress logs: {}", + log_config.log_tmp_path.display() + ) + })?; + + // Collect any leftover logfiles in the tmp folder + let next_cid = recover_old_logs(&log_config.log_tmp_path, &mut on_log_completion)?; + + let in_memory_lines = if log_config.in_memory_lines > Self::MAX_IN_MEMORY_LINES { + warn!( + "Too many lines captured in coredump ({}), clamping to {}", + log_config.in_memory_lines, + Self::MAX_IN_MEMORY_LINES + ); + Self::MAX_IN_MEMORY_LINES + } else { + log_config.in_memory_lines + }; + Ok(Self { + inner: Arc::new(Mutex::new(Some(Inner { + log_file_control: LogFileControlImpl::open( + log_config.log_tmp_path, + next_cid, + log_config.log_max_size, + log_config.log_max_duration, + log_config.log_compression_level, + on_log_completion, + )?, + rate_limiter: RateLimiter::new(log_config.max_lines_per_minute), + headroom_limiter, + #[cfg(feature = "log-to-metrics")] + log_to_metrics: LogToMetrics::new( + log_config.log_to_metrics_rules, + heartbeat_manager, + ), + log_queue: CircularQueue::new(in_memory_lines), + storage_config: log_config.storage_config, + }))), + }) + } + + /// Spawn a thread to read log records from receiver. + pub fn spawn_collect_from + Send + 'static>(&self, source: T) { + // Clone the atomic reference counting "pointer" (not the inner struct itself) + let c = self.inner.clone(); + + thread::spawn(move || { + for line in source { + match c.lock() { + Ok(mut inner_opt) => { + match &mut *inner_opt { + Some(inner) => { + let log_val = match serde_json::to_value(line) { + Ok(val) => val, + Err(e) => { + warn!("Error converting log to json: {}", e); + continue; + } + }; + if let Err(e) = inner.process_log_record(log_val) { + warn!("Error writing log: {:?}", e); + } + } + // log_collector has shutdown. exit the thread cleanly. + None => return, + } + } + Err(e) => { + // This should never happen but we are unable to recover from this so bail out. + error!("Log collector got into an unrecoverable state: {}", e); + std::process::exit(-1); + } + } + } + trace!("Log collection thread shutting down - Channel closed"); + }); + } + + /// Get a handler for the /api/v1/crash-logs endpoint + pub fn crash_log_handler(&self) -> CrashLogHandler { + CrashLogHandler::new(self.inner.clone()) + } + + /// Force the log_collector to close the current log and generate a MAR entry. + pub fn flush_logs(&mut self) -> Result<()> { + self.with_mut_inner(|inner| inner.log_file_control.rotate_unless_empty().map(|_| ())) + } + + /// Rotate the logs if needed + pub fn rotate_if_needed(&mut self) -> Result { + self.with_mut_inner(|inner| inner.rotate_if_needed()) + } + + /// Try to get the inner log_collector or return an error + fn with_mut_inner) -> Result>(&mut self, fun: F) -> Result { + let mut inner_opt = self + .inner + .lock() + // This should never happen so we choose to panic in this case. + .expect("Fatal: log_collector mutex is poisoned."); + + match &mut *inner_opt { + Some(inner) => fun(inner), + None => Err(eyre!("Log collector has already shutdown.")), + } + } + + /// Close and dispose of the inner log collector. + /// This is not public because it does not consume self (to be compatible with drop()). + fn close_internal(&mut self) -> Result<()> { + match self.inner.lock() { + Ok(mut inner_opt) => { + match (*inner_opt).take() { + Some(inner) => inner.log_file_control.close(), + None => { + // Already closed. + Ok(()) + } + } + } + Err(_) => { + // Should never happen. + panic!("Log collector is poisoned.") + } + } + } +} + +impl Drop for LogCollector { + fn drop(&mut self) { + if let Err(e) = self.close_internal() { + warn!("Error closing log collector: {}", e); + } + } +} + +/// The log collector keeps one Inner struct behind a Arc> so it can be +/// shared by multiple threads. +struct Inner { + // We use an Option here because we have no typed-guarantee that every + // log message will include a `ts` key. + rate_limiter: RateLimiter>, + log_file_control: LogFileControlImpl, + headroom_limiter: H, + #[cfg(feature = "log-to-metrics")] + log_to_metrics: LogToMetrics, + log_queue: CircularQueue, + storage_config: StorageConfig, +} + +impl Inner { + // Process one log record - To call this, the caller must have acquired a + // mutex on the Inner object. + // Be careful to not try to acquire other mutexes here to avoid a + // dead-lock. Everything we need should be in Inner. + fn process_log_record(&mut self, log: Value) -> Result<()> { + let log_timestamp = log.get("ts"); + + #[cfg(feature = "log-to-metrics")] + if let Err(e) = self.log_to_metrics.process(&log) { + warn!("Error processing log to metrics: {:?}", e); + } + + if !self + .headroom_limiter + .check(log_timestamp, &mut self.log_file_control)? + { + return Ok(()); + } + self.log_queue.push(log.clone()); + + // Return early and do not write a log message to file if not persisting + if !self.should_persist() { + return Ok(()); + } + + // Rotate before writing (in case log file is now too old) + self.log_file_control.rotate_if_needed()?; + + let logfile = self.log_file_control.current_log()?; + self.rate_limiter + .run_within_limits(log_timestamp.cloned(), |rate_limited_calls| { + // Print a message if some previous calls were rate limited. + if let Some(limited) = rate_limited_calls { + logfile.write_log( + limited.latest_call, + format!("Memfaultd rate limited {} messages.", limited.count), + )?; + } + logfile.write_json_line(log)?; + Ok(()) + })?; + + // Rotate after writing (in case log file is now too large) + self.log_file_control.rotate_if_needed()?; + Ok(()) + } + + fn should_persist(&self) -> bool { + matches!(self.storage_config, StorageConfig::Persist) + } + + fn rotate_if_needed(&mut self) -> Result { + self.log_file_control.rotate_if_needed() + } + + pub fn get_log_queue(&mut self) -> Result> { + let logs = self.log_queue.iter().map(|v| v.to_string()).collect(); + + Ok(logs) + } +} + +pub struct LogCollectorConfig { + /// Folder where to store logfiles while they are being written + pub log_tmp_path: PathBuf, + + /// Files will be rotated when they reach this size (so they may be slightly larger) + log_max_size: usize, + + /// MAR entry will be rotated when they get this old. + log_max_duration: Duration, + + /// Compression level to use for compressing the logs. + log_compression_level: Compression, + + /// Maximum number of lines written per second continuously + max_lines_per_minute: NonZeroU32, + + /// Rules to convert logs to metrics + #[cfg_attr(not(feature = "log-to-metrics"), allow(dead_code))] + log_to_metrics_rules: Vec, + + /// Maximum number of lines to keep in memory + in_memory_lines: usize, + + /// Whether or not to persist log lines + storage_config: StorageConfig, +} + +impl From<&Config> for LogCollectorConfig { + fn from(config: &Config) -> Self { + Self { + log_tmp_path: config.logs_path(), + log_max_size: config.config_file.logs.rotate_size, + log_max_duration: config.config_file.logs.rotate_after, + log_compression_level: config.config_file.logs.compression_level, + max_lines_per_minute: config.config_file.logs.max_lines_per_minute, + log_to_metrics_rules: config + .config_file + .logs + .log_to_metrics + .as_ref() + .map(|c| c.rules.clone()) + .unwrap_or_default(), + in_memory_lines: config.config_file.coredump.log_lines, + storage_config: config.config_file.logs.storage, + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +/// A list of crash logs. +/// +/// This structure is passed to the client when they request the crash logs. +pub struct CrashLogs { + pub logs: Vec, +} + +/// A handler for the /api/v1/crash-logs endpoint. +pub struct CrashLogHandler { + inner: Arc>>>, +} + +impl CrashLogHandler { + fn new(inner: Arc>>>) -> Self { + Self { inner } + } + + /// Handle a GET request to /api/v1/crash-logs + /// + /// Will take a snapshot of the current circular queue and return it as a JSON array. + fn handle_get_crash_logs(&self) -> Result { + let logs = self + .inner + .lock() + .expect("Log collector mutex poisoned") + .as_mut() + .ok_or_else(|| eyre!("Log collector has already shutdown."))? + .get_log_queue()?; + let crash_logs = CrashLogs { logs }; + + let serialized_logs = serde_json::to_string(&crash_logs)?; + let logs_len = serialized_logs.as_bytes().len(); + Ok(Response::new( + StatusCode(200), + vec![Header::from_strings("Content-Type", "application/json")?], + Cursor::new(serialized_logs), + Some(logs_len), + None, + ) + .boxed()) + } +} + +impl HttpHandler for CrashLogHandler { + fn handle_request(&self, request: &mut Request) -> HttpHandlerResult { + if request.url() == CRASH_LOGS_URL { + match *request.method() { + Method::Get => self.handle_get_crash_logs().into(), + _ => HttpHandlerResult::Response(Response::empty(405).boxed()), + } + } else { + HttpHandlerResult::NotHandled + } + } +} + +#[cfg(test)] +mod tests { + use std::cmp::min; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::mpsc::{channel, Receiver}; + use std::sync::Arc; + use std::{fs::remove_file, sync::Mutex}; + use std::{io::Write, path::PathBuf, time::Duration}; + use std::{mem::replace, num::NonZeroU32}; + + use crate::logs::log_file::{LogFile, LogFileControl}; + use crate::test_utils::setup_logger; + use crate::{logs::completed_log::CompletedLog, metrics::MetricReportManager}; + use crate::{logs::headroom::HeadroomCheck, util::circular_queue::CircularQueue}; + use eyre::Context; + use flate2::Compression; + use rstest::{fixture, rstest}; + use serde_json::{json, Value}; + use tempfile::{tempdir, TempDir}; + use tiny_http::{Method, TestRequest}; + use uuid::Uuid; + + use super::*; + + const IN_MEMORY_LINES: usize = 100; + + #[rstest] + fn write_logs_to_disk(mut fixture: LogFixture) { + fixture.write_log(json!({"ts": 0, "MESSAGE": "xxx"})); + assert_eq!(fixture.count_log_files(), 1); + assert_eq!(fixture.on_log_completion_calls(), 0); + } + + #[rstest] + #[case(50)] + #[case(100)] + #[case(150)] + fn circular_log_queue(#[case] mut log_count: usize, mut fixture: LogFixture) { + for i in 0..log_count { + fixture.write_log(json!({"ts": i, "MESSAGE": "xxx"})); + } + + let log_queue = fixture.get_log_queue(); + + // Assert that the last value in the queue has the correct timestamp + let last_val = log_queue.back().unwrap(); + let ts = last_val.get("ts").unwrap().as_u64().unwrap(); + assert_eq!(ts, log_count as u64 - 1); + + // Clamp the log_count to the maximum size of the queue + log_count = min(log_count, IN_MEMORY_LINES); + assert_eq!(log_queue.len(), log_count); + } + + #[rstest] + fn clamp_coredump_log_count(fixture: LogFixture) { + let config = LogCollectorConfig { + log_tmp_path: fixture.logs_dir.path().to_owned(), + log_max_size: 1024, + log_max_duration: Duration::from_secs(3600), + log_compression_level: Compression::default(), + max_lines_per_minute: NonZeroU32::new(1_000).unwrap(), + log_to_metrics_rules: vec![], + in_memory_lines: 1000, + storage_config: StorageConfig::Persist, + }; + + let mut collector = LogCollector::open( + config, + |CompletedLog { path, .. }| { + remove_file(&path) + .with_context(|| format!("rm {path:?}")) + .unwrap(); + Ok(()) + }, + StubHeadroomLimiter, + Arc::new(Mutex::new(MetricReportManager::new())), + ) + .unwrap(); + + let log_queue = collector + .with_mut_inner(|inner| Ok(replace(&mut inner.log_queue, CircularQueue::new(1000)))) + .unwrap(); + + // The log queue should be clamped to the maximum size + assert_eq!( + log_queue.capacity(), + LogCollector::::MAX_IN_MEMORY_LINES + ); + } + + #[rstest] + fn do_not_create_newfile_on_close(mut fixture: LogFixture) { + fixture.write_log(json!({"ts": 0, "MESSAGE": "xxx"})); + fixture.collector.close_internal().expect("error closing"); + // 0 because the fixture "on_log_completion" moves the file out + assert_eq!(fixture.count_log_files(), 0); + assert_eq!(fixture.on_log_completion_calls(), 1); + } + + #[rstest] + #[case(StorageConfig::Persist, 25)] + #[case(StorageConfig::Disabled, 0)] + fn log_persistence( + #[case] storage_config: StorageConfig, + #[case] expected_size: usize, + mut fixture: LogFixture, + _setup_logger: (), + ) { + fixture.set_log_config(storage_config); + + fixture.write_log(json!({"ts": 0, "MESSAGE": "xxx"})); + fixture.flush_log_writes().unwrap(); + + assert_eq!(fixture.count_log_files(), 1); + assert_eq!(fixture.read_log_len(), expected_size); + } + + #[rstest] + fn forced_rotation_with_nonempty_log(mut fixture: LogFixture) { + fixture.write_log(json!({"ts": 0, "MESSAGE": "xxx"})); + + fixture.collector.flush_logs().unwrap(); + + assert_eq!(fixture.count_log_files(), 0); + assert_eq!(fixture.on_log_completion_calls(), 1); + } + + #[rstest] + fn delete_log_after_failed_on_completion_callback(mut fixture: LogFixture) { + fixture + .on_completion_should_fail + .store(true, Ordering::Relaxed); + fixture.write_log(test_line()); + + fixture.collector.flush_logs().unwrap(); + + assert_eq!(fixture.on_log_completion_calls(), 1); + + // The old log should have been deleted, to avoid accumulating logs that fail to be moved. + // No new file will be created without a subsequent write + assert_eq!(fixture.count_log_files(), 0); + } + + #[rstest] + fn forced_rotation_with_empty_log(mut fixture: LogFixture) { + fixture.collector.flush_logs().unwrap(); + + assert_eq!(fixture.count_log_files(), 0); + assert_eq!(fixture.on_log_completion_calls(), 0); + } + + #[rstest] + fn forced_rotation_with_write_after_rotate(mut fixture: LogFixture) { + fixture.write_log(test_line()); + fixture.collector.flush_logs().unwrap(); + + fixture.write_log(test_line()); + assert_eq!(fixture.count_log_files(), 1); + assert_eq!(fixture.on_log_completion_calls(), 1); + } + + #[rstest] + fn recover_old_logfiles() { + let (tmp_logs, _old_file_path) = existing_tmplogs_with_log(&(Uuid::new_v4().to_string())); + let fixture = collector_with_logs_dir(tmp_logs); + + // We should have generated a MAR entry for the pre-existing logfile. + assert_eq!(fixture.on_log_completion_calls(), 1); + } + + #[rstest] + fn delete_files_that_are_not_uuids() { + let (tmp_logs, old_file_path) = existing_tmplogs_with_log("testfile"); + let fixture = collector_with_logs_dir(tmp_logs); + + // And we should have removed the bogus file + assert!(!old_file_path.exists()); + + // We should NOT have generated a MAR entry for the pre-existing bogus file. + assert_eq!(fixture.on_log_completion_calls(), 0); + } + + #[rstest] + fn http_handler_log_get(mut fixture: LogFixture) { + let logs = vec![ + json!({"ts": 0, "MESSAGE": "xxx"}), + json!({"ts": 1, "MESSAGE": "yyy"}), + json!({"ts": 2, "MESSAGE": "zzz"}), + ]; + let log_strings = logs.iter().map(|l| l.to_string()).collect::>(); + + for log in &logs { + fixture.write_log(log.clone()); + } + + let inner = fixture.collector.inner.clone(); + let handler = CrashLogHandler::new(inner); + + let log_response = handler.handle_get_crash_logs().unwrap(); + let mut log_response_string = String::new(); + log_response + .into_reader() + .read_to_string(&mut log_response_string) + .unwrap(); + + let crash_logs: CrashLogs = serde_json::from_str(&log_response_string).unwrap(); + assert_eq!(crash_logs.logs, log_strings); + } + + #[rstest] + #[case(Method::Post)] + #[case(Method::Put)] + #[case(Method::Delete)] + #[case(Method::Patch)] + fn http_handler_unsupported_method(fixture: LogFixture, #[case] method: Method) { + let inner = fixture.collector.inner.clone(); + let handler = CrashLogHandler::new(inner); + + let request = TestRequest::new() + .with_path(CRASH_LOGS_URL) + .with_method(method); + let response = handler + .handle_request(&mut request.into()) + .expect("Error handling request"); + assert_eq!(response.status_code().0, 405); + } + + #[rstest] + fn unhandled_url(fixture: LogFixture) { + let inner = fixture.collector.inner.clone(); + let handler = CrashLogHandler::new(inner); + + let request = TestRequest::new().with_path("/api/v1/other"); + let response = handler.handle_request(&mut request.into()); + assert!(matches!(response, HttpHandlerResult::NotHandled)); + } + + fn existing_tmplogs_with_log(filename: &str) -> (TempDir, PathBuf) { + let tmp_logs = tempdir().unwrap(); + let file_path = tmp_logs + .path() + .to_path_buf() + .join(filename) + .with_extension("log.zlib"); + + let mut file = std::fs::File::create(&file_path).unwrap(); + file.write_all(b"some content in the log").unwrap(); + drop(file); + (tmp_logs, file_path) + } + + struct LogFixture { + collector: LogCollector, + // TempDir needs to be after the collector, otherwise we fail to delete + // the file in LogCollector::Drop because the tempdir is gone + logs_dir: TempDir, + on_log_completion_receiver: Receiver<(PathBuf, Uuid)>, + on_completion_should_fail: Arc, + } + impl LogFixture { + fn count_log_files(&self) -> usize { + std::fs::read_dir(&self.logs_dir).unwrap().count() + } + + fn write_log(&mut self, line: Value) { + self.collector + .with_mut_inner(|inner| inner.process_log_record(line)) + .unwrap(); + } + + fn read_log_len(&mut self) -> usize { + self.collector + .with_mut_inner(|inner| { + let log = inner.log_file_control.current_log()?; + Ok(log.bytes_written()) + }) + .unwrap() + } + + fn flush_log_writes(&mut self) -> Result<()> { + self.collector + .with_mut_inner(|inner| inner.log_file_control.current_log()?.flush()) + } + + fn on_log_completion_calls(&self) -> usize { + self.on_log_completion_receiver.try_iter().count() + } + + fn get_log_queue(&mut self) -> CircularQueue { + self.collector + .with_mut_inner(|inner| Ok(replace(&mut inner.log_queue, CircularQueue::new(100)))) + .unwrap() + } + + fn set_log_config(&mut self, storage_config: StorageConfig) { + self.collector + .with_mut_inner(|inner| { + inner.storage_config = storage_config; + Ok(()) + }) + .unwrap() + } + } + + #[fixture] + fn fixture() -> LogFixture { + collector_with_logs_dir(tempdir().unwrap()) + } + + struct StubHeadroomLimiter; + + impl HeadroomCheck for StubHeadroomLimiter { + fn check( + &mut self, + _log_timestamp: Option<&Value>, + _log_file_control: &mut impl LogFileControl, + ) -> eyre::Result { + Ok(true) + } + } + + fn collector_with_logs_dir(logs_dir: TempDir) -> LogFixture { + let config = LogCollectorConfig { + log_tmp_path: logs_dir.path().to_owned(), + log_max_size: 1024, + log_max_duration: Duration::from_secs(3600), + log_compression_level: Compression::default(), + max_lines_per_minute: NonZeroU32::new(1_000).unwrap(), + log_to_metrics_rules: vec![], + in_memory_lines: IN_MEMORY_LINES, + storage_config: StorageConfig::Persist, + }; + + let (on_log_completion_sender, on_log_completion_receiver) = channel(); + + let on_completion_should_fail = Arc::new(AtomicBool::new(false)); + + let heartbeat_manager = Arc::new(Mutex::new(MetricReportManager::new())); + + let collector = { + let on_completion_should_fail = on_completion_should_fail.clone(); + let on_log_completion = move |CompletedLog { path, cid, .. }| { + on_log_completion_sender.send((path.clone(), cid)).unwrap(); + if on_completion_should_fail.load(Ordering::Relaxed) { + // Don't move / unlink the log file. The LogCollector should clean up now. + Err(eyre::eyre!("on_log_completion failure!")) + } else { + // Unlink the log file. The real implementation moves it into the MAR staging area. + remove_file(&path) + .with_context(|| format!("rm {path:?}")) + .unwrap(); + Ok(()) + } + }; + + LogCollector::open( + config, + on_log_completion, + StubHeadroomLimiter, + heartbeat_manager, + ) + .unwrap() + }; + + LogFixture { + logs_dir, + collector, + on_log_completion_receiver, + on_completion_should_fail, + } + } + + fn test_line() -> Value { + json!({"ts": 0, "MESSAGE": "xxx"}) + } +} diff --git a/memfaultd/src/logs/log_entry.rs b/memfaultd/src/logs/log_entry.rs new file mode 100644 index 0000000..70f66c1 --- /dev/null +++ b/memfaultd/src/logs/log_entry.rs @@ -0,0 +1,87 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::collections::HashMap; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::util::serialization::datetime_to_rfc3339; + +// journal fields that should always be captured by memfaultd: +// https://man7.org/linux/man-pages/man7/systemd.journal-fields.7.html +const ALWAYS_INCLUDE_KEYS: &[&str] = &["MESSAGE", "_PID", "_SYSTEMD_UNIT", "PRIORITY"]; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(untagged)] +pub enum LogValue { + String(String), + Float(f64), +} + +/// Represents a structured log that could come from a variety of sources. +#[derive(Debug, Serialize, Deserialize)] +pub struct LogEntry { + #[serde(with = "datetime_to_rfc3339")] + pub ts: DateTime, + pub data: HashMap, +} + +impl LogEntry { + /// Filter log fields to only include defaults and those specified in `extra_fields`. + /// + /// This function modifies the log entry in place by removing fields that are not in the + /// `ALWAYS_INCLUDE_KEYS` list or in `extra_fields`. This is useful for reducing the size of + /// log entries sent to Memfault, as there are fields that are not useful or displayed. + pub fn filter_fields(&mut self, extra_fields: &[String]) { + self.data + .retain(|k, _| ALWAYS_INCLUDE_KEYS.contains(&k.as_str()) || extra_fields.contains(k)); + } +} + +#[cfg(test)] +mod tests { + use chrono::{DateTime, TimeZone, Utc}; + use insta::{assert_json_snapshot, with_settings}; + use rstest::{fixture, rstest}; + + use super::*; + + #[rstest] + #[case("empty", "{}", "")] + #[case("only_message", r#"{"MESSAGE":"TEST" }"#, "")] + #[case("extra_key", r#"{"MESSAGE":"TEST", "SOME_EXTRA_KEY":"XX" }"#, "")] + #[case( + "multi_key_match", + r#"{"MESSAGE":"TEST", "SOME_EXTRA_KEY":"XX", "_PID": "44", "_SYSTEMD_UNIT": "some.service", "PRIORITY": "6" }"#, + "" + )] + #[case( + "extra_attribute_filter", + r#"{"MESSAGE":"TEST", "SOME_EXTRA_KEY":"XX" }"#, + "SOME_EXTRA_KEY" + )] + fn test_filtering( + time: DateTime, + #[case] test_name: String, + #[case] input: String, + #[case] extras: String, + ) { + let mut entry = LogEntry { + ts: time, + data: serde_json::from_str(&input).unwrap(), + }; + + let extra_attributes = extras.split(',').map(String::from).collect::>(); + entry.filter_fields(&extra_attributes); + + with_settings!({sort_maps => true}, { + assert_json_snapshot!(test_name, entry); + }); + } + + #[fixture] + fn time() -> DateTime { + Utc.timestamp_millis_opt(1334250000000).unwrap() + } +} diff --git a/memfaultd/src/logs/log_file.rs b/memfaultd/src/logs/log_file.rs new file mode 100644 index 0000000..23ce6d9 --- /dev/null +++ b/memfaultd/src/logs/log_file.rs @@ -0,0 +1,267 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Contains LogFile and LogFileControl traits and their real implementations. +//! +use crate::logs::completed_log::CompletedLog; +use crate::mar::CompressionAlgorithm; +use eyre::{eyre, Result, WrapErr}; +use flate2::write::ZlibEncoder; +use flate2::Compression; +use log::{trace, warn}; +use serde_json::{json, Value}; +use std::fs::{remove_file, File}; +use std::io::{BufWriter, Write}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; +use uuid::Uuid; + +pub trait LogFile { + fn write_json_line(&mut self, json: Value) -> Result<()>; + fn write_log>(&mut self, ts: Option, msg: S) -> Result<()> { + self.write_json_line(json!({ + "ts": ts, + "data": { "MESSAGE": msg.as_ref() } + })) + } + fn flush(&mut self) -> Result<()>; +} + +/// In memory representation of one logfile while it is being written to. +pub struct LogFileImpl { + cid: Uuid, + path: PathBuf, + writer: BufWriter>, + bytes_written: usize, + since: Instant, +} + +impl LogFileImpl { + fn open(log_tmp_path: &Path, cid: Uuid, compression_level: Compression) -> Result { + let filename = cid.to_string() + ".log.zlib"; + let path = log_tmp_path.join(filename); + let file = File::create(&path)?; + let writer = BufWriter::new(ZlibEncoder::new(file, compression_level)); + + trace!("Now writing logs to: {}", path.display()); + Ok(LogFileImpl { + cid, + path, + writer, + bytes_written: 0, + since: Instant::now(), + }) + } + + #[cfg(test)] + pub fn bytes_written(&self) -> usize { + self.bytes_written + } +} + +impl LogFile for LogFileImpl { + fn write_json_line(&mut self, json: Value) -> Result<()> { + let bytes = serde_json::to_vec(&json)?; + let mut written = self.writer.write(&bytes)?; + written += self.writer.write("\n".as_bytes())?; + self.bytes_written += written; + Ok(()) + } + + fn flush(&mut self) -> Result<()> { + self.writer.flush().wrap_err("Flush error") + } +} + +pub trait LogFileControl { + fn rotate_if_needed(&mut self) -> Result; + fn rotate_unless_empty(&mut self) -> Result<()>; + fn current_log(&mut self) -> Result<&mut L>; + fn close(self) -> Result<()>; +} + +/// Controls the creation and rotation of logfiles. +pub struct LogFileControlImpl { + current_log: Option, + tmp_path: PathBuf, + max_size: usize, + max_duration: Duration, + compression_level: Compression, + on_log_completion: Box<(dyn FnMut(CompletedLog) -> Result<()> + Send)>, + next_cid: Uuid, +} + +impl LogFileControlImpl { + pub fn open Result<()> + Send + 'static>( + tmp_path: PathBuf, + next_cid: Uuid, + max_size: usize, + max_duration: Duration, + compression_level: Compression, + on_log_completion: R, + ) -> Result { + Ok(LogFileControlImpl { + current_log: None, + tmp_path, + max_size, + max_duration, + compression_level, + on_log_completion: Box::new(on_log_completion), + next_cid, + }) + } + + /// Close current logfile, create a MAR entry and starts a new one. + fn rotate_log(&mut self) -> Result<()> { + let current_log = self.current_log.take(); + + if let Some(current_log) = current_log { + self.next_cid = Uuid::new_v4(); + + Self::dispatch_on_log_completion( + &mut self.on_log_completion, + current_log, + self.next_cid, + ); + } + + Ok(()) + } + + fn dispatch_on_log_completion( + on_log_completion: &mut Box<(dyn FnMut(CompletedLog) -> Result<()> + Send)>, + mut log: LogFileImpl, + next_cid: Uuid, + ) { + // Drop the old log, finishing the compression, closing the buffered writer and the file. + log.writer.flush().unwrap_or_else(|e| { + warn!("Failed to flush logs: {}", e); + }); + + let LogFileImpl { path, cid, .. } = log; + + // The callback is responsible for moving the file to its final location (or deleting it): + (on_log_completion)(CompletedLog { + path: path.clone(), + cid, + next_cid, + compression: CompressionAlgorithm::Zlib, + }) + .unwrap_or_else(|e| { + warn!( + "Dropping log due to failed on_log_completion callback: {}", + e + ); + remove_file(&path).unwrap_or_else(|e| { + warn!("Failed to remove log file: {}", e); + }); + }); + } +} + +impl LogFileControl for LogFileControlImpl { + fn rotate_if_needed(&mut self) -> Result { + if let Some(current_log) = &mut self.current_log { + if current_log.bytes_written >= self.max_size + || current_log.since.elapsed() > self.max_duration + { + self.rotate_log()?; + Ok(true) + } else { + Ok(false) + } + } else { + Ok(false) + } + } + + fn rotate_unless_empty(&mut self) -> Result<()> { + if let Some(current_log) = &self.current_log { + if current_log.bytes_written > 0 { + self.rotate_log()?; + } + } + Ok(()) + } + + fn current_log(&mut self) -> Result<&mut LogFileImpl> { + if self.current_log.is_none() { + self.current_log = Some( + LogFileImpl::open(&self.tmp_path, self.next_cid, self.compression_level) + .map_err(|e| eyre!("Failed to open log file: {e}"))?, + ); + } + + // NOTE: The error case should not be possible here as it is always set above. + // still this is better than panicking. + self.current_log + .as_mut() + .ok_or_else(|| eyre!("No current log")) + } + + fn close(mut self) -> Result<()> { + if let Some(current_log) = self.current_log { + if current_log.bytes_written > 0 { + Self::dispatch_on_log_completion( + &mut self.on_log_completion, + current_log, + Uuid::new_v4(), + ); + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::io::Read; + + use super::*; + use flate2::bufread::ZlibDecoder; + use rand::distributions::{Alphanumeric, DistString}; + use rstest::rstest; + use tempfile::tempdir; + + // We saw this bug when we tried switching to the rust-based backend for flate2 (miniz-oxide) + // With miniz 0.7.1 and flate2 1.0.28, this test does not pass. + #[rstest] + fn test_write_without_corruption() { + let tmp = tempdir().expect("tmpdir"); + + // Generate a logfile with lots of bogus data + let mut log = LogFileImpl::open(tmp.path(), Uuid::new_v4(), Compression::fast()) + .expect("open log error"); + let mut count_lines = 0; + while log.bytes_written < 1024 * 1024 { + let message = format!( + "bogus {} bogum {} bodoum", + Alphanumeric.sample_string(&mut rand::thread_rng(), 16), + Alphanumeric.sample_string(&mut rand::thread_rng(), 20), + ); + log.write_json_line(json!({ "data": { "message": message, "prio": 42, "unit": "systemd"}, "ts": "2023-22-22T22:22:22Z"})).expect("error writing json line"); + count_lines += 1; + } + + let logfile = log.path.clone(); + drop(log); + + // Decompress without error + let bytes = std::fs::read(&logfile).expect("Unable to read {filename}"); + let mut z = ZlibDecoder::new(&bytes[..]); + let mut loglines = String::new(); + z.read_to_string(&mut loglines).expect("read error"); + + // Check we have all the lines + assert_eq!(count_lines, loglines.lines().count()); + + // Check all lines are valid json + let mut count_invalid_lines = 0; + for line in loglines.lines() { + if serde_json::from_str::(line).is_err() { + count_invalid_lines += 1; + } + } + assert_eq!(count_invalid_lines, 0); + } +} diff --git a/memfaultd/src/logs/log_to_metrics.rs b/memfaultd/src/logs/log_to_metrics.rs new file mode 100644 index 0000000..708e8cc --- /dev/null +++ b/memfaultd/src/logs/log_to_metrics.rs @@ -0,0 +1,204 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; + +use eyre::Result; +use log::{debug, warn}; +use regex::Regex; +use serde_json::{Map, Value}; + +use crate::{config::LogToMetricRule, metrics::MetricReportManager}; + +const SEARCH_FIELD: &str = "MESSAGE"; + +pub struct LogToMetrics { + rules: Vec, + heartbeat_manager: Arc>, + regex_cache: HashMap, +} + +impl LogToMetrics { + pub fn new( + rules: Vec, + heartbeat_manager: Arc>, + ) -> Self { + Self { + rules, + heartbeat_manager, + regex_cache: HashMap::new(), + } + } + + pub fn process(&mut self, structured_log: &Value) -> Result<()> { + if let Some(data) = structured_log["data"].as_object() { + if !self.rules.is_empty() { + debug!("LogToMetrics: Processing log: {:?}", data); + for rule in &self.rules { + match rule { + LogToMetricRule::CountMatching { + pattern, + metric_name, + filter, + } => Self::apply_count_matching( + data, + pattern, + &mut self.regex_cache, + metric_name, + filter, + self.heartbeat_manager.clone(), + ), + } + } + } + } + Ok(()) + } + + fn get_metric_name_with_captures(metric_name: &str, captures: regex::Captures) -> String { + let mut metric_name_with_captures = metric_name.to_string(); + for (i, capture) in captures.iter().enumerate() { + if let Some(capture) = capture { + metric_name_with_captures = + metric_name_with_captures.replace(&format!("${}", i), capture.as_str()); + } + } + metric_name_with_captures + } + + fn apply_count_matching( + data: &Map, + pattern: &str, + regex_cache: &mut HashMap, + metric_name: &str, + filter: &HashMap, + heartbeat_manager: Arc>, + ) { + // Use filter to quickly disqualify a log entry + for (key, value) in filter { + if let Some(log_value) = data.get(key) { + if log_value != value { + return; + } + } else { + return; + } + } + + let regex = regex_cache + .entry(pattern.to_string()) + .or_insert_with(|| Regex::new(pattern).unwrap()); + if let Some(search_value) = data[SEARCH_FIELD].as_str() { + let captures = regex.captures(search_value); + debug!( + "LogToMetrics Pattern '{}'=> MATCH={} Captures={:?}", + &pattern, + captures.is_some(), + captures + ); + + if let Some(captures) = captures { + let metric_name_with_captures = + Self::get_metric_name_with_captures(metric_name, captures); + + if let Err(e) = heartbeat_manager + .lock() + .unwrap() + .increment_counter(&metric_name_with_captures) + { + warn!("Failed to increment metric: {}", e) + } + } + } + } +} + +#[cfg(test)] +mod tests { + use crate::{metrics::MetricValue, test_utils::setup_logger}; + + use super::*; + use rstest::rstest; + use serde_json::json; + + #[rstest] + #[case(vec![LogToMetricRule::CountMatching { + pattern: "foo".to_string(), + metric_name: "foo".to_string(), + filter: HashMap::default() + }], vec![json!({"MESSAGE": "foo"})], "foo", 1.0)] + #[case(vec![LogToMetricRule::CountMatching { + pattern: "session opened for user (\\w*)\\(uid=".to_string(), + metric_name: "ssh_sessions_$1_count".to_string(), + filter: HashMap::default() + }], vec![json!({"MESSAGE": "pam_unix(sshd:session): session opened for user thomas(uid=1000) by (uid=0)"})], "ssh_sessions_thomas_count", 1.0)] + #[case(vec![LogToMetricRule::CountMatching { + pattern: "(.*): Scheduled restart job, restart counter is at".to_string(), + metric_name: "$1_restarts".to_string(), + filter: HashMap::default() + }], vec![json!({"MESSAGE": /* systemd[1]: */"docker.service: Scheduled restart job, restart counter is at 1."})], "docker.service_restarts", 1.0)] + #[case(vec![LogToMetricRule::CountMatching { + pattern: "(.*): Scheduled restart job, restart counter is at".to_string(), + metric_name: "$1_restarts".to_string(), + filter: HashMap::default() + }], + vec![ + json!({"MESSAGE": /* systemd[1]: */"docker.service: Scheduled restart job, restart counter is at 1."}), + json!({"MESSAGE": /* systemd[1]: */"sshd.service: Scheduled restart job, restart counter is at 1."}), + json!({"MESSAGE": /* systemd[1]: */"docker.service: Scheduled restart job, restart counter is at 2."}), + ], "docker.service_restarts", 2.0) + ] + #[case(vec![LogToMetricRule::CountMatching { + pattern: "(.*): Scheduled restart job, restart counter is at".to_string(), + metric_name: "$1_restarts".to_string(), + filter: HashMap::from([("UNIT".to_owned(), "systemd".to_owned())]) + }], vec![json!({"MESSAGE": /* systemd[1]: */"docker.service: Scheduled restart job, restart counter is at 1.", "UNIT": "systemd"})], "docker.service_restarts", 1.0)] + #[case(vec![LogToMetricRule::CountMatching { + pattern: "(.*): Scheduled restart job, restart counter is at".to_string(), + metric_name: "$1_restarts".to_string(), + filter: HashMap::from([("_SYSTEMD_UNIT".to_owned(), "ssh.service".to_owned())]) + }], vec![json!({"MESSAGE": /* systemd[1]: */"docker.service: Scheduled restart job, restart counter is at 1.", "_SYSTEMD_UNIT": ""})], "docker.service_restarts", 0.0)] + #[case(vec![LogToMetricRule::CountMatching { + pattern: "Out of memory: Killed process \\d+ \\((.*)\\)".to_string(), + metric_name: "oomkill_$1".to_string(), + filter: HashMap::default() + }], vec![json!({"MESSAGE": "Out of memory: Killed process 423 (wefaultd) total-vm:553448kB, anon-rss:284496kB, file-rss:0kB, shmem-rss:0kB, UID:0 pgtables:624kB oom_score_adj:0"})], "oomkill_wefaultd", 1.0)] + + fn test_log_to_metrics( + #[case] rules: Vec, + #[case] logs: Vec, + #[case] metric_name: &str, + #[case] expected_value: f64, + _setup_logger: (), + ) { + let metric_report_manager = Arc::new(Mutex::new(MetricReportManager::new())); + let mut log_to_metrics = LogToMetrics::new(rules, metric_report_manager.clone()); + + for log in logs { + log_to_metrics + .process(&json!({ "data": log })) + .expect("process error"); + } + let metrics = metric_report_manager + .lock() + .unwrap() + .take_heartbeat_metrics(); + + if expected_value == 0.0 { + assert!(!metrics.iter().any(|m| m.0.as_str() == metric_name)); + } else { + let m = metrics + .iter() + .find(|m| m.0.as_str() == metric_name) + .unwrap(); + + match m.1 { + MetricValue::Number(v) => assert_eq!(*v, expected_value), + _ => panic!("This test only expects number metric values!"), + } + } + } +} diff --git a/memfaultd/src/logs/mod.rs b/memfaultd/src/logs/mod.rs new file mode 100644 index 0000000..059788b --- /dev/null +++ b/memfaultd/src/logs/mod.rs @@ -0,0 +1,21 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +pub mod completed_log; +pub use completed_log::CompletedLog; +pub mod fluent_bit_adapter; +pub use fluent_bit_adapter::FluentBitAdapter; +pub mod log_collector; +pub use log_collector::{LogCollector, LogCollectorConfig}; +pub mod headroom; +pub use headroom::HeadroomLimiter; +#[cfg(feature = "systemd")] +mod journald_parser; +#[cfg(feature = "systemd")] +pub mod journald_provider; +pub mod log_entry; +mod log_file; +mod recovery; + +#[cfg(feature = "log-to-metrics")] +mod log_to_metrics; diff --git a/memfaultd/src/logs/recovery.rs b/memfaultd/src/logs/recovery.rs new file mode 100644 index 0000000..b8d95c0 --- /dev/null +++ b/memfaultd/src/logs/recovery.rs @@ -0,0 +1,282 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Recovery of old log files after restarting the memfaultd service. +//! +use std::fs; +use std::fs::remove_file; +use std::iter::{once, zip}; +use std::mem::take; +use std::path::{Path, PathBuf}; + +use crate::logs::completed_log::CompletedLog; +use eyre::Result; +use log::{debug, warn}; +use uuid::Uuid; + +use crate::mar::CompressionAlgorithm; +use crate::util::fs::get_files_sorted_by_mtime; +use crate::util::path::file_prefix; + +struct FileInfo { + path: PathBuf, + uuid: Option, + size: Option, +} + +#[derive(Debug, PartialEq)] +struct LogFileToRecover { + path: PathBuf, + cid: Uuid, + next_cid: Uuid, +} + +#[derive(Debug, PartialEq)] +struct Recovery { + to_delete: Vec, + to_recover: Vec, + next_cid: Uuid, +} + +fn should_recover(file_info: &FileInfo) -> bool { + match file_info { + FileInfo { uuid: None, .. } => false, + FileInfo { size: None, .. } => false, + FileInfo { + size: Some(size), .. + } if *size == 0 => false, + // Only keep files that are not empty and have a valid uuid: + _ => true, + } +} + +/// The "functional core" of the recovery logic. It is pure for unit-testing sake. +/// Note: the file_infos must be sorted by mtime, newest last. +fn get_recovery(file_infos: Vec, gen_uuid: fn() -> Uuid) -> Recovery { + // If the last file was empty we'll delete it, but we want to reuse the CID because it's + // possible the previous file is already uploaded and references that CID: + let last_cid = file_infos + .iter() + .filter_map(|info| match info { + FileInfo { + uuid: Some(uuid), .. + } => match should_recover(info) { + true => Some(gen_uuid()), + false => Some(*uuid), + }, + _ => None, + }) + .last() + .unwrap_or_else(gen_uuid); + + let (mut to_recover_infos, to_delete_infos): (Vec, Vec) = + file_infos.into_iter().partition(should_recover); + + #[allow(clippy::needless_collect)] + let next_cids: Vec = to_recover_infos + .iter() + .skip(1) + .map(|i| i.uuid.unwrap()) + .chain(once(last_cid)) + .collect(); + + Recovery { + to_delete: to_delete_infos.into_iter().map(|info| info.path).collect(), + to_recover: zip(to_recover_infos.iter_mut(), next_cids) + .map(|(info, next_cid)| LogFileToRecover { + path: take(&mut info.path), + cid: info.uuid.unwrap(), + next_cid, + }) + .collect(), + next_cid: last_cid, + } +} + +pub fn recover_old_logs Result<()> + Send + 'static>( + tmp_logs: &Path, + on_log_recovery: &mut R, +) -> Result { + // Make a list of all the info of the files we want to collect, parsing the CID from the + // filename and getting the size on disk: + let file_infos = get_files_sorted_by_mtime(tmp_logs)? + .into_iter() + .map(|path| { + let uuid = file_prefix(&path) + .and_then(|prefix| Uuid::parse_str(&prefix.to_string_lossy()).ok()); + let size = fs::metadata(&path).ok().map(|metadata| metadata.len()); + FileInfo { path, uuid, size } + }) + .collect(); + + let Recovery { + next_cid, + to_delete, + to_recover, + } = get_recovery(file_infos, Uuid::new_v4); + + // Delete any unwanted files from disk: + for path in to_delete { + if let Err(e) = remove_file(&path) { + warn!( + "Unable to delete bogus log file: {} - {}.", + path.display(), + e + ); + } + } + + for LogFileToRecover { + path, + cid, + next_cid, + } in to_recover + { + // Write the MAR entry which will move the logfile + debug!("Recovering logfile: {:?}", path.display()); + + if let Err(e) = (on_log_recovery)(CompletedLog { + path, + cid, + next_cid, + compression: CompressionAlgorithm::Zlib, + }) { + warn!("Unable to recover log file: {}", e); + } + } + + Ok(next_cid) +} + +#[cfg(test)] +mod tests { + use super::*; + + const UUID_A: Uuid = Uuid::from_u128(1); + const UUID_B: Uuid = Uuid::from_u128(2); + const UUID_NEW: Uuid = Uuid::from_u128(3); + + const PATH_A: &str = "/tmp_log/11111111-1111-1111-1111-111111111111"; + const PATH_B: &str = "/tmp_log/22222222-2222-2222-2222-222222222222"; + + #[test] + fn empty_logging_directory() { + let file_infos: Vec = vec![]; + let expected = Recovery { + to_delete: vec![], + to_recover: vec![], + next_cid: UUID_NEW, + }; + assert_eq!(get_recovery(file_infos, gen_uuid), expected); + } + + #[test] + fn delete_improperly_named_files() { + // File w/o proper UUID name should get deleted, even if it's not empty: + let file_infos = vec![FileInfo { + path: PathBuf::from("/tmp_log/foo"), + uuid: None, + size: Some(1), + }]; + let expected = Recovery { + to_delete: vec![PathBuf::from("/tmp_log/foo")], + to_recover: vec![], + next_cid: UUID_NEW, + }; + assert_eq!(get_recovery(file_infos, gen_uuid), expected); + } + + #[test] + fn use_empty_trailing_uuid_named_file_as_next_cid() { + // Any trailing but empty UUID-named file should be used as next_cid: + let file_infos = vec![ + FileInfo { + path: PATH_A.into(), + uuid: Some(UUID_A), + size: Some(1), + }, + FileInfo { + path: PATH_B.into(), + uuid: Some(UUID_B), + size: Some(0), + }, + ]; + let expected = Recovery { + to_delete: vec![PATH_B.into()], + to_recover: vec![LogFileToRecover { + path: PATH_A.into(), + cid: UUID_A, + next_cid: UUID_B, + }], + next_cid: UUID_B, + }; + assert_eq!(get_recovery(file_infos, gen_uuid), expected); + } + + #[test] + fn dont_use_non_trailing_empty_uuid_named_file_as_next_cid() { + // An empty UUID-named file that is not trailing (has newer, non-empty files following) + // should not be used as next_cid: + let file_infos = vec![ + FileInfo { + path: PATH_A.into(), + uuid: Some(UUID_A), + size: Some(0), + }, + FileInfo { + path: PATH_B.into(), + uuid: Some(UUID_B), + size: Some(1), + }, + ]; + let expected = Recovery { + to_delete: vec![PATH_A.into()], + to_recover: vec![LogFileToRecover { + path: PATH_B.into(), + cid: UUID_B, + next_cid: UUID_NEW, + }], + next_cid: UUID_NEW, + }; + assert_eq!(get_recovery(file_infos, gen_uuid), expected); + } + + #[test] + fn chain_cids() { + // Chain CIDs, setting next_cid to the CID of the next file or a newly generated one if it's the + // last file (and it's not empty): + let file_infos = vec![ + FileInfo { + path: PATH_A.into(), + uuid: Some(UUID_A), + size: Some(1), + }, + FileInfo { + path: PATH_B.into(), + uuid: Some(UUID_B), + size: Some(1), + }, + ]; + let expected = Recovery { + to_delete: vec![], + to_recover: vec![ + LogFileToRecover { + path: PATH_A.into(), + cid: UUID_A, + next_cid: UUID_B, + }, + LogFileToRecover { + path: PATH_B.into(), + cid: UUID_B, + next_cid: UUID_NEW, + }, + ], + next_cid: UUID_NEW, + }; + assert_eq!(get_recovery(file_infos, gen_uuid), expected); + } + + fn gen_uuid() -> Uuid { + UUID_NEW + } +} diff --git a/memfaultd/src/logs/snapshots/memfaultd__logs__fluent_bit_adapter__tests__fluent_bit_adapter.snap b/memfaultd/src/logs/snapshots/memfaultd__logs__fluent_bit_adapter__tests__fluent_bit_adapter.snap new file mode 100644 index 0000000..2f0a909 --- /dev/null +++ b/memfaultd/src/logs/snapshots/memfaultd__logs__fluent_bit_adapter__tests__fluent_bit_adapter.snap @@ -0,0 +1,10 @@ +--- +source: memfaultd/src/logs/fluent_bit_adapter.rs +expression: log_entry +--- +{ + "ts": "2012-04-12T17:00:00+00:00", + "data": { + "MESSAGE": "test" + } +} diff --git a/memfaultd/src/logs/snapshots/memfaultd__logs__journald_parser__test__from_raw_journal_entry.snap b/memfaultd/src/logs/snapshots/memfaultd__logs__journald_parser__test__from_raw_journal_entry.snap new file mode 100644 index 0000000..7624f97 --- /dev/null +++ b/memfaultd/src/logs/snapshots/memfaultd__logs__journald_parser__test__from_raw_journal_entry.snap @@ -0,0 +1,11 @@ +--- +source: memfaultd/src/logs/journald_parser.rs +expression: entry +--- +{ + "ts": "1970-01-20T19:57:42.571Z", + "fields": { + "MESSAGE": "audit: type=1400 audit(1713462571.968:7508): apparmor=\"DENIED\" operation=\"open\" class=\"file\" profile=\"snap.firefox.firefox\" name=\"/etc/fstab\" pid=10122 comm=\"firefox\" requested_mask=\"r\" denied_mask=\"r\" fsuid=1000 ouid=0", + "_SYSTEMD_UNIT": "user@1000.service" + } +} diff --git a/memfaultd/src/logs/snapshots/memfaultd__logs__journald_parser__test__journal_happy_path.snap b/memfaultd/src/logs/snapshots/memfaultd__logs__journald_parser__test__journal_happy_path.snap new file mode 100644 index 0000000..ecc66d2 --- /dev/null +++ b/memfaultd/src/logs/snapshots/memfaultd__logs__journald_parser__test__journal_happy_path.snap @@ -0,0 +1,11 @@ +--- +source: memfaultd/src/logs/journald_parser.rs +expression: entry.unwrap() +--- +{ + "ts": "1970-01-20T19:57:42.571Z", + "fields": { + "MESSAGE": "audit: type=1400 audit(1713462571.968:7508): apparmor=\"DENIED\" operation=\"open\" class=\"file\" profile=\"snap.firefox.firefox\" name=\"/etc/fstab\" pid=10122 comm=\"firefox\" requested_mask=\"r\" denied_mask=\"r\" fsuid=1000 ouid=0", + "_SYSTEMD_UNIT": "user@1000.service" + } +} diff --git a/memfaultd/src/logs/snapshots/memfaultd__logs__journald_provider__test__happy_path.snap b/memfaultd/src/logs/snapshots/memfaultd__logs__journald_provider__test__happy_path.snap new file mode 100644 index 0000000..a5e13f5 --- /dev/null +++ b/memfaultd/src/logs/snapshots/memfaultd__logs__journald_provider__test__happy_path.snap @@ -0,0 +1,11 @@ +--- +source: memfaultd/src/logs/journald_provider.rs +expression: entry +--- +{ + "ts": "1970-01-01T00:00:01.337+00:00", + "data": { + "MESSAGE": "audit: type=1400 audit(1713462571.968:7508): apparmor=\"DENIED\" operation=\"open\" class=\"file\" profile=\"snap.firefox.firefox\" name=\"/etc/fstab\" pid=10122 comm=\"firefox\" requested_mask=\"r\" denied_mask=\"r\" fsuid=1000 ouid=0", + "_SYSTEMD_UNIT": "user@1000.service" + } +} diff --git a/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__empty.snap b/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__empty.snap new file mode 100644 index 0000000..603ec3f --- /dev/null +++ b/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__empty.snap @@ -0,0 +1,8 @@ +--- +source: memfaultd/src/logs/log_entry.rs +expression: entry +--- +{ + "ts": "2012-04-12T17:00:00+00:00", + "data": {} +} diff --git a/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__extra_attribute_filter.snap b/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__extra_attribute_filter.snap new file mode 100644 index 0000000..6bb2e0d --- /dev/null +++ b/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__extra_attribute_filter.snap @@ -0,0 +1,11 @@ +--- +source: memfaultd/src/logs/log_entry.rs +expression: entry +--- +{ + "ts": "2012-04-12T17:00:00+00:00", + "data": { + "MESSAGE": "TEST", + "SOME_EXTRA_KEY": "XX" + } +} diff --git a/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__extra_key.snap b/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__extra_key.snap new file mode 100644 index 0000000..57de0a8 --- /dev/null +++ b/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__extra_key.snap @@ -0,0 +1,10 @@ +--- +source: memfaultd/src/logs/log_entry.rs +expression: entry +--- +{ + "ts": "2012-04-12T17:00:00+00:00", + "data": { + "MESSAGE": "TEST" + } +} diff --git a/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__multi_key_match.snap b/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__multi_key_match.snap new file mode 100644 index 0000000..694bbee --- /dev/null +++ b/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__multi_key_match.snap @@ -0,0 +1,13 @@ +--- +source: memfaultd/src/logs/log_entry.rs +expression: entry +--- +{ + "ts": "2012-04-12T17:00:00+00:00", + "data": { + "MESSAGE": "TEST", + "PRIORITY": "6", + "_PID": "44", + "_SYSTEMD_UNIT": "some.service" + } +} diff --git a/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__only_message.snap b/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__only_message.snap new file mode 100644 index 0000000..57de0a8 --- /dev/null +++ b/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__only_message.snap @@ -0,0 +1,10 @@ +--- +source: memfaultd/src/logs/log_entry.rs +expression: entry +--- +{ + "ts": "2012-04-12T17:00:00+00:00", + "data": { + "MESSAGE": "TEST" + } +} diff --git a/memfaultd/src/mar/chunks.rs b/memfaultd/src/mar/chunks.rs new file mode 100644 index 0000000..43f714a --- /dev/null +++ b/memfaultd/src/mar/chunks.rs @@ -0,0 +1,14 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +mod chunk; +mod chunk_header; +mod chunk_message; +mod chunk_wrapper; +mod crc_padded_stream; + +pub use chunk::Chunk; +pub use chunk_message::ChunkMessage; +pub use chunk_message::ChunkMessageType; +pub use chunk_wrapper::ChunkWrapper; +pub use crc_padded_stream::CRCPaddedStream; diff --git a/memfaultd/src/mar/chunks/chunk.rs b/memfaultd/src/mar/chunks/chunk.rs new file mode 100644 index 0000000..014837e --- /dev/null +++ b/memfaultd/src/mar/chunks/chunk.rs @@ -0,0 +1,81 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{Chain, Cursor, Read}; + +use crate::util::io::StreamLen; + +use super::{chunk_header::ChunkHeader, CRCPaddedStream}; + +/// One chunk of a message. +/// This implementation only supports Chunks v2 (with CRC deferred to the end of the last chunk composing the message). +pub struct Chunk { + stream: Chain>, CRCPaddedStream>, +} + +impl Chunk { + /// Create a new single chunk for a message. + pub fn new_single(message: M) -> Self { + Self { + stream: Cursor::new(ChunkHeader::new_single().as_bytes().to_vec()) + .chain(CRCPaddedStream::new(message)), + } + } +} + +impl Read for Chunk { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.stream.read(buf) + } +} + +impl StreamLen for Chunk { + fn stream_len(&self) -> u64 { + self.stream.stream_len() + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::mar::ChunkMessage; + + use super::*; + use std::io::copy; + + #[rstest] + fn test_single_chunk_message() { + // https://docs.memfault.com/docs/mcu/test-patterns-for-chunks-endpoint/#event-message-encoded-in-a-single-chunk + let known_good_chunk = [ + 0x8, 0x2, 0xa7, 0x2, 0x1, 0x3, 0x1, 0x7, 0x6a, 0x54, 0x45, 0x53, 0x54, 0x53, 0x45, + 0x52, 0x49, 0x41, 0x4c, 0xa, 0x6d, 0x74, 0x65, 0x73, 0x74, 0x2d, 0x73, 0x6f, 0x66, + 0x74, 0x77, 0x61, 0x72, 0x65, 0x9, 0x6a, 0x31, 0x2e, 0x30, 0x2e, 0x30, 0x2d, 0x74, + 0x65, 0x73, 0x74, 0x6, 0x6d, 0x74, 0x65, 0x73, 0x74, 0x2d, 0x68, 0x61, 0x72, 0x64, + 0x77, 0x61, 0x72, 0x65, 0x4, 0xa1, 0x1, 0xa1, 0x72, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, + 0x74, 0x65, 0x73, 0x74, 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x1, 0x31, + 0xe4, + ]; + + // Remove 2 bytes chunk header and 2 bytes CRC + let data = known_good_chunk[2..known_good_chunk.len() - 2].to_vec(); + + let mut chunk = Chunk::new_single(ChunkMessage::new( + crate::mar::ChunkMessageType::Event, + Cursor::new(data), + )); + + let mut buf: Cursor> = Cursor::new(vec![]); + assert_eq!( + copy(&mut chunk, &mut buf).expect("copy ok"), + known_good_chunk.len() as u64 + ); + assert_eq!(buf.get_ref().as_slice(), known_good_chunk); + } + + impl StreamLen for std::io::Cursor> { + fn stream_len(&self) -> u64 { + self.get_ref().len() as u64 + } + } +} diff --git a/memfaultd/src/mar/chunks/chunk_header.rs b/memfaultd/src/mar/chunks/chunk_header.rs new file mode 100644 index 0000000..49408b4 --- /dev/null +++ b/memfaultd/src/mar/chunks/chunk_header.rs @@ -0,0 +1,59 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +/// Chunk protocol supports two types of chunks: Init and Cont. +#[derive(Copy, Clone, PartialEq, Eq)] +pub enum ChunkType { + /// First chunk of a message (with `more_data` bit set if there is more data to follow). + Init = 0, + /// All following chunks of a message (with more_data bit set, except for the last one). + #[allow(dead_code)] + Cont = 0x80, +} + +/// Header of one Memfault Chunk +pub struct ChunkHeader { + channel_id: u8, + chunk_type: ChunkType, + more_data: bool, + + // Note: we only support single messages for now so header is always 1 byte. + // We can use Box<[u8]> when we want to support including the length/offset in the header + as_bytes: [u8; 1], +} + +impl ChunkHeader { + const CHUNK_CHANNEL: u8 = 0; + const CHUNK_CHANNEL_MASK: u8 = 0x7; + const CHUNK_MORE_DATA_BIT: u8 = 0x40; + const CHUNK_CRC_DEFERRED_BIT: u8 = 0x08; + + /// Returns a header for one message encoded as a single chunk with deferred CRC. + pub fn new_single() -> Self { + let mut header = Self { + channel_id: Self::CHUNK_CHANNEL, + chunk_type: ChunkType::Init, + more_data: false, + as_bytes: [0], + }; + header.calculate_header_bytes(); + header + } + + fn calculate_header_bytes(&mut self) { + let byte = &mut self.as_bytes[0]; + *byte = 0; + *byte |= self.channel_id & Self::CHUNK_CHANNEL_MASK; + if self.more_data { + *byte |= Self::CHUNK_MORE_DATA_BIT; + } + *byte |= self.chunk_type as u8; + *byte |= Self::CHUNK_CRC_DEFERRED_BIT; + } + + /// Return a byte representation of the header + /// length can vary from 1 to 5 bytes (one byte + one varint) + pub fn as_bytes(&self) -> &[u8] { + &self.as_bytes + } +} diff --git a/memfaultd/src/mar/chunks/chunk_message.rs b/memfaultd/src/mar/chunks/chunk_message.rs new file mode 100644 index 0000000..ebd8ebb --- /dev/null +++ b/memfaultd/src/mar/chunks/chunk_message.rs @@ -0,0 +1,45 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{Chain, Cursor, Read}; + +use crate::util::io::StreamLen; + +/// A one-byte discriminator for the type of message being sent. +#[derive(Copy, Clone, PartialEq, Eq)] +#[allow(dead_code)] +pub enum ChunkMessageType { + Null = 0, + McuCoredump = 1, + Event = 2, + Logs = 3, + CustomDataRecording = 4, + Mar = 5, +} + +/// All data sent in chunks must be wrapped in a ChunkMessage. +/// +/// It adds a one-byte header indicating the type of message being sent. +pub struct ChunkMessage { + stream: Chain, R>, +} + +impl ChunkMessage { + pub fn new(message_type: ChunkMessageType, data: R) -> Self { + Self { + stream: Cursor::new([message_type as u8]).chain(data), + } + } +} + +impl Read for ChunkMessage { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.stream.read(buf) + } +} + +impl StreamLen for ChunkMessage { + fn stream_len(&self) -> u64 { + self.stream.stream_len() + } +} diff --git a/memfaultd/src/mar/chunks/chunk_wrapper.rs b/memfaultd/src/mar/chunks/chunk_wrapper.rs new file mode 100644 index 0000000..0dcd2f0 --- /dev/null +++ b/memfaultd/src/mar/chunks/chunk_wrapper.rs @@ -0,0 +1,43 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{Chain, Cursor, Read}; + +use crate::util::io::StreamLen; + +/// Add a little wrapper around each chunk containing an identifier and the length. +/// This makes it possible to concatenate multiple chunks in one file. +/// +/// Format: +/// - 4 identifier bytes: 'C' 'H' 'N' 'K' +/// - Chunk length (4 bytes, big endian) +/// +/// /!\ This format is not accepted by memfault API. You have to remove that +/// wrapper before sending the chunks to Memfault! +pub struct ChunkWrapper { + stream: Chain>, R>, +} + +impl ChunkWrapper { + pub fn new(chunk: R) -> Self { + let mut header: [u8; 8] = [0; 8]; + header[0..4].copy_from_slice(b"CHNK"); + header[4..8].copy_from_slice(&(chunk.stream_len() as u32).to_be_bytes()); + + Self { + stream: Cursor::new(header.to_vec()).chain(chunk), + } + } +} + +impl Read for ChunkWrapper { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.stream.read(buf) + } +} + +impl StreamLen for ChunkWrapper { + fn stream_len(&self) -> u64 { + self.stream.stream_len() + } +} diff --git a/memfaultd/src/mar/chunks/crc_padded_stream.rs b/memfaultd/src/mar/chunks/crc_padded_stream.rs new file mode 100644 index 0000000..39c6284 --- /dev/null +++ b/memfaultd/src/mar/chunks/crc_padded_stream.rs @@ -0,0 +1,64 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{Cursor, Read}; + +use crc::{Crc, Digest}; +use crc_catalog::CRC_16_XMODEM; +use once_cell::sync::Lazy; + +use crate::util::io::StreamLen; + +static CRC16_XMODEM: Lazy> = Lazy::new(|| Crc::::new(&CRC_16_XMODEM)); +static CRC16_XMODEM_LENGTH: u64 = 2; + +/// A stream which will be followed by a CRC. +pub struct CRCPaddedStream { + stream: R, + /// Keep a running CRC as the data is being read. + /// Will be None once all the stream data has been read. + crc: Option>, + /// Remaining crc bytes to be read. + crc_bytes: Cursor>, +} + +impl CRCPaddedStream { + pub fn new(stream: R) -> Self { + Self { + stream, + crc: Some(CRC16_XMODEM.digest()), + crc_bytes: Cursor::new(vec![]), + } + } +} + +impl Read for CRCPaddedStream { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let result = self.stream.read(buf)?; + + if result > 0 { + if let Some(crc) = &mut self.crc { + crc.update(&buf[..result]); + } else { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + "CRC already finalized", + )); + } + return Ok(result); + } + + // When done reading, write the CRC. + if let Some(crc) = self.crc.take() { + self.crc_bytes = Cursor::new(crc.finalize().to_le_bytes().to_vec()); + } + + self.crc_bytes.read(buf) + } +} + +impl StreamLen for CRCPaddedStream { + fn stream_len(&self) -> u64 { + self.stream.stream_len() + CRC16_XMODEM_LENGTH + } +} diff --git a/memfaultd/src/mar/clean.rs b/memfaultd/src/mar/clean.rs new file mode 100644 index 0000000..25fc279 --- /dev/null +++ b/memfaultd/src/mar/clean.rs @@ -0,0 +1,1072 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::{ + mar::MarEntry, + util::disk_size::{get_disk_space, get_size, DiskSize}, +}; +use eyre::{eyre, Result, WrapErr}; +use log::{debug, trace, warn}; +use std::fmt::{Display, Formatter, Result as FmtResult}; +use std::fs::read_dir; +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime}; + +pub struct MarStagingCleaner { + mar_staging_path: PathBuf, + max_total_size: DiskSize, + min_headroom: DiskSize, + max_age: Duration, +} + +impl MarStagingCleaner { + pub fn new( + mar_staging_path: &Path, + max_total_size: DiskSize, + min_headroom: DiskSize, + max_age: Duration, + ) -> Self { + Self { + mar_staging_path: mar_staging_path.to_owned(), + max_total_size, + min_headroom, + max_age, + } + } + + /// Cleans up MAR entries in the staging folder. + /// Returns the amount of space left until the max_total_size will be reached or until + /// min_headroom will be exceeded (the smallest of the two values). + pub fn clean(&self, required_space: DiskSize) -> Result { + trace!("Cleaning MAR staging area..."); + clean_mar_staging( + &self.mar_staging_path, + self.max_total_size.saturating_sub(required_space), + get_disk_space(&self.mar_staging_path).unwrap_or(DiskSize::ZERO), + self.min_headroom + required_space, + SystemTime::now(), + self.max_age, + ) + .map_err(|e| { + warn!("Unable to clean MAR entries: {}", e); + e + }) + } +} + +#[derive(Debug)] +enum DeletionReason { + Expired, + DiskQuota, +} +impl Display for DeletionReason { + fn fmt(&self, f: &mut Formatter) -> FmtResult { + match self { + DeletionReason::Expired => write!(f, "Expired"), + DeletionReason::DiskQuota => write!(f, "Disk quota"), + } + } +} + +#[derive(Debug, Clone)] +struct AgeSizePath { + age: Duration, + size: DiskSize, + path: PathBuf, +} + +impl AgeSizePath { + fn new( + path: &Path, + size: DiskSize, + timestamp: SystemTime, + reference_date: SystemTime, + ) -> AgeSizePath { + Self { + age: (reference_date.duration_since(timestamp)).unwrap_or(Duration::ZERO), + size, + path: path.to_owned(), + } + } +} + +fn clean_mar_staging( + mar_staging: &Path, + max_total_size: DiskSize, + available_space: DiskSize, + min_space: DiskSize, + reference_date: SystemTime, + max_age: Duration, +) -> Result { + let (entries, total_space_used) = collect_mar_entries(mar_staging, reference_date)?; + + let marked_entries = mark_entries_for_deletion( + entries, + total_space_used, + max_total_size, + available_space, + min_space, + max_age, + ); + + let space_freed = remove_marked_entries(marked_entries); + + let remaining_quota = + max_total_size.saturating_sub(total_space_used.saturating_sub(space_freed)); + let usable_space = (available_space + space_freed).saturating_sub(min_space); + + // Available space to write is the min of bytes and inodes remaining. + Ok(DiskSize::min(remaining_quota, usable_space)) +} + +fn collect_mar_entries( + mar_staging: &Path, + reference_date: SystemTime, +) -> Result<(Vec, DiskSize)> { + let entries: Vec = read_dir(mar_staging) + .wrap_err(eyre!( + "Unable to open MAR staging area: {}", + mar_staging.display() + ))? + .filter_map(|r| r.map_err(|e| warn!("Unable to read DirEntry: {}", e)).ok()) + .map(|dir_entry| match MarEntry::from_path(dir_entry.path()) { + // Use the collection time from the manifest or folder creation time if the manifest cannot be parsed: + Ok(entry) => AgeSizePath::new( + &entry.path, + get_size(&entry.path).unwrap_or(DiskSize::ZERO), + entry.manifest.collection_time.timestamp.into(), + reference_date, + ), + Err(_) => { + let path = dir_entry.path(); + let timestamp = path + .metadata() + .and_then(|m| m.created()) + .unwrap_or_else(|_| SystemTime::now()); + AgeSizePath::new( + &path, + get_size(&path).unwrap_or(DiskSize::ZERO), + timestamp, + reference_date, + ) + } + }) + .collect(); + let total_space_used = entries + .iter() + .fold(DiskSize::ZERO, |space_used, entry| entry.size + space_used); + Ok((entries, total_space_used)) +} + +fn mark_entries_for_deletion( + entries: Vec, + total_space_used: DiskSize, + max_total_size: DiskSize, + available_space: DiskSize, + min_space: DiskSize, + max_age: Duration, +) -> Vec<(AgeSizePath, DeletionReason)> { + // Sort entries with oldest first + let mut entries_by_age = entries; + entries_by_age.sort_by(|a, b| b.age.cmp(&a.age)); + + // Note that (total_space_used).exceeds(max_total_size) would not + // work here because it would be false when only one of bytes/inode is + // exceeded. (we have a test to verify this) + let max_total_size_exceeded = !max_total_size.exceeds(&(total_space_used)); + // Same here, min_space.exceeds(available_space) is sufficient but not necessary. + let min_headroom_exceeded = !available_space.exceeds(&min_space); + + // Calculate how much space needs to be freed based on quotas + let need_to_free = match (max_total_size_exceeded, min_headroom_exceeded) { + (false, false) => DiskSize::ZERO, + (true, false) => total_space_used.saturating_sub(max_total_size), + (false, true) => min_space.saturating_sub(available_space), + // If we have exceeded both quotas, need to free the larger difference + // for both inodes and bytes + (true, true) => DiskSize::max( + total_space_used.saturating_sub(max_total_size), + min_space.saturating_sub(available_space), + ), + }; + + let mut space_to_be_freed = DiskSize::ZERO; + + // Ignore max_age if it is configured to 0 + let delete_expired_entries = !max_age.is_zero(); + + // Since the vector is sorted from oldest to newest, + // older entries will be marked for deletion first + entries_by_age + .into_iter() + .filter_map(|entry| { + if need_to_free != DiskSize::ZERO && !space_to_be_freed.exceeds(&need_to_free) { + space_to_be_freed += entry.size; + Some((entry, DeletionReason::DiskQuota)) + } else if delete_expired_entries && entry.age > max_age { + space_to_be_freed += entry.size; + Some((entry, DeletionReason::Expired)) + } else { + None + } + }) + .collect() +} + +fn remove_marked_entries(marked_entries: Vec<(AgeSizePath, DeletionReason)>) -> DiskSize { + let mut space_freed = DiskSize::ZERO; + for (entry, deletion_reason) in marked_entries { + debug!( + "Cleaning up MAR entry: {} ({} bytes / {} inodes, ~{} seconds old). Deletion reason: {}", + entry.path.display(), + entry.size.bytes, + entry.size.inodes, + entry.age.as_secs(), + deletion_reason + ); + if let Err(e) = std::fs::remove_dir_all(&entry.path) { + warn!("Unable to remove MAR entry: {}", e); + } else { + debug!( + "Removed MAR entry: {} {:?}", + entry.path.display(), + entry.size + ); + space_freed += entry.size; + } + } + space_freed +} + +#[cfg(test)] +mod test { + use crate::mar::test_utils::MarCollectorFixture; + use crate::test_utils::create_file_with_size; + use crate::test_utils::setup_logger; + use rstest::{fixture, rstest}; + + use super::*; + #[rstest] + fn test_collect_entries(mut mar_fixture: MarCollectorFixture) { + let now = SystemTime::now(); + + let path_a = mar_fixture.create_logentry_with_size_and_age(100, now); + let path_b = + mar_fixture.create_logentry_with_size_and_age(100, now + Duration::from_secs(5)); + let path_c = + mar_fixture.create_logentry_with_size_and_age(100, now + Duration::from_secs(10)); + let (entries, _total_space_used) = + collect_mar_entries(&mar_fixture.mar_staging, SystemTime::now()).unwrap(); + assert!(entries.len() == 3); + assert!([path_a, path_b, path_c].iter().all(|path| entries + .iter() + .any(|entry| entry.path.to_str() == path.to_str()))); + } + + #[rstest] + fn test_no_delete_reasons_when_within_quota() { + let now = SystemTime::now(); + + let max_total_size = DiskSize::new_capacity(25000); + let min_headroom = DiskSize { + bytes: 20000, + inodes: 20, + }; + let available_space = DiskSize { + bytes: min_headroom.bytes, + inodes: 100, + }; + + let entries = vec![ + AgeSizePath::new( + Path::new("/mock/a"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now + Duration::from_secs(1), + now, + ), + AgeSizePath::new( + Path::new("/mock/b"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now + Duration::from_secs(10), + now, + ), + AgeSizePath::new( + Path::new("/mock/c"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now + Duration::from_secs(15), + now, + ), + ]; + + let total_space_used = entries + .iter() + .fold(DiskSize::ZERO, |space_used, entry| entry.size + space_used); + + let marked_entries = mark_entries_for_deletion( + entries, + total_space_used, + max_total_size, + available_space, + min_headroom, + Duration::from_secs(0), // No max age for this test + ); + + let do_not_deletes = ["/mock/a", "/mock/b"]; + + assert!(marked_entries.iter().all(|(entry, _reason)| !do_not_deletes + .iter() + .any(|&do_not_delete_path| entry.path.to_str().unwrap() == do_not_delete_path))); + } + + #[rstest] + #[case(DiskSize::new_capacity(2500))] // 1 entry over quota by bytes + #[case(DiskSize {bytes: 10000, inodes: 4})] // 1 entry over quota by inodes + fn test_oldest_marked_when_over_max_total_size(#[case] max_total_size: DiskSize) { + let now = SystemTime::now(); + let entries = vec![ + AgeSizePath::new( + Path::new("/mock/c"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(15), + now, + ), + AgeSizePath::new( + Path::new("/mock/a"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(1), + now, + ), + AgeSizePath::new( + Path::new("/mock/b"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(10), + now, + ), + ]; + + let total_space_used = entries + .iter() + .fold(DiskSize::ZERO, |space_used, entry| entry.size + space_used); + let marked_entries = mark_entries_for_deletion( + entries, + total_space_used, + max_total_size, + DiskSize::new_capacity(10000), + DiskSize::ZERO, // No min headroom quota for this test + Duration::from_secs(0), // No max age for this test + ); + + let do_not_deletes = ["/mock/a", "/mock/b"]; + + assert!(marked_entries.iter().all(|(entry, _reason)| !do_not_deletes + .iter() + .any(|&do_not_delete_path| entry.path.to_str().unwrap() == do_not_delete_path))); + + for (entry, reason) in marked_entries { + if entry.path.to_str().unwrap() == "/mock/c" { + assert!(matches!(reason, DeletionReason::DiskQuota)); + } + } + } + + #[rstest] + #[case(DiskSize {bytes: 3000, inodes: 100}, DiskSize::new_capacity(1500))] // 2 entries over quota by bytes + #[case(DiskSize {bytes: 10000, inodes: 4}, DiskSize {bytes: 10000, inodes: 0})] // 2 entries over quota by inodes + fn test_two_oldest_marked_when_under_min_headroom( + #[case] min_headroom: DiskSize, + #[case] available_space: DiskSize, + ) { + let now = SystemTime::now(); + let entries = vec![ + AgeSizePath::new( + Path::new("/mock/c"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(15), + now, + ), + AgeSizePath::new( + Path::new("/mock/a"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now, + now - Duration::from_secs(1), + ), + AgeSizePath::new( + Path::new("/mock/b"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(10), + now, + ), + ]; + + let total_space_used = entries + .iter() + .fold(DiskSize::ZERO, |space_used, entry| entry.size + space_used); + + let marked_entries = mark_entries_for_deletion( + entries, + total_space_used, + DiskSize::new_capacity(10000), + available_space, + min_headroom, + Duration::from_secs(0), // No max age for this test + ); + + assert!(marked_entries + .iter() + .all(|(entry, _reason)| entry.path.to_str().unwrap() != "/mock/a")); + + for (entry, reason) in marked_entries { + match entry.path.to_str().unwrap() { + "/mock/c" => assert!(matches!(reason, DeletionReason::DiskQuota)), + "/mock/b" => assert!(matches!(reason, DeletionReason::DiskQuota)), + _ => unreachable!(), + } + } + } + + #[rstest] + fn expired_entries_marked() { + let now = SystemTime::now(); + let entries = vec![ + AgeSizePath::new( + Path::new("/mock/c"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(15), + now, + ), + AgeSizePath::new( + Path::new("/mock/a"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(5), + now, + ), + AgeSizePath::new( + Path::new("/mock/b"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(10), + now, + ), + ]; + + let total_space_used = entries + .iter() + .fold(DiskSize::ZERO, |space_used, entry| entry.size + space_used); + let marked_entries = mark_entries_for_deletion( + entries, + total_space_used, + DiskSize::new_capacity(10000), + DiskSize::new_capacity(10000), + DiskSize::ZERO, // No min headroom quota for this test + Duration::from_secs(1), // low max age so all entries are marked as expired + ); + + assert!(marked_entries + .iter() + .all(|(_entry, reason)| matches!(reason, DeletionReason::Expired))); + } + + #[rstest] + // 2 oldest entries need to be deleted to get under quota + // 3rd oldest needs to be deleted because it is expired + // 2 most recent should not be deleted + fn test_marks_quota_and_expired_entries() { + let now = SystemTime::now(); + let max_total_size = DiskSize::new_capacity(3000); + + let entries = vec![ + AgeSizePath::new( + Path::new("/mock/c"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(200), + now, + ), + AgeSizePath::new( + Path::new("/mock/d"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(250), + now, + ), + AgeSizePath::new( + Path::new("/mock/e"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(300), + now, + ), + AgeSizePath::new( + Path::new("/mock/a"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(15), + now, + ), + AgeSizePath::new( + Path::new("/mock/b"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(30), + now, + ), + ]; + + let total_space_used = entries + .iter() + .fold(DiskSize::ZERO, |space_used, entry| entry.size + space_used); + let marked_entries = mark_entries_for_deletion( + entries, + total_space_used, + max_total_size, + DiskSize::new_capacity(10000), + DiskSize::ZERO, + Duration::from_secs(100), + ); + + let do_not_deletes = ["/mock/a", "/mock/b"]; + + assert!(marked_entries.iter().all(|(entry, _reason)| !do_not_deletes + .iter() + .any(|&do_not_delete_path| entry.path.to_str().unwrap() == do_not_delete_path))); + + for (entry, reason) in marked_entries { + match entry.path.to_str().unwrap() { + "/mock/c" => assert!(matches!(reason, DeletionReason::Expired)), + "/mock/d" => assert!(matches!(reason, DeletionReason::DiskQuota)), + "/mock/e" => assert!(matches!(reason, DeletionReason::DiskQuota)), + _ => unreachable!(), + } + } + } + + #[rstest] + // Over the max total space used quota by 2000 bytes + // and under the minimum headroom quota by 6 inodes + // Need to delete 3 entries altogether + fn test_marks_when_different_quotas_exceeded() { + let now = SystemTime::now(); + let max_total_size = DiskSize::new_capacity(3000); + let min_headroom = DiskSize { + bytes: 1024, + inodes: 10, + }; + let available_space = DiskSize { + bytes: 10000, + inodes: 6, + }; + + // 2 oldest entries need to be deleted to get under quota + // 3rd oldest needs to be deleted because it is expired + // 2 most recent should not be deleted + let entries = vec![ + AgeSizePath::new( + Path::new("/mock/c"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(200), + now, + ), + AgeSizePath::new( + Path::new("/mock/d"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(250), + now, + ), + AgeSizePath::new( + Path::new("/mock/e"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(300), + now, + ), + AgeSizePath::new( + Path::new("/mock/a"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(15), + now, + ), + AgeSizePath::new( + Path::new("/mock/b"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(30), + now, + ), + ]; + + let total_space_used = entries + .iter() + .fold(DiskSize::ZERO, |space_used, entry| entry.size + space_used); + let marked_entries = mark_entries_for_deletion( + entries, + total_space_used, + max_total_size, + available_space, + min_headroom, + Duration::from_secs(0), // No max age in this test + ); + + let do_not_deletes = ["/mock/a", "/mock/b"]; + + assert!(marked_entries.iter().all(|(entry, _reason)| !do_not_deletes + .iter() + .any(|&do_not_delete_path| entry.path.to_str().unwrap() == do_not_delete_path))); + for (entry, reason) in marked_entries { + match entry.path.to_str().unwrap() { + "/mock/c" => assert!(matches!(reason, DeletionReason::DiskQuota)), + "/mock/d" => assert!(matches!(reason, DeletionReason::DiskQuota)), + "/mock/e" => assert!(matches!(reason, DeletionReason::DiskQuota)), + _ => unreachable!(), + } + } + } + + #[rstest] + fn empty_staging_area( + mar_fixture: MarCollectorFixture, + max_total_size: DiskSize, + available_space: DiskSize, + min_headroom: DiskSize, + ) { + let size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + SystemTime::now(), + Duration::from_secs(604800), + ) + .unwrap(); + assert_eq!( + size_avail, + DiskSize::min(max_total_size, available_space.saturating_sub(min_headroom)) + ); + } + + #[rstest] + fn keeps_recent_unfinished_mar_entry( + mut mar_fixture: MarCollectorFixture, + max_total_size: DiskSize, + available_space: DiskSize, + min_headroom: DiskSize, + ) { + let path = mar_fixture.create_empty_entry(); + let size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + SystemTime::now(), + Duration::from_secs(604800), + ) + .unwrap(); + assert_eq!( + size_avail, + DiskSize::min(max_total_size, available_space.saturating_sub(min_headroom)) + ); + assert!(path.exists()); + } + + #[rstest] + fn removes_unfinished_mar_entry_exceeding_max_total_size( + mut mar_fixture: MarCollectorFixture, + max_total_size: DiskSize, + available_space: DiskSize, + min_headroom: DiskSize, + ) { + let path = mar_fixture.create_empty_entry(); + + create_file_with_size(&path.join("log.txt"), max_total_size.bytes + 1).unwrap(); + let size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space.saturating_sub(DiskSize { + bytes: 0, + inodes: 1, + }), + min_headroom, + SystemTime::now(), + Duration::from_secs(604800), + ) + .unwrap(); + assert_eq!( + size_avail, + DiskSize::min(max_total_size, available_space.saturating_sub(min_headroom)) + ); + assert!(!path.exists()); + } + + #[rstest] + fn keeps_recent_mar_entry( + mut mar_fixture: MarCollectorFixture, + max_total_size: DiskSize, + available_space: DiskSize, + min_headroom: DiskSize, + ) { + let now = SystemTime::now(); + let path = mar_fixture.create_logentry_with_size_and_age(1, now); + let size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + now, + Duration::from_secs(604800), + ) + .unwrap(); + assert!(max_total_size.exceeds(&size_avail)); + assert!(path.exists()); + } + + #[rstest] + fn removes_mar_entry_exceeding_max_total_size( + mut mar_fixture: MarCollectorFixture, + max_total_size: DiskSize, + available_space: DiskSize, + min_headroom: DiskSize, + ) { + let now = SystemTime::now(); + // NOTE: the entire directory will be larger than max_total_size due to the manifest.json. + let path = mar_fixture.create_logentry_with_size_and_age(max_total_size.bytes, now); + let size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space.saturating_sub(DiskSize { + bytes: 0, + inodes: 2, + }), + min_headroom, + now, + Duration::from_secs(604800), + ) + .unwrap(); + assert_eq!( + size_avail, + DiskSize::min(max_total_size, available_space.saturating_sub(min_headroom)) + ); + assert!(!path.exists()); + } + + #[rstest] + fn removes_mar_entry_exceeding_min_headroom(mut mar_fixture: MarCollectorFixture) { + let now = SystemTime::now(); + let max_total_size = DiskSize::new_capacity(4096); + let min_headroom = DiskSize { + bytes: 1024, + inodes: 10, + }; + let available_space = DiskSize { + bytes: min_headroom.bytes - 1, + inodes: 100, + }; + // NOTE: the entire directory will be larger than 1 byte due to the manifest.json. + let path = mar_fixture.create_logentry_with_size_and_age(1, now); + let size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + now, + Duration::from_secs(604800), + ) + .unwrap(); + assert!(size_avail.bytes >= 1); + assert!(!path.exists()); + } + + #[rstest] + fn removes_oldest_mar_entry_exceeding_max_total_size_when_multiple( + mut mar_fixture: MarCollectorFixture, + ) { + let now = SystemTime::now(); + let max_total_size = DiskSize::new_capacity(23000); + let min_headroom = DiskSize { + bytes: 1024, + inodes: 10, + }; + let available_space = DiskSize { + bytes: max_total_size.bytes * 2, + inodes: 100, + }; + let oldest = + mar_fixture.create_logentry_with_size_and_age(8000, now - Duration::from_secs(120)); + let middle = + mar_fixture.create_logentry_with_size_and_age(8000, now - Duration::from_secs(30)); + let most_recent = mar_fixture.create_logentry_with_size_and_age(8000, now); + let _size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + now, + Duration::from_secs(604800), + ) + .unwrap(); + assert!(!oldest.exists()); + assert!(middle.exists()); + assert!(most_recent.exists()); + } + + #[rstest] + fn removes_entries_exceeding_min_headroom_size_by_age( + mut mar_fixture: MarCollectorFixture, + _setup_logger: (), + ) { + let now = SystemTime::now(); + let max_total_size = DiskSize::new_capacity(80000); + let min_headroom = DiskSize { + bytes: 20000, + inodes: 20, + }; + let available_space = DiskSize { + bytes: min_headroom.bytes - 20000, + inodes: 100, + }; + let oldest = + mar_fixture.create_logentry_with_size_and_age(10000, now - Duration::from_secs(120)); + let second_oldest = + mar_fixture.create_logentry_with_size_and_age(10000, now - Duration::from_secs(30)); + let second_newest = + mar_fixture.create_logentry_with_size_and_age(10000, now - Duration::from_secs(10)); + let most_recent = mar_fixture.create_logentry_with_size_and_age(10000, now); + + // Need to delete 2 entries to free up required headroom + let _size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + now, + Duration::from_secs(604800), + ) + .unwrap(); + assert!(most_recent.exists()); + assert!(second_newest.exists()); + assert!(!second_oldest.exists()); + assert!(!oldest.exists()); + } + + #[rstest] + fn removes_entries_exceeding_max_total_size_by_age( + mut mar_fixture: MarCollectorFixture, + _setup_logger: (), + ) { + let now = SystemTime::now(); + let max_total_size = DiskSize::new_capacity(25000); + let min_headroom = DiskSize { + bytes: 1024, + inodes: 20, + }; + let available_space = DiskSize { + bytes: max_total_size.bytes * 2, + inodes: 100, + }; + let oldest = + mar_fixture.create_logentry_with_size_and_age(10000, now - Duration::from_secs(120)); + let second_oldest = + mar_fixture.create_logentry_with_size_and_age(10000, now - Duration::from_secs(30)); + let second_newest = + mar_fixture.create_logentry_with_size_and_age(10000, now - Duration::from_secs(10)); + let most_recent = mar_fixture.create_logentry_with_size_and_age(10000, now); + let _size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + now, + Duration::from_secs(604800), + ) + .unwrap(); + assert!(second_newest.exists()); + assert!(most_recent.exists()); + assert!(!oldest.exists()); + assert!(!second_oldest.exists()); + } + + #[rstest] + fn removes_mar_entry_exceeding_min_headroom_inodes( + _setup_logger: (), + mut mar_fixture: MarCollectorFixture, + ) { + let now = SystemTime::now(); + let max_total_size = DiskSize::new_capacity(10 * 1024 * 1024); + let min_headroom = DiskSize { + bytes: 1024, + inodes: 10, + }; + let available_space = DiskSize { + bytes: max_total_size.bytes, + inodes: 5, + }; + // NOTE: the entire directory will be larger than 1 byte due to the manifest.json. + let path = mar_fixture.create_logentry_with_size_and_age(1, now); + let size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + now, + Duration::from_secs(604800), + ) + .unwrap(); + assert!(size_avail.bytes >= 1); + assert!(!path.exists()); + } + + #[rstest] + fn removes_mar_entry_exceeding_max_age( + mut mar_fixture: MarCollectorFixture, + max_total_size: DiskSize, + available_space: DiskSize, + min_headroom: DiskSize, + ) { + let now = SystemTime::now(); + let thirty_seconds_ago = now - Duration::from_secs(30); + let path_unexpired = mar_fixture.create_logentry_with_size_and_age(1, thirty_seconds_ago); + let ten_min_ago = now - Duration::from_secs(600); + let path_expired = mar_fixture.create_logentry_with_size_and_age(1, ten_min_ago); + + let size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + now, + Duration::from_secs(60), + ) + .unwrap(); + assert!(size_avail.bytes >= 1); + assert!(path_unexpired.exists()); + assert!(!path_expired.exists()); + } + + #[rstest] + fn keeps_mar_entry_within_max_age( + mut mar_fixture: MarCollectorFixture, + max_total_size: DiskSize, + available_space: DiskSize, + min_headroom: DiskSize, + ) { + let now = SystemTime::now(); + let thirty_seconds_ago = now - Duration::from_secs(30); + let path = mar_fixture.create_logentry_with_size_and_age(1, thirty_seconds_ago); + let size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + now, + Duration::from_secs(60), + ) + .unwrap(); + assert!(size_avail.bytes >= 1); + assert!(path.exists()); + } + + #[rstest] + fn keeps_mar_entry_when_max_age_is_zero( + mut mar_fixture: MarCollectorFixture, + max_total_size: DiskSize, + available_space: DiskSize, + min_headroom: DiskSize, + ) { + let now = SystemTime::now(); + let over_one_week_ago = now - Duration::from_secs(604801); + let path = mar_fixture.create_logentry_with_size_and_age(1, over_one_week_ago); + let size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + now, + Duration::from_secs(0), + ) + .unwrap(); + assert!(size_avail.bytes >= 1); + assert!(path.exists()); + } + + #[fixture] + fn max_total_size() -> DiskSize { + DiskSize::new_capacity(1024) + } + + #[fixture] + fn available_space() -> DiskSize { + DiskSize { + bytes: u64::MAX / 2, + inodes: u64::MAX / 2, + } + } + + #[fixture] + fn min_headroom() -> DiskSize { + DiskSize { + bytes: 0, + inodes: 0, + } + } + + #[fixture] + fn mar_fixture() -> MarCollectorFixture { + MarCollectorFixture::new() + } +} diff --git a/memfaultd/src/mar/export.rs b/memfaultd/src/mar/export.rs new file mode 100644 index 0000000..80334d7 --- /dev/null +++ b/memfaultd/src/mar/export.rs @@ -0,0 +1,421 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::{ + http_server::{ConvenientHeader, HttpHandler, HttpHandlerResult}, + mar::{ + gather_mar_entries_to_zip, Chunk, ChunkMessage, ChunkWrapper, ExportFormat, MarEntry, + MarZipContents, + }, + util::{io::StreamLen, zip::ZipEncoder}, +}; +use eyre::{eyre, Result}; +use log::{debug, trace, warn}; +use std::{ + collections::hash_map::DefaultHasher, + fs::remove_dir_all, + io::BufReader, + path::PathBuf, + sync::{Arc, Mutex}, +}; +use std::{hash::Hasher, os::unix::prelude::OsStrExt}; +use tiny_http::{Header, Method, Request, Response, ResponseBox, StatusCode}; + +pub const EXPORT_MAR_URL: &str = "/v1/export"; + +/// Information on the most recent export that was proposed to a client. +/// We will keep offering the same content until we receive a DELETE call. +struct ExportInfo { + content: MarZipContents, + hash: String, +} +impl ExportInfo { + fn new(content: MarZipContents) -> Self { + let mut hasher = DefaultHasher::new(); + for p in content.entry_paths.iter() { + hasher.write_usize(p.as_os_str().len()); + hasher.write(p.as_os_str().as_bytes()); + } + let hash = hasher.finish().to_string(); + ExportInfo { content, hash } + } + + /// Verifies if the files are still available on disk + fn is_valid(&self) -> bool { + self.content.entry_paths.iter().all(|p| p.exists()) + } +} + +#[derive(Clone)] +/// An `HttpHandler` to manage exporting memfaultd data to local clients. +/// +/// Clients should first make a GET request to download the data and write it to +/// disk. When the file has safely transmitted, they can all DELETE to delete +/// the data secured in the MAR staging directory. +/// +/// For additional security (if multiple clients are accidentally +/// reading/deleting concurrently), we recommend using the If-Match header when +/// calling DELETE and passing the ETag returned by the GET call. This will +/// confirm that the data being deleted is the data that was just saved. +pub struct MarExportHandler { + mar_staging: PathBuf, + current_export: Arc>>, +} + +impl MarExportHandler { + pub fn new(mar_staging: PathBuf) -> Self { + MarExportHandler { + mar_staging, + current_export: Arc::new(Mutex::new(None)), + } + } +} + +const DEFAULT_MAX_ZIP_FILE: usize = 10 * 1024 * 1024; + +impl MarExportHandler { + /// Looks at data in the mar_staging folder and returns content that should be included in next ZIP download + fn prepare_next_export(&self) -> Result> { + let mut entries = MarEntry::iterate_from_container(&self.mar_staging)?; + + let zip_files = gather_mar_entries_to_zip(&mut entries, DEFAULT_MAX_ZIP_FILE); + match zip_files.into_iter().next() { + Some(zip) => Ok(Some(ExportInfo::new(zip))), + None => Ok(None), + } + } + + fn handle_get_mar(&self, request: &Request) -> Result { + let mut export = self + .current_export + .lock() + .map_err(|e| eyre!("Export Mutex poisoned: {:#}", e))?; + + // If we have already prepared a package, make sure the file still exists on disk - or reset the package. + if let Some(false) = (export.as_ref()).map(|export| export.is_valid()) { + *export = None; + } + + // Prepare a new package if needed + if export.is_none() { + *export = self.prepare_next_export()?; + } + + // Parse the accept-header, return 406 NotAcceptable if the requested format is not supported. + let accept_header = request.headers().iter().find(|h| h.field.equiv("Accept")); + let format = match accept_header { + Some(header) => match ExportFormat::from_accept_header(header.value.as_str()) { + Ok(format) => format, + Err(_) => return Ok(Response::empty(406).boxed()), + }, + None => ExportFormat::default(), + }; + + // If we have data to serve, prime the ZIP encoder and stream the data + // Otherwise, return 204. + match &*export { + Some(export) => match format { + ExportFormat::Mar => Self::export_as_zip(export), + ExportFormat::Chunk => Self::export_as_chunk(export), + ExportFormat::ChunkWrapped => Self::export_as_chunk_wrapped(export), + }, + None => Ok(Response::empty(204).boxed()), + } + } + + fn export_as_zip(export: &ExportInfo) -> Result { + let zip_encoder = ZipEncoder::new(export.content.zip_infos.clone()); + let len = zip_encoder.stream_len(); + + Ok(Response::new( + StatusCode(200), + vec![ + Header::from_strings("Content-Type", "application/zip")?, + Header::from_strings("ETag", &format!("\"{}\"", export.hash))?, + ], + BufReader::new(zip_encoder), + Some(len as usize), + None, + ) + .boxed()) + } + + fn export_as_chunk(export: &ExportInfo) -> Result { + let zip_encoder = ZipEncoder::new(export.content.zip_infos.clone()); + + let chunk_stream = Chunk::new_single(ChunkMessage::new( + super::chunks::ChunkMessageType::Mar, + zip_encoder, + )); + + let len = chunk_stream.stream_len(); + + Ok(Response::new( + StatusCode(200), + vec![ + Header::from_strings("Content-Type", ExportFormat::Chunk.to_content_type())?, + Header::from_strings("ETag", &format!("\"{}\"", export.hash))?, + ], + BufReader::new(chunk_stream), + Some(len as usize), + None, + ) + .boxed()) + } + + fn export_as_chunk_wrapped(export: &ExportInfo) -> Result { + let zip_encoder = ZipEncoder::new(export.content.zip_infos.clone()); + + let chunk = ChunkWrapper::new(Chunk::new_single(ChunkMessage::new( + super::chunks::ChunkMessageType::Mar, + zip_encoder, + ))); + let len = chunk.stream_len(); + + Ok(Response::new( + StatusCode(200), + vec![ + Header::from_strings("Content-Type", ExportFormat::Chunk.to_content_type())?, + Header::from_strings("ETag", &format!("\"{}\"", export.hash))?, + ], + BufReader::new(chunk), + Some(len as usize), + None, + ) + .boxed()) + } + + fn handle_delete(&self, request: &Request) -> Result { + let mut export_opt = self + .current_export + .lock() + .map_err(|e| eyre!("Mutex poisoned: {:#}", e))?; + + if let Some(export) = export_opt.as_ref() { + // Optionally, check that the ETag matches (to confirm we are deleting the data client just read). + if let Some(if_match_header) = + request.headers().iter().find(|h| h.field.equiv("If-Match")) + { + if if_match_header.value != export.hash { + debug!( + "Delete error - Wrong hash. Got {}, expected {}", + if_match_header.value, export.hash + ); + return Ok(Response::from_string("Precondition Failed") + .with_status_code(412) + .boxed()); + } + } + + trace!("Deleting MAR entries: {:?}", export.content.entry_paths); + export.content.entry_paths.iter().for_each(|f| { + if let Err(e) = remove_dir_all(f) { + warn!("Error deleting MAR entry: {} ({})", f.display(), e) + } + }); + *export_opt = None; + Ok(Response::empty(204).boxed()) + } else { + trace!("Export delete called but no current content to delete."); + Ok(Response::empty(404).boxed()) + } + } +} + +impl HttpHandler for MarExportHandler { + // TODO: MFLT-11507 Handle locking the mar_cleaner while we are reading it! + fn handle_request(&self, request: &mut Request) -> HttpHandlerResult { + if request.url() == EXPORT_MAR_URL { + match *request.method() { + Method::Get => self.handle_get_mar(request).into(), + Method::Delete => self.handle_delete(request).into(), + _ => HttpHandlerResult::Response(Response::empty(405).boxed()), + } + } else { + HttpHandlerResult::NotHandled + } + } +} + +#[cfg(test)] +mod tests { + use std::{fs::remove_dir_all, str::FromStr}; + + use rstest::{fixture, rstest}; + use tiny_http::{Header, ResponseBox, StatusCode, TestRequest}; + + use crate::{ + http_server::HttpHandler, mar::test_utils::MarCollectorFixture, util::disk_size::get_size, + }; + + use super::{MarExportHandler, EXPORT_MAR_URL}; + + #[rstest] + fn answer_204_when_empty(mut fixture: Fixture) { + let r = fixture.do_download(); + assert_eq!(r.status_code(), StatusCode(204)); + assert_eq!(r.etag(), None); + } + + #[rstest] + fn download_zip(mut fixture: Fixture) { + fixture.mar_fixture.create_logentry_with_size(512); + + let r = fixture.do_download(); + assert_eq!(r.status_code(), StatusCode(200)); + + r.etag().expect("e-tag header should be included"); + + // Files should still be there. + assert!(fixture.count_mar_inodes() > 0); + } + + #[rstest] + fn download_twice(mut fixture: Fixture) { + fixture.mar_fixture.create_logentry_with_size(512); + + let r = fixture.do_download(); + assert_eq!(r.status_code(), StatusCode(200)); + + // Another GET should yield the same response - even if we have added files in between + fixture.mar_fixture.create_logentry_with_size(1024); + let r2 = fixture.do_download(); + + assert_eq!(r2.status_code(), StatusCode(200)); + assert_eq!(r.data_length().unwrap(), r2.data_length().unwrap()); + assert_eq!(r.etag().unwrap(), r2.etag().unwrap()); + } + + #[rstest] + fn download_reset_on_cleanup(mut fixture: Fixture) { + let log1 = fixture.mar_fixture.create_logentry_with_size(512); + + let r = fixture.do_download(); + assert_eq!(r.status_code(), StatusCode(200)); + + // Simulate mar cleaner removing some files + remove_dir_all(log1).expect("delete failed"); + + // Another GET should yield a new response (because the old files are not available anymore) + fixture.mar_fixture.create_logentry_with_size(1024); + let r2 = fixture.do_download(); + + assert_eq!(r2.status_code(), StatusCode(200)); + assert_ne!(r.data_length().unwrap(), r2.data_length().unwrap()); + assert_ne!(r.etag().unwrap(), r2.etag().unwrap()); + } + + #[rstest] + fn files_should_be_deleted_with_etag(mut fixture: Fixture) { + fixture.mar_fixture.create_logentry_with_size(512); + + let r = fixture.do_download(); + assert_eq!(r.status_code(), StatusCode(200)); + + let delete_response = fixture.do_delete(Some(r.etag().unwrap())); + assert_eq!(delete_response.status_code(), StatusCode(204)); + + // Files should have been deleted. + assert_eq!(fixture.count_mar_inodes(), 0); + } + + #[rstest] + fn files_should_be_deleted_without_etag(mut fixture: Fixture) { + fixture.mar_fixture.create_logentry_with_size(512); + + let r = fixture.do_download(); + assert_eq!(r.status_code(), StatusCode(200)); + + let delete_response = fixture.do_delete(None); + assert_eq!(delete_response.status_code(), StatusCode(204)); + + // Files should have been deleted. + assert_eq!(fixture.count_mar_inodes(), 0); + } + + #[rstest] + fn files_should_not_delete_if_etag_does_not_match(mut fixture: Fixture) { + fixture.mar_fixture.create_logentry_with_size(512); + + let r = fixture.do_download(); + assert_eq!(r.status_code(), StatusCode(200)); + + let delete_response = fixture.do_delete(Some("bogus".to_owned())); + assert_eq!(delete_response.status_code(), StatusCode(412)); + + // Files should NOT have been deleted. + assert!(fixture.count_mar_inodes() > 0); + } + + #[rstest] + fn error_404_for_deletes(mut fixture: Fixture) { + fixture.mar_fixture.create_logentry_with_size(512); + + // Not calling download before calling delete + + let delete_response = fixture.do_delete(None); + assert_eq!(delete_response.status_code(), StatusCode(404)); + + // Files should NOT have been deleted. + assert!(fixture.count_mar_inodes() > 0); + } + + struct Fixture { + mar_fixture: MarCollectorFixture, + handler: MarExportHandler, + } + + impl Fixture { + fn do_download(&mut self) -> ResponseBox { + let r = TestRequest::new() + .with_method(tiny_http::Method::Get) + .with_path(EXPORT_MAR_URL); + + self.handler + .handle_request(&mut r.into()) + .expect("should process the request") + } + + fn do_delete(&mut self, hash: Option) -> ResponseBox { + let mut r = TestRequest::new() + .with_method(tiny_http::Method::Delete) + .with_path(EXPORT_MAR_URL); + + if let Some(hash) = hash { + r = r.with_header(Header::from_str(&format!("If-Match: {}", hash)).unwrap()) + } + + self.handler + .handle_request(&mut r.into()) + .expect("should process the request") + } + + fn count_mar_inodes(&self) -> usize { + get_size(&self.mar_fixture.mar_staging) + .expect("count mar files") + .inodes as usize + } + } + + #[fixture] + fn fixture() -> Fixture { + let mar_fixture = MarCollectorFixture::new(); + + Fixture { + handler: MarExportHandler::new(mar_fixture.mar_staging.clone()), + mar_fixture, + } + } + + trait ResponseUtils { + fn etag(&self) -> Option; + } + impl ResponseUtils for ResponseBox { + fn etag(&self) -> Option { + self.headers() + .iter() + .find(|h| h.field.equiv("ETag")) + .map(|header| header.value.as_str().trim_matches('"').to_string()) + } + } +} diff --git a/memfaultd/src/mar/export_format.rs b/memfaultd/src/mar/export_format.rs new file mode 100644 index 0000000..b5674a0 --- /dev/null +++ b/memfaultd/src/mar/export_format.rs @@ -0,0 +1,78 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::{eyre, Result}; +use strum_macros::EnumString; + +#[derive(EnumString, Default, PartialEq, Eq, Debug)] +#[strum(serialize_all = "kebab-case")] +/// Supported export formats +pub enum ExportFormat { + #[default] + /// Default format is a MAR file (zip archive) + Mar, + /// Chunk is a memfault-specific format (it's a container on top of MAR). + /// It's mostly useful when you are already exporting memfault chunks from a + /// MCU device and want the Linux data in the same format. + Chunk, + /// Memfault chunks do not include a header to identify the format. They + /// also do not include the length of the data. This format adds a 'CHNK' + /// header and a length field. + ChunkWrapped, +} + +const CONTENT_TYPE_ZIP: &str = "application/zip"; +const CONTENT_TYPE_CHUNK: &str = "application/vnd.memfault.chunk"; +const CONTENT_TYPE_CHUNK_WRAPPED: &str = "application/vnd.memfault.chunk-wrapped"; + +impl ExportFormat { + pub fn to_content_type(&self) -> &'static str { + match self { + ExportFormat::Mar => CONTENT_TYPE_ZIP, + ExportFormat::Chunk => CONTENT_TYPE_CHUNK, + ExportFormat::ChunkWrapped => CONTENT_TYPE_CHUNK_WRAPPED, + } + } + + fn from_mime_type(value: &str) -> Option { + match value { + "*/*" => Some(Self::Mar), + CONTENT_TYPE_ZIP => Some(Self::Mar), + CONTENT_TYPE_CHUNK => Some(Self::Chunk), + CONTENT_TYPE_CHUNK_WRAPPED => Some(Self::ChunkWrapped), + _ => None, + } + } + + pub fn from_accept_header(value: &str) -> Result { + value + .split(',') + .find_map(|mime_type| { + let mime_type = mime_type.trim(); + Self::from_mime_type(mime_type) + }) + .ok_or_else(|| eyre!("Requested format not supported (Accept: {})", value)) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case("*/*", ExportFormat::Mar)] + #[case(CONTENT_TYPE_ZIP, ExportFormat::Mar)] + #[case(format!("{}, {}", CONTENT_TYPE_ZIP, CONTENT_TYPE_CHUNK), ExportFormat::Mar)] + #[case(format!("{}, {}", CONTENT_TYPE_CHUNK, CONTENT_TYPE_ZIP), ExportFormat::Chunk)] + fn test_accept_header_parser>( + #[case] header_value: H, + #[case] expected_format: ExportFormat, + ) { + assert_eq!( + ExportFormat::from_accept_header(&header_value.into()).unwrap(), + expected_format + ); + } +} diff --git a/memfaultd/src/mar/manifest.rs b/memfaultd/src/mar/manifest.rs new file mode 100644 index 0000000..d7a7fd3 --- /dev/null +++ b/memfaultd/src/mar/manifest.rs @@ -0,0 +1,463 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{collections::HashMap, time::Duration}; + +use chrono::Utc; +use eyre::Result; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use uuid::Uuid; + +use crate::{ + build_info::VERSION, + metrics::MetricStringKey, + metrics::{MetricReportType, MetricValue}, + network::DeviceConfigRevision, + network::NetworkConfig, + reboot::RebootReason, + util::serialization::{milliseconds_to_duration, optional_milliseconds_to_duration}, + util::system::{get_system_clock, read_system_boot_id, Clock}, +}; + +#[derive(Serialize, Deserialize)] +pub struct Manifest { + schema_version: u32, + pub collection_time: CollectionTime, + device: Device, + #[serde(default)] + producer: Producer, + #[serde(flatten)] + pub metadata: Metadata, +} + +#[derive(Serialize, Deserialize)] +pub struct CollectionTime { + pub timestamp: chrono::DateTime, + #[serde(rename = "uptime_ms", with = "milliseconds_to_duration")] + uptime: Duration, + linux_boot_id: Uuid, + #[serde(rename = "elapsed_realtime_ms", with = "milliseconds_to_duration")] + elapsed_realtime: Duration, + // TODO: MFLT-9012 - remove these android only fields + boot_count: u32, +} + +#[derive(Serialize, Deserialize)] +struct Device { + project_key: String, + hardware_version: String, + software_version: String, + software_type: String, + device_serial: String, +} + +#[derive(Serialize, Deserialize)] +pub struct Producer { + pub id: String, + pub version: String, +} + +impl Default for Producer { + fn default() -> Self { + Self { + id: "memfaultd".into(), + version: VERSION.to_owned(), + } + } +} + +#[derive(Serialize, Deserialize, Copy, Clone)] +pub enum CompressionAlgorithm { + None, + #[serde(rename = "zlib")] + Zlib, + #[serde(rename = "gzip")] + Gzip, +} + +impl CompressionAlgorithm { + pub fn is_none(&self) -> bool { + matches!(self, CompressionAlgorithm::None) + } +} + +#[derive(Serialize, Deserialize)] +#[serde(tag = "type", content = "metadata")] +pub enum Metadata { + #[serde(rename = "linux-logs")] + LinuxLogs { + format: LinuxLogsFormat, + // PathBuf.file_name() -> OsString but serde does not handle it well + // so we use a String here. + log_file_name: String, + #[serde(skip_serializing_if = "CompressionAlgorithm::is_none")] + compression: CompressionAlgorithm, + cid: Cid, + next_cid: Cid, + }, + #[serde(rename = "device-attributes")] + DeviceAttributes { attributes: Vec }, + #[serde(rename = "device-config")] + DeviceConfig { revision: DeviceConfigRevision }, + #[serde(rename = "elf-coredump")] + ElfCoredump { + coredump_file_name: String, + #[serde(skip_serializing_if = "CompressionAlgorithm::is_none")] + compression: CompressionAlgorithm, + }, + #[serde(rename = "linux-reboot")] + LinuxReboot { reason: RebootReason }, + // DEPRECATED but need to keep the variant for backwards compatibility + // with MARs produced by earlier SDK versions + #[serde(rename = "linux-heartbeat")] + LinuxHeartbeat { + #[serde(serialize_with = "crate::util::serialization::sorted_map::sorted_map")] + metrics: HashMap, + #[serde( + default, + rename = "duration_ms", + skip_serializing_if = "Option::is_none", + with = "optional_milliseconds_to_duration" + )] + duration: Option, + }, + #[serde(rename = "linux-metric-report")] + LinuxMetricReport { + #[serde(serialize_with = "crate::util::serialization::sorted_map::sorted_map")] + metrics: HashMap, + #[serde(rename = "duration_ms", with = "milliseconds_to_duration")] + duration: Duration, + report_type: MetricReportType, + }, + #[serde(rename = "linux-memfault-watch-logs")] + LinuxMemfaultWatch { + cmdline: Vec, + exit_code: i32, + duration: Duration, + stdio_log_file_name: String, + compression: CompressionAlgorithm, + }, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct LinuxLogsFormat { + id: String, + serialization: String, +} + +impl Default for LinuxLogsFormat { + fn default() -> Self { + Self { + id: "v1".into(), + serialization: "json-lines".into(), + } + } +} + +// Note: Memfault manifest defines Cid as an object containing a Uuid. +#[derive(Serialize, Deserialize)] +pub struct Cid { + uuid: Uuid, +} + +#[derive(Serialize, Deserialize, PartialEq, Eq, Debug)] +pub struct DeviceAttribute { + string_key: MetricStringKey, + value: Value, +} + +impl DeviceAttribute { + pub fn new(string_key: MetricStringKey, value: Value) -> Self { + Self { string_key, value } + } +} + +impl, V: Into> TryFrom<(K, V)> for DeviceAttribute { + type Error = String; + + fn try_from(value: (K, V)) -> std::result::Result { + Ok(DeviceAttribute { + string_key: str::parse(value.0.as_ref())?, + value: value.1.into(), + }) + } +} + +impl Metadata { + pub fn new_coredump(coredump_file_name: String, compression: CompressionAlgorithm) -> Self { + Self::ElfCoredump { + coredump_file_name, + compression, + } + } + + pub fn new_log( + log_file_name: String, + cid: Uuid, + next_cid: Uuid, + compression: CompressionAlgorithm, + ) -> Self { + Self::LinuxLogs { + log_file_name, + compression, + cid: Cid { uuid: cid }, + next_cid: Cid { uuid: next_cid }, + format: LinuxLogsFormat::default(), + } + } + + pub fn new_device_attributes(attributes: Vec) -> Self { + Self::DeviceAttributes { attributes } + } + + pub fn new_device_config(revision: DeviceConfigRevision) -> Self { + Self::DeviceConfig { revision } + } + + pub fn new_reboot(reason: RebootReason) -> Self { + Self::LinuxReboot { reason } + } + + pub fn new_metric_report( + metrics: HashMap, + duration: Duration, + report_type: MetricReportType, + ) -> Self { + Self::LinuxMetricReport { + metrics, + duration, + report_type, + } + } +} + +impl CollectionTime { + pub fn now() -> Result { + Ok(Self { + timestamp: Utc::now(), + linux_boot_id: read_system_boot_id()?, + uptime: get_system_clock(Clock::Monotonic)?, + elapsed_realtime: get_system_clock(Clock::Boottime)?, + // TODO: MFLT-9012 - remove these android only fields + boot_count: 0, + }) + } + + #[cfg(test)] + pub fn test_fixture() -> Self { + use chrono::TimeZone; + use uuid::uuid; + + Self { + timestamp: Utc.timestamp_millis_opt(1334250000000).unwrap(), + uptime: Duration::new(10, 0), + linux_boot_id: uuid!("413554b8-a727-11ed-b307-0317a0ffbea7"), + elapsed_realtime: Duration::new(10, 0), + // TODO: MFLT-9012 - remove these android only fields + boot_count: 0, + } + } +} + +impl From<&NetworkConfig> for Device { + fn from(config: &NetworkConfig) -> Self { + Self { + project_key: config.project_key.clone(), + device_serial: config.device_id.clone(), + hardware_version: config.hardware_version.clone(), + software_type: config.software_type.clone(), + software_version: config.software_version.clone(), + } + } +} + +impl Manifest { + pub fn new( + config: &NetworkConfig, + collection_time: CollectionTime, + metadata: Metadata, + ) -> Self { + Manifest { + collection_time, + device: Device::from(config), + producer: Producer::default(), + schema_version: 1, + metadata, + } + } + + pub fn attachments(&self) -> Vec { + match &self.metadata { + Metadata::ElfCoredump { + coredump_file_name, .. + } => vec![coredump_file_name.clone()], + Metadata::LinuxLogs { log_file_name, .. } => vec![log_file_name.clone()], + Metadata::DeviceAttributes { .. } => vec![], + Metadata::DeviceConfig { .. } => vec![], + Metadata::LinuxHeartbeat { .. } => vec![], + Metadata::LinuxMetricReport { .. } => vec![], + Metadata::LinuxReboot { .. } => vec![], + Metadata::LinuxMemfaultWatch { + stdio_log_file_name, + .. + } => vec![stdio_log_file_name.clone()], + } + } +} + +#[cfg(test)] +impl Metadata { + pub fn test_fixture() -> Self { + Metadata::new_device_config(0) + } +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, path::PathBuf, str::FromStr}; + + use crate::{ + mar::CompressionAlgorithm, + metrics::{MetricReportType, MetricValue}, + reboot::RebootReasonCode, + }; + use rstest::rstest; + use uuid::uuid; + + use crate::network::NetworkConfig; + use crate::reboot::RebootReason; + + use super::{CollectionTime, Manifest}; + + #[rstest] + #[case("coredump-gzip", CompressionAlgorithm::Gzip)] + #[case("coredump-none", CompressionAlgorithm::None)] + fn serialization_of_coredump(#[case] name: &str, #[case] compression: CompressionAlgorithm) { + let config = NetworkConfig::test_fixture(); + + let manifest = Manifest::new( + &config, + CollectionTime::test_fixture(), + super::Metadata::new_coredump("/tmp/core.elf".into(), compression), + ); + insta::assert_json_snapshot!(name, manifest, { ".producer.version" => "tests"}); + } + + #[rstest] + #[case("log-zlib", CompressionAlgorithm::Zlib)] + #[case("log-none", CompressionAlgorithm::None)] + fn serialization_of_log(#[case] name: &str, #[case] compression: CompressionAlgorithm) { + let config = NetworkConfig::test_fixture(); + + let this_cid = uuid!("99686390-a728-11ed-a68b-e7ff3cd0c7e7"); + let next_cid = uuid!("9e1ece10-a728-11ed-918e-5be35a10c7e7"); + let manifest = Manifest::new( + &config, + CollectionTime::test_fixture(), + super::Metadata::new_log("/var/log/syslog".into(), this_cid, next_cid, compression), + ); + insta::assert_json_snapshot!(name, manifest, { ".producer.version" => "tests" }); + } + + #[rstest] + fn serialization_of_device_attributes() { + let config = NetworkConfig::test_fixture(); + let manifest = Manifest::new( + &config, + CollectionTime::test_fixture(), + super::Metadata::new_device_attributes(vec![ + ("my_string", "foo").try_into().unwrap(), + ("my_int", 123).try_into().unwrap(), + ("my_float", 123.456).try_into().unwrap(), + ("my_bool", true).try_into().unwrap(), + ]), + ); + insta::assert_json_snapshot!(manifest, { ".producer.version" => "tests"}); + } + + #[rstest] + fn serialization_of_device_configc() { + let config = NetworkConfig::test_fixture(); + let manifest = Manifest::new( + &config, + CollectionTime::test_fixture(), + super::Metadata::new_device_config(42), + ); + insta::assert_json_snapshot!(manifest, { ".producer.version" => "tests"}); + } + + #[rstest] + fn serialization_of_reboot() { + let config = NetworkConfig::test_fixture(); + let manifest = Manifest::new( + &config, + CollectionTime::test_fixture(), + super::Metadata::new_reboot(RebootReason::from(RebootReasonCode::UserShutdown)), + ); + insta::assert_json_snapshot!(manifest, { ".producer.version" => "tests"}); + } + + #[rstest] + fn serialization_of_custom_reboot() { + let config = NetworkConfig::test_fixture(); + let manifest = Manifest::new( + &config, + CollectionTime::test_fixture(), + super::Metadata::new_reboot( + RebootReason::from_str("CustomRebootReason").unwrap_or_else(|e| panic!("{}", e)), + ), + ); + insta::assert_json_snapshot!(manifest, { ".producer.version" => "tests"}); + } + + #[rstest] + fn serialization_of_custom_unexpected_reboot() { + let config = NetworkConfig::test_fixture(); + let manifest = Manifest::new( + &config, + CollectionTime::test_fixture(), + super::Metadata::new_reboot( + RebootReason::from_str("!CustomUnexpectedRebootReason") + .unwrap_or_else(|e| panic!("{}", e)), + ), + ); + insta::assert_json_snapshot!(manifest, { ".producer.version" => "tests"}); + } + + #[rstest] + fn serialization_of_linux_heartbeat() { + let config = NetworkConfig::test_fixture(); + let manifest = Manifest::new( + &config, + CollectionTime::test_fixture(), + super::Metadata::LinuxMetricReport { + metrics: HashMap::from([ + ("n1".parse().unwrap(), MetricValue::Number(1.0)), + ("n2".parse().unwrap(), MetricValue::Number(42.0)), + ]), + duration: std::time::Duration::from_secs(42), + report_type: MetricReportType::Heartbeat, + }, + ); + insta::assert_json_snapshot!(manifest, { ".producer.version" => "tests"}); + } + + #[rstest] + #[case("heartbeat")] + #[case("heartbeat_with_duration")] + #[case("metric_report")] + #[case("device_config")] + #[case("reboot")] + #[case("attributes")] + #[case("elf_coredump")] + fn can_parse_test_manifests(#[case] name: &str) { + let input_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/mar/test-manifests") + .join(name) + .with_extension("json"); + let manifest_json = std::fs::read_to_string(input_path).unwrap(); + let manifest: Manifest = serde_json::from_str(manifest_json.as_str()).unwrap(); + insta::assert_json_snapshot!(name, manifest, { ".producer.version" => "tests"}); + } +} diff --git a/memfaultd/src/mar/mar_entry.rs b/memfaultd/src/mar/mar_entry.rs new file mode 100644 index 0000000..730cd34 --- /dev/null +++ b/memfaultd/src/mar/mar_entry.rs @@ -0,0 +1,152 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! MAR Entry +//! +//! Represents a MAR entry on disk and provides a parsing utility for the MAR staging area. +//! +//! A MAR entry is a folder with a unique name, a manifest and some optional attachments. +//! +use std::fs::{read_dir, File}; +use std::io::BufReader; +use std::iter::once; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::{collections::VecDeque, time::SystemTime}; + +use eyre::{eyre, Context, Result}; +use uuid::Uuid; + +use super::manifest::Manifest; + +/// A candidate folder for inclusion in a MAR zip file +pub struct MarEntry { + /// Path to the directory on disk where the MAR entry is stored. + pub path: PathBuf, + pub uuid: Uuid, + pub manifest: Manifest, +} + +/// An iterator over a list of paths that may contain a MarEntry. +/// Each is lazily transformed into a MarEntry. +pub struct MarEntryIterator { + directories: VecDeque, +} + +impl Iterator for MarEntryIterator { + type Item = Result; + + fn next(&mut self) -> Option { + self.directories.pop_front().map(MarEntry::from_path) + } + + fn size_hint(&self) -> (usize, Option) { + (self.directories.len(), Some(self.directories.len())) + } +} + +impl MarEntry { + /// Go through all files in our staging area and make a list of paths to valid + /// MAR entries to include in the next MAR file. + /// + /// A valid MAR entry is a directory, named with a valid uuid. and it must + /// contain a manifest.json file. To avoid synchronization issue, writers of + /// MAR entries should write to manifest.lock and rename manifest.json when + /// they are done (atomic operation). + pub fn iterate_from_container( + mar_staging: &Path, + ) -> Result>> { + let mut entries: Vec<(PathBuf, Option)> = read_dir(mar_staging) + .wrap_err(eyre!( + "Unable to open MAR staging area: {}", + mar_staging.display() + ))? + .filter_map(std::io::Result::ok) + // Keep only directories + .filter(|d| d.path().is_dir()) + // Collect the creation time so we can sort them + .map(|d| (d.path(), d.metadata().and_then(|m| m.created()).ok())) + .collect(); + + // Sort entries from oldest to newest + entries.sort_by(|a, b| a.1.cmp(&b.1)); + + Ok(MarEntryIterator { + directories: entries.into_iter().map(|e| e.0).collect(), + }) + } + + /// Creates a MarEntry instance from a directory containing a manifest.json file. + pub fn from_path(path: PathBuf) -> Result { + let uuid = Uuid::from_str( + path.file_name() + .ok_or_else(|| eyre!("{} is not a directory", path.display()))? + .to_str() + .ok_or_else(|| eyre!("{} is not a valid directory name", path.display()))?, + )?; + let manifest = path.join("manifest.json"); + if !manifest.exists() { + return Err(eyre!( + "{} does not contain a manifest file.", + path.display() + )); + } + let buf_reader = + BufReader::new(File::open(manifest).wrap_err("Error reading manifest file")?); + let manifest: Manifest = + serde_json::from_reader(buf_reader).wrap_err("Error parsing manifest file")?; + Ok(Self { + path, + uuid, + manifest, + }) + } + + /// Returns an iterator over the filenames of the manifest.json and attachments of this MAR entry. + pub fn filenames(&self) -> impl Iterator { + once("manifest.json".to_owned()).chain(self.manifest.attachments()) + } +} + +#[cfg(test)] +mod tests { + use rstest::{fixture, rstest}; + + use crate::mar::test_utils::MarCollectorFixture; + use crate::test_utils::setup_logger; + + use super::*; + + #[rstest] + fn collecting_from_empty_folder(_setup_logger: (), mar_fixture: MarCollectorFixture) { + assert_eq!( + MarEntry::iterate_from_container(&mar_fixture.mar_staging) + .unwrap() + .count(), + 0 + ) + } + + #[rstest] + fn collecting_from_folder_with_partial_entries( + _setup_logger: (), + mut mar_fixture: MarCollectorFixture, + ) { + mar_fixture.create_empty_entry(); + mar_fixture.create_logentry(); + + assert_eq!( + MarEntry::iterate_from_container(&mar_fixture.mar_staging) + .unwrap() + .filter(|e| e.is_ok()) + .count(), + // Only one entry should be picked up. The other one is ignored. + 1 + ) + } + + #[fixture] + fn mar_fixture() -> MarCollectorFixture { + MarCollectorFixture::new() + } +} diff --git a/memfaultd/src/mar/mar_entry_builder.rs b/memfaultd/src/mar/mar_entry_builder.rs new file mode 100644 index 0000000..1d0fc30 --- /dev/null +++ b/memfaultd/src/mar/mar_entry_builder.rs @@ -0,0 +1,272 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! MAR Entry Builder +//! +use crate::mar::{CollectionTime, Manifest, MarEntry, Metadata}; +use crate::network::NetworkConfig; +use crate::util::disk_size::DiskSize; +use crate::util::fs::move_file; +use eyre::WrapErr; +use std::fs::{create_dir, remove_dir_all, rename, File}; +use std::mem::take; +use std::path::{Path, PathBuf}; +use uuid::Uuid; + +const MAR_ENTRY_OVERHEAD_SIZE_ESTIMATE: u64 = 4096; + +/// A tool to build new MAR entries. Use one of the constructor functions and +/// call save() to write to disk. Any files attached to this MAR entry will +/// be moved when save is called. +pub struct MarEntryBuilder { + entry_dir: MarEntryDir, + uuid: Uuid, + collection_time: CollectionTime, + metadata: M, + attachments: Vec, +} + +pub struct NoMetadata; + +impl MarEntryBuilder { + fn entry_dir_path(&self) -> &Path { + &self.entry_dir.path + } + + pub fn make_attachment_path_in_entry_dir>(&self, filename: F) -> PathBuf { + self.entry_dir_path().join(filename.as_ref()) + } + + pub fn add_attachment(mut self, file: PathBuf) -> MarEntryBuilder { + assert!(file.is_file()); + assert!(file.is_absolute()); + self.attachments.push(file); + self + } +} + +impl MarEntryBuilder { + pub fn new(mar_staging: &Path) -> eyre::Result> { + let collection_time = CollectionTime::now()?; + + // Create a directory for this entry. Make sure this is the last fallible operation here, + // to avoid complicating cleanup in failure scenarios. + let uuid = Uuid::new_v4(); + let path = mar_staging.to_owned().join(uuid.to_string()); + create_dir(&path)?; + + Ok(Self { + entry_dir: MarEntryDir::new(path), + uuid, + collection_time, + metadata: NoMetadata, + attachments: vec![], + }) + } + + pub fn set_metadata(self, metadata: Metadata) -> MarEntryBuilder { + MarEntryBuilder { + entry_dir: self.entry_dir, + uuid: self.uuid, + collection_time: self.collection_time, + attachments: self.attachments, + metadata, + } + } +} + +impl MarEntryBuilder { + /// Consume this builder, writes the manifest and moves the attachment to the + /// MAR storage area and returns a MAR entry. + pub fn save(self, network_config: &NetworkConfig) -> eyre::Result { + // Move attachments + for filepath in self.attachments { + // We already check that attachments are file in the constructor so we ignore + // non-files here. + if let Some(filename) = filepath.file_name() { + let target = self.entry_dir.path.join(filename); + + // Note: if the attachment path was created using make_attachment_path_in_entry_dir(), + // filepath and target will be the same and this will be a no-op. + move_file(&filepath, &target)?; + } + } + + // Prepare manifest + let manifest = Manifest::new(network_config, self.collection_time, self.metadata); + + // Write the manifest to a temp file + let manifest_path = self.entry_dir.path.join("manifest.tmp"); + let manifest_file = File::create(&manifest_path) + .wrap_err_with(|| format!("Error opening manifest {}", manifest_path.display()))?; + serde_json::to_writer(manifest_file, &manifest)?; + + // Rename the manifest to signal that this folder is complete + let manifest_json_path = manifest_path.with_extension("json"); + rename(&manifest_path, &manifest_json_path).wrap_err_with(|| { + format!( + "Error renaming manifest {} to {}", + manifest_path.display(), + manifest_json_path.display() + ) + })?; + + Ok(MarEntry { + path: self.entry_dir.mark_saved(), + uuid: self.uuid, + manifest, + }) + } + + pub fn estimated_entry_size(&self) -> DiskSize { + let attachments_size_bytes: u64 = self + .attachments + .iter() + .filter_map(|p| p.metadata().ok()) + .map(|m| m.len()) + .sum(); + + // Add a bit extra for the overhead of the manifest.json and directory inode: + DiskSize { + bytes: attachments_size_bytes + MAR_ENTRY_OVERHEAD_SIZE_ESTIMATE, + inodes: self.attachments.len() as u64 + 1, + } + } + + pub fn get_metadata(&self) -> &Metadata { + &self.metadata + } +} + +/// Helper structure that will clean up the entry directory on Drop if mark_saved() was not called. +struct MarEntryDir { + path: PathBuf, + saved: bool, +} + +impl MarEntryDir { + fn new(path: PathBuf) -> Self { + Self { path, saved: false } + } + + fn mark_saved(mut self) -> PathBuf { + self.saved = true; + take(&mut self.path) + } +} + +impl Drop for MarEntryDir { + fn drop(&mut self) { + if !self.saved { + let _ = remove_dir_all(&self.path); + } + } +} + +#[cfg(test)] +mod tests { + use super::MAR_ENTRY_OVERHEAD_SIZE_ESTIMATE; + use crate::mar::MarEntryBuilder; + use crate::mar::Metadata; + use crate::network::NetworkConfig; + use crate::test_utils::create_file_with_size; + use rstest::{fixture, rstest}; + use std::path::PathBuf; + use tempfile::{tempdir, TempDir}; + + #[rstest] + fn cleans_up_entry_dir_when_save_was_not_called(fixture: Fixture) { + let builder = MarEntryBuilder::new(&fixture.mar_staging).unwrap(); + let entry_dir = builder.entry_dir_path().to_owned(); + assert!(entry_dir.exists()); + create_file_with_size(&entry_dir.join("attachment"), 1024).unwrap(); + drop(builder); + assert!(!entry_dir.exists()); + } + + #[rstest] + fn save_keeps_entry_dir_and_adds_manifest_json(fixture: Fixture) { + let mut entry_dir_option = None; + { + let builder = MarEntryBuilder::new(&fixture.mar_staging).unwrap(); + let _ = entry_dir_option.insert(builder.entry_dir_path().to_owned()); + builder + .set_metadata(Metadata::test_fixture()) + .save(&NetworkConfig::test_fixture()) + .unwrap(); + } + let entry_dir = entry_dir_option.unwrap(); + assert!(entry_dir.exists()); + assert!(entry_dir.join("manifest.json").exists()); + } + + #[rstest] + fn create_attachment_inside_entry_dir(fixture: Fixture) { + let builder = MarEntryBuilder::new(&fixture.mar_staging).unwrap(); + let orig_attachment_path = builder.make_attachment_path_in_entry_dir("attachment"); + create_file_with_size(&orig_attachment_path, 1024).unwrap(); + + builder + .add_attachment(orig_attachment_path.clone()) + .set_metadata(Metadata::test_fixture()) + .save(&NetworkConfig::test_fixture()) + .unwrap(); + + // Attachment is still where it was written: + assert!(orig_attachment_path.exists()); + } + + #[rstest] + fn attachment_outside_entry_dir_is_moved_into_entry_dir_upon_save(fixture: Fixture) { + let builder = MarEntryBuilder::new(&fixture.mar_staging).unwrap(); + let entry_dir = builder.entry_dir_path().to_owned(); + + let tempdir = tempdir().unwrap(); + let orig_attachment_path = tempdir.path().join("attachment"); + create_file_with_size(&orig_attachment_path, 1024).unwrap(); + + builder + .add_attachment(orig_attachment_path.clone()) + .set_metadata(Metadata::test_fixture()) + .save(&NetworkConfig::test_fixture()) + .unwrap(); + + // Attachment has been moved into the entry dir: + assert!(!orig_attachment_path.exists()); + assert!(entry_dir + .join(orig_attachment_path.file_name().unwrap()) + .exists()); + } + + #[rstest] + fn can_estimate_size_of_a_mar_entry(fixture: Fixture) { + let builder = MarEntryBuilder::new(&fixture.mar_staging).unwrap(); + let orig_attachment_path = builder.make_attachment_path_in_entry_dir("attachment"); + create_file_with_size(&orig_attachment_path, 1024).unwrap(); + + let builder = builder + .add_attachment(orig_attachment_path) + .set_metadata(Metadata::test_fixture()); + + assert_eq!( + builder.estimated_entry_size().bytes, + 1024 + MAR_ENTRY_OVERHEAD_SIZE_ESTIMATE + ); + assert_eq!(builder.estimated_entry_size().inodes, 2); + } + + struct Fixture { + _tempdir: TempDir, + mar_staging: PathBuf, + } + + #[fixture] + fn fixture() -> Fixture { + let tempdir = tempdir().unwrap(); + let mar_staging = tempdir.path().to_owned(); + Fixture { + _tempdir: tempdir, + mar_staging, + } + } +} diff --git a/memfaultd/src/mar/mod.rs b/memfaultd/src/mar/mod.rs new file mode 100644 index 0000000..00ff401 --- /dev/null +++ b/memfaultd/src/mar/mod.rs @@ -0,0 +1,25 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +pub mod clean; +pub mod manifest; +pub mod mar_entry; +pub mod mar_entry_builder; +pub mod upload; + +pub use clean::*; +pub use manifest::*; +pub use mar_entry::*; +pub use mar_entry_builder::*; +pub use upload::*; + +mod chunks; +mod export; +mod export_format; + +pub use chunks::{Chunk, ChunkMessage, ChunkMessageType, ChunkWrapper}; +pub use export::{MarExportHandler, EXPORT_MAR_URL}; +pub use export_format::ExportFormat; + +#[cfg(test)] +mod test_utils; diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__attributes.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__attributes.snap new file mode 100644 index 0000000..63c65dc --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__attributes.snap @@ -0,0 +1,46 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "1970-01-01T00:00:04.321Z", + "uptime_ms": 800000, + "linux_boot_id": "230295cb-04d4-40b8-8624-ec37089b9b75", + "elapsed_realtime_ms": 900000, + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "evt", + "software_version": "1.0.0", + "software_type": "linux-build", + "device_serial": "DEMOSERIAL" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "device-attributes", + "metadata": { + "attributes": [ + { + "string_key": "my_string", + "value": "foo" + }, + { + "string_key": "my_integer", + "value": 123 + }, + { + "string_key": "my_float", + "value": 123.456 + }, + { + "string_key": "my_bool", + "value": true + } + ] + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__coredump-gzip.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__coredump-gzip.snap new file mode 100644 index 0000000..a716082 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__coredump-gzip.snap @@ -0,0 +1,30 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "elf-coredump", + "metadata": { + "coredump_file_name": "/tmp/core.elf", + "compression": "gzip" + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__coredump-none.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__coredump-none.snap new file mode 100644 index 0000000..03cb837 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__coredump-none.snap @@ -0,0 +1,29 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "elf-coredump", + "metadata": { + "coredump_file_name": "/tmp/core.elf" + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__device_config.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__device_config.snap new file mode 100644 index 0000000..4f49a1c --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__device_config.snap @@ -0,0 +1,29 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "device-config", + "metadata": { + "revision": 42 + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__elf_coredump.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__elf_coredump.snap new file mode 100644 index 0000000..07b4b90 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__elf_coredump.snap @@ -0,0 +1,30 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "1970-01-01T00:00:04.321Z", + "uptime_ms": 800000, + "linux_boot_id": "230295cb-04d4-40b8-8624-ec37089b9b75", + "elapsed_realtime_ms": 900000, + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "evt", + "software_version": "1.0.0", + "software_type": "linux-build", + "device_serial": "DEMOSERIAL" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "elf-coredump", + "metadata": { + "coredump_file_name": "core-da01317a-902f-48f8-8c3f-aabf8b14facc.elf.gz", + "compression": "gzip" + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__heartbeat.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__heartbeat.snap new file mode 100644 index 0000000..e3b18fb --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__heartbeat.snap @@ -0,0 +1,52 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2023-06-08T23:47:39.021439227Z", + "uptime_ms": 35587, + "linux_boot_id": "ebf5f46f-fca6-4149-93cf-1c12a5e4d019", + "elapsed_realtime_ms": 35588, + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "qemuarm64", + "software_version": "0.0.1", + "software_type": "main", + "device_serial": "DEMOSERIAL" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "linux-heartbeat", + "metadata": { + "metrics": { + "cpu/sum/percent/idle": 482.959065568454, + "cpu/sum/percent/interrupt": 0.0, + "cpu/sum/percent/nice": 0.0, + "cpu/sum/percent/softirq": 0.403632694248234, + "cpu/sum/percent/steal": 0.0, + "cpu/sum/percent/system": 4.32826849339439, + "cpu/sum/percent/user": 15.6038091933719, + "cpu/sum/percent/wait": 0.702826448134218, + "df/media/df_complex/free": 416471040.0, + "df/media/df_complex/reserved": 27681792.0, + "df/media/df_complex/used": 19456.0, + "interface/enp0s1/if_dropped/rx": 0.0, + "interface/enp0s1/if_dropped/tx": 0.0, + "interface/enp0s1/if_errors/rx": 0.0, + "interface/enp0s1/if_errors/tx": 0.0, + "interface/enp0s1/if_octets/rx": 523.064016916174, + "interface/enp0s1/if_octets/tx": 152.289523564009, + "interface/enp0s1/if_packets/rx": 1.39990546332198, + "interface/enp0s1/if_packets/tx": 1.1999189685617, + "memory/memory/free": 260657152.0, + "memory/memory/used": 29261824.0, + "uptime/uptime": 26.0 + } + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__heartbeat_with_duration.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__heartbeat_with_duration.snap new file mode 100644 index 0000000..c9e46b4 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__heartbeat_with_duration.snap @@ -0,0 +1,53 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2023-06-08T23:47:39.021439227Z", + "uptime_ms": 35587, + "linux_boot_id": "ebf5f46f-fca6-4149-93cf-1c12a5e4d019", + "elapsed_realtime_ms": 35588, + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "qemuarm64", + "software_version": "0.0.1", + "software_type": "main", + "device_serial": "DEMOSERIAL" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "linux-heartbeat", + "metadata": { + "metrics": { + "cpu/sum/percent/idle": 482.959065568454, + "cpu/sum/percent/interrupt": 0.0, + "cpu/sum/percent/nice": 0.0, + "cpu/sum/percent/softirq": 0.403632694248234, + "cpu/sum/percent/steal": 0.0, + "cpu/sum/percent/system": 4.32826849339439, + "cpu/sum/percent/user": 15.6038091933719, + "cpu/sum/percent/wait": 0.702826448134218, + "df/media/df_complex/free": 416471040.0, + "df/media/df_complex/reserved": 27681792.0, + "df/media/df_complex/used": 19456.0, + "interface/enp0s1/if_dropped/rx": 0.0, + "interface/enp0s1/if_dropped/tx": 0.0, + "interface/enp0s1/if_errors/rx": 0.0, + "interface/enp0s1/if_errors/tx": 0.0, + "interface/enp0s1/if_octets/rx": 523.064016916174, + "interface/enp0s1/if_octets/tx": 152.289523564009, + "interface/enp0s1/if_packets/rx": 1.39990546332198, + "interface/enp0s1/if_packets/tx": 1.1999189685617, + "memory/memory/free": 260657152.0, + "memory/memory/used": 29261824.0, + "uptime/uptime": 26.0 + }, + "duration_ms": 14209 + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__log-none.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__log-none.snap new file mode 100644 index 0000000..ffc4361 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__log-none.snap @@ -0,0 +1,39 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "linux-logs", + "metadata": { + "format": { + "id": "v1", + "serialization": "json-lines" + }, + "log_file_name": "/var/log/syslog", + "cid": { + "uuid": "99686390-a728-11ed-a68b-e7ff3cd0c7e7" + }, + "next_cid": { + "uuid": "9e1ece10-a728-11ed-918e-5be35a10c7e7" + } + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__log-zlib.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__log-zlib.snap new file mode 100644 index 0000000..34fe055 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__log-zlib.snap @@ -0,0 +1,40 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "linux-logs", + "metadata": { + "format": { + "id": "v1", + "serialization": "json-lines" + }, + "log_file_name": "/var/log/syslog", + "compression": "zlib", + "cid": { + "uuid": "99686390-a728-11ed-a68b-e7ff3cd0c7e7" + }, + "next_cid": { + "uuid": "9e1ece10-a728-11ed-918e-5be35a10c7e7" + } + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__metric_report.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__metric_report.snap new file mode 100644 index 0000000..5176a6c --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__metric_report.snap @@ -0,0 +1,56 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2023-06-08T23:47:39.021439227Z", + "uptime_ms": 35587, + "linux_boot_id": "ebf5f46f-fca6-4149-93cf-1c12a5e4d019", + "elapsed_realtime_ms": 35588, + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "qemuarm64", + "software_version": "0.0.1", + "software_type": "main", + "device_serial": "DEMOSERIAL" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "linux-metric-report", + "metadata": { + "metrics": { + "cpu/sum/percent/idle": 482.959065568454, + "cpu/sum/percent/interrupt": 0.0, + "cpu/sum/percent/nice": 0.0, + "cpu/sum/percent/softirq": 0.403632694248234, + "cpu/sum/percent/steal": 0.0, + "cpu/sum/percent/system": 4.32826849339439, + "cpu/sum/percent/user": 15.6038091933719, + "cpu/sum/percent/wait": 0.702826448134218, + "df/media/df_complex/free": 416471040.0, + "df/media/df_complex/reserved": 27681792.0, + "df/media/df_complex/used": 19456.0, + "interface/enp0s1/if_dropped/rx": 0.0, + "interface/enp0s1/if_dropped/tx": 0.0, + "interface/enp0s1/if_errors/rx": 0.0, + "interface/enp0s1/if_errors/tx": 0.0, + "interface/enp0s1/if_octets/rx": 523.064016916174, + "interface/enp0s1/if_octets/tx": 152.289523564009, + "interface/enp0s1/if_packets/rx": 1.39990546332198, + "interface/enp0s1/if_packets/tx": 1.1999189685617, + "memory/memory/free": 260657152.0, + "memory/memory/used": 29261824.0, + "uptime/uptime": 26.0 + }, + "duration_ms": 14209, + "report_type": { + "session": "test-fixture-session" + } + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__reboot.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__reboot.snap new file mode 100644 index 0000000..7c92718 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__reboot.snap @@ -0,0 +1,29 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2023-05-24T18:20:21.371852623Z", + "uptime_ms": 17445527, + "linux_boot_id": "48af0b97-79c3-4e95-ba80-8c436b4b9e8f", + "elapsed_realtime_ms": 17445527, + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "qemuarm64", + "software_version": "0.0.1", + "software_type": "main", + "device_serial": "DEMOSERIAL" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "linux-reboot", + "metadata": { + "reason": 3 + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_custom_reboot.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_custom_reboot.snap new file mode 100644 index 0000000..f911a44 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_custom_reboot.snap @@ -0,0 +1,32 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "linux-reboot", + "metadata": { + "reason": { + "unexpected": false, + "name": "CustomRebootReason" + } + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_custom_unexpected_reboot.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_custom_unexpected_reboot.snap new file mode 100644 index 0000000..6319932 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_custom_unexpected_reboot.snap @@ -0,0 +1,32 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "linux-reboot", + "metadata": { + "reason": { + "unexpected": true, + "name": "CustomUnexpectedRebootReason" + } + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_device_attributes.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_device_attributes.snap new file mode 100644 index 0000000..c7d93f9 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_device_attributes.snap @@ -0,0 +1,46 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "device-attributes", + "metadata": { + "attributes": [ + { + "string_key": "my_string", + "value": "foo" + }, + { + "string_key": "my_int", + "value": 123 + }, + { + "string_key": "my_float", + "value": 123.456 + }, + { + "string_key": "my_bool", + "value": true + } + ] + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_device_configc.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_device_configc.snap new file mode 100644 index 0000000..4f49a1c --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_device_configc.snap @@ -0,0 +1,29 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "device-config", + "metadata": { + "revision": 42 + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_linux_heartbeat.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_linux_heartbeat.snap new file mode 100644 index 0000000..4bec52e --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_linux_heartbeat.snap @@ -0,0 +1,34 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "linux-metric-report", + "metadata": { + "metrics": { + "n1": 1.0, + "n2": 42.0 + }, + "duration_ms": 42000, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_reboot.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_reboot.snap new file mode 100644 index 0000000..cea2c31 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_reboot.snap @@ -0,0 +1,29 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "linux-reboot", + "metadata": { + "reason": 1 + } +} diff --git a/memfaultd/src/mar/test-manifests/attributes.json b/memfaultd/src/mar/test-manifests/attributes.json new file mode 100644 index 0000000..af5fdbb --- /dev/null +++ b/memfaultd/src/mar/test-manifests/attributes.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "collection_time": { + "timestamp": "1970-01-01T00:00:04.321Z", + "uptime_ms": 800000, + "elapsed_realtime_ms": 900000, + "linux_boot_id": "230295cb-04d4-40b8-8624-ec37089b9b75", + "boot_count": 0 + }, + "type": "device-attributes", + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "evt", + "software_version": "1.0.0", + "software_type": "linux-build", + "device_serial": "DEMOSERIAL" + }, + "metadata": { + "attributes": [ + { + "string_key": "my_string", + "value": "foo" + }, + { + "string_key": "my_integer", + "value": 123 + }, + { + "string_key": "my_float", + "value": 123.456 + }, + { + "string_key": "my_bool", + "value": true + } + ] + } +} diff --git a/memfaultd/src/mar/test-manifests/device_config.json b/memfaultd/src/mar/test-manifests/device_config.json new file mode 100644 index 0000000..9833f09 --- /dev/null +++ b/memfaultd/src/mar/test-manifests/device_config.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "type": "device-config", + "metadata": { + "revision": 42 + } +} diff --git a/memfaultd/src/mar/test-manifests/elf_coredump.json b/memfaultd/src/mar/test-manifests/elf_coredump.json new file mode 100644 index 0000000..b00bc62 --- /dev/null +++ b/memfaultd/src/mar/test-manifests/elf_coredump.json @@ -0,0 +1,22 @@ +{ + "schema_version": 1, + "collection_time": { + "timestamp": "1970-01-01T00:00:04.321Z", + "uptime_ms": 800000, + "elapsed_realtime_ms": 900000, + "linux_boot_id": "230295cb-04d4-40b8-8624-ec37089b9b75", + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "evt", + "software_version": "1.0.0", + "software_type": "linux-build", + "device_serial": "DEMOSERIAL" + }, + "type": "elf-coredump", + "metadata": { + "coredump_file_name": "core-da01317a-902f-48f8-8c3f-aabf8b14facc.elf.gz", + "compression": "gzip" + } +} diff --git a/memfaultd/src/mar/test-manifests/heartbeat.json b/memfaultd/src/mar/test-manifests/heartbeat.json new file mode 100644 index 0000000..8cf4cba --- /dev/null +++ b/memfaultd/src/mar/test-manifests/heartbeat.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2023-06-08T23:47:39.021439227Z", + "uptime_ms": 35587, + "linux_boot_id": "ebf5f46f-fca6-4149-93cf-1c12a5e4d019", + "elapsed_realtime_ms": 35588, + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "qemuarm64", + "software_version": "0.0.1", + "software_type": "main", + "device_serial": "DEMOSERIAL" + }, + "producer": { + "id": "memfaultd", + "version": "dev" + }, + "type": "linux-heartbeat", + "metadata": { + "metrics": { + "cpu/sum/percent/idle": 482.959065568454, + "cpu/sum/percent/interrupt": 0, + "cpu/sum/percent/nice": 0, + "cpu/sum/percent/softirq": 0.403632694248234, + "cpu/sum/percent/steal": 0, + "cpu/sum/percent/system": 4.32826849339439, + "cpu/sum/percent/user": 15.6038091933719, + "cpu/sum/percent/wait": 0.702826448134218, + "df/media/df_complex/free": 416471040, + "df/media/df_complex/reserved": 27681792, + "df/media/df_complex/used": 19456, + "interface/enp0s1/if_dropped/rx": 0, + "interface/enp0s1/if_dropped/tx": 0, + "interface/enp0s1/if_errors/rx": 0, + "interface/enp0s1/if_errors/tx": 0, + "interface/enp0s1/if_octets/rx": 523.064016916174, + "interface/enp0s1/if_octets/tx": 152.289523564009, + "interface/enp0s1/if_packets/rx": 1.39990546332198, + "interface/enp0s1/if_packets/tx": 1.1999189685617, + "memory/memory/free": 260657152, + "memory/memory/used": 29261824, + "uptime/uptime": 26 + } + } +} diff --git a/memfaultd/src/mar/test-manifests/heartbeat_with_duration.json b/memfaultd/src/mar/test-manifests/heartbeat_with_duration.json new file mode 100644 index 0000000..e17b93a --- /dev/null +++ b/memfaultd/src/mar/test-manifests/heartbeat_with_duration.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2023-06-08T23:47:39.021439227Z", + "uptime_ms": 35587, + "linux_boot_id": "ebf5f46f-fca6-4149-93cf-1c12a5e4d019", + "elapsed_realtime_ms": 35588, + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "qemuarm64", + "software_version": "0.0.1", + "software_type": "main", + "device_serial": "DEMOSERIAL" + }, + "producer": { + "id": "memfaultd", + "version": "dev" + }, + "type": "linux-heartbeat", + "metadata": { + "metrics": { + "cpu/sum/percent/idle": 482.959065568454, + "cpu/sum/percent/interrupt": 0, + "cpu/sum/percent/nice": 0, + "cpu/sum/percent/softirq": 0.403632694248234, + "cpu/sum/percent/steal": 0, + "cpu/sum/percent/system": 4.32826849339439, + "cpu/sum/percent/user": 15.6038091933719, + "cpu/sum/percent/wait": 0.702826448134218, + "df/media/df_complex/free": 416471040, + "df/media/df_complex/reserved": 27681792, + "df/media/df_complex/used": 19456, + "interface/enp0s1/if_dropped/rx": 0, + "interface/enp0s1/if_dropped/tx": 0, + "interface/enp0s1/if_errors/rx": 0, + "interface/enp0s1/if_errors/tx": 0, + "interface/enp0s1/if_octets/rx": 523.064016916174, + "interface/enp0s1/if_octets/tx": 152.289523564009, + "interface/enp0s1/if_packets/rx": 1.39990546332198, + "interface/enp0s1/if_packets/tx": 1.1999189685617, + "memory/memory/free": 260657152, + "memory/memory/used": 29261824, + "uptime/uptime": 26 + }, + "duration_ms": 14209 + } +} diff --git a/memfaultd/src/mar/test-manifests/log.json b/memfaultd/src/mar/test-manifests/log.json new file mode 100644 index 0000000..cfc461f --- /dev/null +++ b/memfaultd/src/mar/test-manifests/log.json @@ -0,0 +1,25 @@ +{ + "schema_version": 1, + "collection_time": { + "timestamp": "1970-01-01T00:00:04.321Z", + "uptime_ms": 800000, + "elapsed_realtime_ms": 900000, + "linux_boot_id": "230295cb-04d4-40b8-8624-ec37089b9b75", + "boot_count": 0 + }, + "type": "linux-logs", + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "evt", + "software_version": "1.0.0", + "software_type": "linux-build", + "device_serial": "DEMOSERIAL" + }, + "metadata": { + "format": { "id": "v1", "serialization": "json-lines" }, + "producer": { "id": "memfaultd", "version": "2.0.0" }, + "log_file_name": "logs.txt", + "cid": { "uuid": "271e5249-1e3f-4c89-b4a5-ed96cc8f42a7" }, + "next_cid": { "uuid": "4276a26b-ba89-4c57-832f-d9e90e9629dd" } + } +} diff --git a/memfaultd/src/mar/test-manifests/metric_report.json b/memfaultd/src/mar/test-manifests/metric_report.json new file mode 100644 index 0000000..af64563 --- /dev/null +++ b/memfaultd/src/mar/test-manifests/metric_report.json @@ -0,0 +1,52 @@ +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2023-06-08T23:47:39.021439227Z", + "uptime_ms": 35587, + "linux_boot_id": "ebf5f46f-fca6-4149-93cf-1c12a5e4d019", + "elapsed_realtime_ms": 35588, + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "qemuarm64", + "software_version": "0.0.1", + "software_type": "main", + "device_serial": "DEMOSERIAL" + }, + "producer": { + "id": "memfaultd", + "version": "dev" + }, + "type": "linux-metric-report", + "metadata": { + "metrics": { + "cpu/sum/percent/idle": 482.959065568454, + "cpu/sum/percent/interrupt": 0, + "cpu/sum/percent/nice": 0, + "cpu/sum/percent/softirq": 0.403632694248234, + "cpu/sum/percent/steal": 0, + "cpu/sum/percent/system": 4.32826849339439, + "cpu/sum/percent/user": 15.6038091933719, + "cpu/sum/percent/wait": 0.702826448134218, + "df/media/df_complex/free": 416471040, + "df/media/df_complex/reserved": 27681792, + "df/media/df_complex/used": 19456, + "interface/enp0s1/if_dropped/rx": 0, + "interface/enp0s1/if_dropped/tx": 0, + "interface/enp0s1/if_errors/rx": 0, + "interface/enp0s1/if_errors/tx": 0, + "interface/enp0s1/if_octets/rx": 523.064016916174, + "interface/enp0s1/if_octets/tx": 152.289523564009, + "interface/enp0s1/if_packets/rx": 1.39990546332198, + "interface/enp0s1/if_packets/tx": 1.1999189685617, + "memory/memory/free": 260657152, + "memory/memory/used": 29261824, + "uptime/uptime": 26 + }, + "duration_ms": 14209, + "report_type": { + "session": "test-fixture-session" + } + } +} diff --git a/memfaultd/src/mar/test-manifests/reboot.json b/memfaultd/src/mar/test-manifests/reboot.json new file mode 100644 index 0000000..a927a63 --- /dev/null +++ b/memfaultd/src/mar/test-manifests/reboot.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2023-05-24T18:20:21.371852623Z", + "uptime_ms": 17445527, + "linux_boot_id": "48af0b97-79c3-4e95-ba80-8c436b4b9e8f", + "elapsed_realtime_ms": 17445527, + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "qemuarm64", + "software_version": "0.0.1", + "software_type": "main", + "device_serial": "DEMOSERIAL" + }, + "type": "linux-reboot", + "metadata": { + "reason": 3 + } +} diff --git a/memfaultd/src/mar/test_utils.rs b/memfaultd/src/mar/test_utils.rs new file mode 100644 index 0000000..25688d2 --- /dev/null +++ b/memfaultd/src/mar/test_utils.rs @@ -0,0 +1,243 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + collections::HashMap, + time::{Duration, SystemTime}, +}; +use std::{ + fs::{create_dir, create_dir_all, set_permissions, File}, + io::{BufWriter, Write}, + os::unix::prelude::PermissionsExt, + path::PathBuf, +}; + +use crate::{ + mar::{CompressionAlgorithm, DeviceAttribute}, + metrics::{MetricReportType, MetricStringKey, MetricValue}, + reboot::RebootReason, +}; +use tempfile::{tempdir, TempDir}; +use uuid::Uuid; + +use crate::network::NetworkConfig; +use crate::test_utils::create_file_with_size; +use crate::util::zip::ZipEncoder; + +use super::manifest::{CollectionTime, Manifest, Metadata}; + +pub struct MarCollectorFixture { + pub mar_staging: PathBuf, + // Keep a reference to the tempdir so it is automatically + // deleted *after* the fixture + _tempdir: TempDir, + config: NetworkConfig, +} + +impl MarCollectorFixture { + pub fn new() -> Self { + let tempdir = tempdir().unwrap(); + let mar_staging = tempdir.path().to_owned(); + create_dir_all(&mar_staging).unwrap(); + Self { + mar_staging, + _tempdir: tempdir, + config: NetworkConfig::test_fixture(), + } + } + + pub fn create_empty_entry(&mut self) -> PathBuf { + let uuid = Uuid::new_v4(); + let path = self.mar_staging.join(uuid.to_string()); + create_dir(&path).unwrap(); + path + } + + pub fn create_device_attributes_entry( + &mut self, + attributes: Vec, + timestamp: SystemTime, + ) -> PathBuf { + let path = self.create_empty_entry(); + let manifest_path = path.join("manifest.json"); + + let manifest_file = File::create(manifest_path).unwrap(); + + let mut collection_time = CollectionTime::test_fixture(); + collection_time.timestamp = timestamp.into(); + + let manifest = Manifest::new( + &self.config, + collection_time, + Metadata::new_device_attributes(attributes), + ); + serde_json::to_writer(BufWriter::new(manifest_file), &manifest).unwrap(); + + path + } + + pub fn create_logentry_with_size(&mut self, size: u64) -> PathBuf { + self.create_logentry_with_size_and_age(size, SystemTime::now()) + } + + pub fn create_logentry_with_size_and_age( + &mut self, + size: u64, + timestamp: SystemTime, + ) -> PathBuf { + let path = self.create_empty_entry(); + let manifest_path = path.join("manifest.json"); + + let log_name = "system.log".to_owned(); + let log_path = path.join(&log_name); + create_file_with_size(&log_path, size).unwrap(); + + let manifest_file = File::create(manifest_path).unwrap(); + + let mut collection_time = CollectionTime::test_fixture(); + collection_time.timestamp = timestamp.into(); + + let manifest = Manifest::new( + &self.config, + collection_time, + Metadata::new_log( + log_name, + Uuid::new_v4(), + Uuid::new_v4(), + CompressionAlgorithm::Zlib, + ), + ); + serde_json::to_writer(BufWriter::new(manifest_file), &manifest).unwrap(); + + path + } + + pub fn create_logentry(&mut self) -> PathBuf { + self.create_logentry_with_size_and_age(0, SystemTime::now()) + } + + pub fn create_logentry_with_unreadable_attachment(&mut self) -> PathBuf { + let path = self.create_empty_entry(); + let manifest_path = path.join("manifest.json"); + + let log_name = "system.log".to_owned(); + let log_path = path.join(&log_name); + let log = File::create(&log_path).unwrap(); + drop(log); + + let mut permissions = log_path.metadata().unwrap().permissions(); + permissions.set_mode(0o0); + set_permissions(&log_path, permissions).unwrap(); + + let manifest_file = File::create(manifest_path).unwrap(); + let manifest = Manifest::new( + &self.config, + CollectionTime::test_fixture(), + Metadata::new_log( + log_name, + Uuid::new_v4(), + Uuid::new_v4(), + CompressionAlgorithm::Zlib, + ), + ); + serde_json::to_writer(BufWriter::new(manifest_file), &manifest).unwrap(); + + path + } + + pub fn create_entry_with_bogus_json(&mut self) -> PathBuf { + let path = self.create_empty_entry(); + let manifest_path = path.join("manifest.json"); + File::create(manifest_path) + .unwrap() + .write_all(b"BOGUS") + .unwrap(); + path + } + + pub fn create_entry_without_directory_read_permission(&mut self) -> PathBuf { + let path = self.create_empty_entry(); + let manifest_path = path.join("manifest.json"); + File::create(manifest_path) + .unwrap() + .write_all(b"BOGUS") + .unwrap(); + + let mut permissions = path.metadata().unwrap().permissions(); + permissions.set_mode(0o0); + set_permissions(&path, permissions).unwrap(); + path + } + + pub fn create_entry_without_manifest_read_permission(&mut self) -> PathBuf { + let path = self.create_empty_entry(); + let manifest_path = path.join("manifest.json"); + File::create(&manifest_path) + .unwrap() + .write_all(b"BOGUS") + .unwrap(); + + let mut permissions = manifest_path.metadata().unwrap().permissions(); + permissions.set_mode(0o0); + set_permissions(manifest_path, permissions).unwrap(); + path + } + + pub fn create_metric_report_entry( + &mut self, + metrics: HashMap, + duration: Duration, + report_type: MetricReportType, + ) -> PathBuf { + let path = self.create_empty_entry(); + let manifest_path = path.join("manifest.json"); + + let manifest_file = File::create(manifest_path).unwrap(); + let manifest = Manifest::new( + &self.config, + CollectionTime::test_fixture(), + Metadata::new_metric_report(metrics, duration, report_type), + ); + serde_json::to_writer(BufWriter::new(manifest_file), &manifest).unwrap(); + + path + } + + pub fn create_reboot_entry(&mut self, reason: RebootReason) -> PathBuf { + let path = self.create_empty_entry(); + let manifest_path = path.join("manifest.json"); + + let manifest_file = File::create(manifest_path).unwrap(); + let manifest = Manifest::new( + &self.config, + CollectionTime::test_fixture(), + Metadata::new_reboot(reason), + ); + serde_json::to_writer(BufWriter::new(manifest_file), &manifest).unwrap(); + + path + } +} + +/// Check the content of a MAR zip encoder against a list of expected files. +/// The first (in zip order) entry name is renamed from "some_uuid/" to +/// "/" before matching and the list is sorted alphabetically. +/// Eg: ZIP(abcd42/manifest.json abcd42/file.txt) => [/file.txt, /manifest.json] +pub fn assert_mar_content_matches(zip_encoder: &ZipEncoder, expected_files: Vec<&str>) -> bool { + // Get the folder name for the first entry, we will s/(entry_uuid)// to make matching friendlier + let file_names = zip_encoder.file_names(); + assert!(!file_names.is_empty()); + + let entry_name = file_names[0] + .split(std::path::MAIN_SEPARATOR) + .next() + .unwrap(); + let mut files_list = file_names + .iter() + .map(|filename| filename.replace(entry_name, "")) + .collect::>(); + files_list.sort(); + + assert_eq!(files_list, *expected_files); + true +} diff --git a/memfaultd/src/mar/upload.rs b/memfaultd/src/mar/upload.rs new file mode 100644 index 0000000..df40622 --- /dev/null +++ b/memfaultd/src/mar/upload.rs @@ -0,0 +1,514 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Collect and upload MAR entries. +//! +//! This module provides the functionality to collect all valid MAR entries, upload them and delete them on success. +//! +//! Whether or not an entry is uploaded depends on the sampling configuration. Each type of entry can have a different +//! level of configuration with device config and reboots always being uploaded. All other will be uploaded based on +//! the below rules: +//! +//! +=================+=====+=====+========+======+ +//! | MAR Type | Off | Low | Medium | High | +//! +=================+=====+=====+========+======+ +//! | heartbeat | | | x | x | +//! +-----------------+-----+-----+--------+------+ +//! | daily-heartbeat | | x | x | x | +//! +-----------------+-----+-----+--------+------+ +//! | session | | | x | x | +//! +-----------------+-----+-----+--------+------+ +//! | attributes | | | x | x | +//! +-----------------+-----+-----+--------+------+ +//! | coredump | | | x | x | +//! +-----------------+-----+-----+--------+------+ +//! | logs | | | x | x | +//! +-----------------+-----+-----+--------+------+ + +use std::fs::{remove_dir_all, File}; +use std::io::BufReader; +use std::path::{Path, PathBuf}; + +use eyre::{Context, Result}; +use itertools::Itertools; +use log::{trace, warn}; + +use crate::{ + config::{Resolution, Sampling}, + mar::{MarEntry, Metadata}, + metrics::MetricReportType, + network::NetworkClient, + util::zip::{zip_stream_len_empty, zip_stream_len_for_file, ZipEncoder, ZipEntryInfo}, +}; + +/// Collect all valid MAR entries, upload them and delete them on success. +/// +/// Returns the number of MAR entries that were uploaded. +/// +/// This function will not do anything with invalid MAR entries (we assume they are "under construction"). +pub fn collect_and_upload( + mar_staging: &Path, + client: &impl NetworkClient, + max_zip_size: usize, + sampling: Sampling, +) -> Result { + let mut entries = MarEntry::iterate_from_container(mar_staging)? + // Apply fleet sampling to the MAR entries + .filter(|entry_result| match entry_result { + Ok(entry) => should_upload(&entry.manifest.metadata, &sampling), + _ => true, + }); + + upload_mar_entries(&mut entries, client, max_zip_size, |included_entries| { + trace!("Uploaded {:?} - deleting...", included_entries); + included_entries.iter().for_each(|f| { + let _ = remove_dir_all(f); + }) + }) +} + +/// Given the current sampling configuration determine if the given MAR entry should be uploaded. +fn should_upload(metadata: &Metadata, sampling: &Sampling) -> bool { + match metadata { + Metadata::DeviceAttributes { .. } => sampling.monitoring_resolution >= Resolution::Normal, + Metadata::DeviceConfig { .. } => true, // Always upload device config + Metadata::ElfCoredump { .. } => sampling.debugging_resolution >= Resolution::Normal, + Metadata::LinuxHeartbeat { .. } => sampling.monitoring_resolution >= Resolution::Normal, + Metadata::LinuxMetricReport { report_type, .. } => match report_type { + MetricReportType::Heartbeat => sampling.monitoring_resolution >= Resolution::Normal, + MetricReportType::Session(_) => sampling.monitoring_resolution >= Resolution::Normal, + MetricReportType::DailyHeartbeat => sampling.monitoring_resolution >= Resolution::Low, + }, + Metadata::LinuxLogs { .. } => sampling.logging_resolution >= Resolution::Normal, + Metadata::LinuxReboot { .. } => true, // Always upload reboots + Metadata::LinuxMemfaultWatch { exit_code, .. } => { + let is_crash = exit_code != &0; + if is_crash { + sampling.debugging_resolution >= Resolution::Normal + || sampling.logging_resolution >= Resolution::Normal + } else { + sampling.logging_resolution >= Resolution::Normal + } + } + } +} + +/// Describes the contents for a single MAR file to upload. +pub struct MarZipContents { + /// All the MAR entry directories to to be included in this file. + pub entry_paths: Vec, + /// All the ZipEntryInfos to be included in this file. + pub zip_infos: Vec, +} + +/// Gather MAR entries and associated ZipEntryInfos, consuming items from the iterator. +/// +/// Return a list of MarZipContents, each containing the list of folders that are included in the +/// zip (and can be deleted after upload) and the list of ZipEntryInfos. +/// Invalid folders will not trigger an error and they will not be included in the returned lists. +pub fn gather_mar_entries_to_zip( + entries: &mut impl Iterator>, + max_zip_size: usize, +) -> Vec { + let entry_paths_with_zip_infos = entries.filter_map(|entry_result| match entry_result { + Ok(entry) => { + trace!("Adding {:?}", &entry.path); + let zip_infos: Option> = (&entry) + .try_into() + .wrap_err_with(|| format!("Unable to add entry {}.", &entry.path.display())) + .ok(); + let entry_and_infos: Option<(PathBuf, Vec)> = + zip_infos.map(|infos| (entry.path, infos)); + entry_and_infos + } + Err(e) => { + warn!("Invalid folder in MAR staging: {:?}", e); + None + } + }); + + let mut zip_size = zip_stream_len_empty(); + let mut zip_file_index: usize = 0; + let grouper = entry_paths_with_zip_infos.group_by(|(_, zip_infos)| { + let entry_zipped_size = zip_infos.iter().map(zip_stream_len_for_file).sum::(); + if zip_size + entry_zipped_size > max_zip_size { + zip_size = zip_stream_len_empty() + entry_zipped_size; + zip_file_index += 1; + } else { + zip_size += entry_zipped_size; + } + zip_file_index + }); + + grouper + .into_iter() + .map(|(_zip_file_index, group)| { + // Convert from Vec<(PathBuf, Vec)> to MarZipContents: + let (entry_paths, zip_infos): (Vec, Vec>) = group.unzip(); + MarZipContents { + entry_paths, + zip_infos: zip_infos + .into_iter() + .flatten() + .collect::>(), + } + }) + .collect() +} + +impl TryFrom<&MarEntry> for Vec { + type Error = eyre::Error; + + fn try_from(entry: &MarEntry) -> Result { + let entry_path = entry.path.clone(); + entry + .filenames() + .map(move |filename| { + let path = entry_path.join(&filename); + + // Open the file to check that it exists and is readable. This is a best effort to avoid + // starting to upload a MAR file only to find out half way through that a file was not + // readable. Yes, this is prone to a race condition where it is no longer readable by + // the time is going to be read by the zip writer, but it is better than nothing. + let file = + File::open(&path).wrap_err_with(|| format!("Error opening {:?}", filename))?; + drop(file); + + let base = entry_path.parent().unwrap(); + ZipEntryInfo::new(path, base) + .wrap_err_with(|| format!("Error adding {:?}", filename)) + }) + .collect::>>() + } +} + +/// Progressively upload the MAR entries. The callback will be called for each batch that is uploaded. +fn upload_mar_entries( + entries: &mut impl Iterator>, + client: &impl NetworkClient, + max_zip_size: usize, + callback: fn(entries: Vec) -> (), +) -> Result { + let zip_files = gather_mar_entries_to_zip(entries, max_zip_size); + let count = zip_files.len(); + + for MarZipContents { + entry_paths, + zip_infos, + } in zip_files.into_iter() + { + client.upload_mar_file(BufReader::new(ZipEncoder::new(zip_infos)))?; + callback(entry_paths); + } + Ok(count) +} + +#[cfg(test)] +mod tests { + use rstest::{fixture, rstest}; + use std::str::FromStr; + use std::{ + collections::HashMap, + time::{Duration, SystemTime}, + }; + + use crate::reboot::{RebootReason, RebootReasonCode}; + use crate::{ + mar::test_utils::{assert_mar_content_matches, MarCollectorFixture}, + metrics::SessionName, + network::MockNetworkClient, + }; + use crate::{ + metrics::{MetricStringKey, MetricValue}, + test_utils::setup_logger, + }; + + use super::*; + + #[rstest] + fn collecting_from_empty_folder(_setup_logger: (), mar_fixture: MarCollectorFixture) { + assert_eq!( + MarEntry::iterate_from_container(&mar_fixture.mar_staging) + .unwrap() + .count(), + 0 + ) + } + + #[rstest] + fn collecting_from_folder_with_partial_entries( + _setup_logger: (), + mut mar_fixture: MarCollectorFixture, + ) { + mar_fixture.create_empty_entry(); + mar_fixture.create_logentry(); + + assert_eq!( + MarEntry::iterate_from_container(&mar_fixture.mar_staging) + .unwrap() + .filter(|e| e.is_ok()) + .count(), + // Only one entry should be picked up. The other one is ignored. + 1 + ) + } + + #[rstest] + fn zipping_two_entries(_setup_logger: (), mut mar_fixture: MarCollectorFixture) { + // Add one valid entry so we can verify that this one is readable. + mar_fixture.create_logentry(); + mar_fixture.create_logentry(); + + let mut entries = MarEntry::iterate_from_container(&mar_fixture.mar_staging) + .expect("We should still be able to collect."); + + let mars = gather_mar_entries_to_zip(&mut entries, usize::MAX); + + assert_eq!(mars.len(), 1); + assert_eq!(mars[0].entry_paths.len(), 2); + assert_eq!(mars[0].zip_infos.len(), 4); // for each entry: manifest.json + log file + } + + #[rstest] + #[case::not_json(MarCollectorFixture::create_entry_with_bogus_json)] + #[case::unreadable_dir(MarCollectorFixture::create_entry_without_directory_read_permission)] + #[case::unreadable_manifest(MarCollectorFixture::create_entry_without_manifest_read_permission)] + fn zipping_with_skipped_entries( + _setup_logger: (), + mut mar_fixture: MarCollectorFixture, + #[case] create_bogus_entry: fn(&mut MarCollectorFixture) -> PathBuf, + ) { + create_bogus_entry(&mut mar_fixture); + // Add one valid entry so we can verify that this one is readable. + mar_fixture.create_logentry(); + + let mut entries = MarEntry::iterate_from_container(&mar_fixture.mar_staging) + .expect("We should still be able to collect."); + + let mars = gather_mar_entries_to_zip(&mut entries, usize::MAX); + + assert_eq!(mars.len(), 1); + assert_eq!(mars[0].entry_paths.len(), 1); + assert_eq!(mars[0].zip_infos.len(), 2); // manifest.json + log file + } + + #[rstest] + fn zipping_an_unreadable_attachment(_setup_logger: (), mut mar_fixture: MarCollectorFixture) { + // Add one valid entry so we can verify that this one is readable. + mar_fixture.create_logentry_with_unreadable_attachment(); + + let mut entries = MarEntry::iterate_from_container(&mar_fixture.mar_staging) + .expect("We should still be able to collect."); + + let mars = gather_mar_entries_to_zip(&mut entries, usize::MAX); + + // No MAR should be created because the attachment is unreadable. + assert_eq!(mars.len(), 0); + } + + #[rstest] + fn new_mar_when_size_limit_is_reached(_setup_logger: (), mut mar_fixture: MarCollectorFixture) { + let max_zip_size = 1024; + mar_fixture.create_logentry_with_size(max_zip_size / 2); + mar_fixture.create_logentry_with_size(max_zip_size); + // Note: the next entry exceeds the size limit, but it is still added to a MAR of its own: + mar_fixture.create_logentry_with_size(max_zip_size * 2); + + let mut entries = MarEntry::iterate_from_container(&mar_fixture.mar_staging) + .expect("We should still be able to collect."); + + let mars = gather_mar_entries_to_zip(&mut entries, max_zip_size as usize); + + // 3 MARs should be created because the size limit was reached after every entry: + assert_eq!(mars.len(), 3); + for contents in mars { + assert_eq!(contents.entry_paths.len(), 1); + assert_eq!(contents.zip_infos.len(), 2); // for each entry: manifest.json + log file + } + } + + #[rstest] + fn uploading_empty_list( + _setup_logger: (), + client: MockNetworkClient, + mar_fixture: MarCollectorFixture, + ) { + // We do not set an expectation on client => it will panic if client.upload_mar is called + collect_and_upload( + &mar_fixture.mar_staging, + &client, + usize::MAX, + Sampling { + debugging_resolution: Resolution::Normal, + logging_resolution: Resolution::Normal, + monitoring_resolution: Resolution::Normal, + }, + ) + .unwrap(); + } + + #[rstest] + #[case::off(Resolution::Off, false)] + #[case::low(Resolution::Low, false)] + #[case::normal(Resolution::Normal, true)] + #[case::high(Resolution::High, true)] + fn uploading_logs( + #[case] resolution: Resolution, + #[case] should_upload: bool, + _setup_logger: (), + client: MockNetworkClient, + mut mar_fixture: MarCollectorFixture, + ) { + mar_fixture.create_logentry(); + + let expected_files = + should_upload.then(|| vec!["/manifest.json", "/system.log"]); + let sampling_config = Sampling { + debugging_resolution: Resolution::Off, + logging_resolution: resolution, + monitoring_resolution: Resolution::Off, + }; + upload_and_verify(mar_fixture, client, sampling_config, expected_files); + } + + #[rstest] + #[case::off(Resolution::Off, false)] + #[case::low(Resolution::Low, false)] + #[case::normal(Resolution::Normal, true)] + #[case::high(Resolution::High, true)] + fn uploading_device_attributes( + #[case] resolution: Resolution, + #[case] should_upload: bool, + _setup_logger: (), + client: MockNetworkClient, + mut mar_fixture: MarCollectorFixture, + ) { + mar_fixture.create_device_attributes_entry(vec![], SystemTime::now()); + + let sampling_config = Sampling { + debugging_resolution: Resolution::Off, + logging_resolution: Resolution::Off, + monitoring_resolution: resolution, + }; + let expected_files = should_upload.then(|| vec!["/manifest.json"]); + upload_and_verify(mar_fixture, client, sampling_config, expected_files); + } + + #[rstest] + // Verify that reboots are always uploaded + #[case::off(Resolution::Off, true)] + #[case::low(Resolution::Low, true)] + #[case::normal(Resolution::Normal, true)] + #[case::high(Resolution::High, true)] + fn uploading_reboots( + #[case] resolution: Resolution, + #[case] should_upload: bool, + _setup_logger: (), + client: MockNetworkClient, + mut mar_fixture: MarCollectorFixture, + ) { + mar_fixture.create_reboot_entry(RebootReason::Code(RebootReasonCode::Unknown)); + + let sampling_config = Sampling { + debugging_resolution: resolution, + logging_resolution: Resolution::Off, + monitoring_resolution: Resolution::Off, + }; + let expected_files = should_upload.then(|| vec!["/manifest.json"]); + upload_and_verify(mar_fixture, client, sampling_config, expected_files); + } + + #[rstest] + // Heartbeat cases + #[case::heartbeat_off(MetricReportType::Heartbeat, Resolution::Off, false)] + #[case::heartbeat_low(MetricReportType::Heartbeat, Resolution::Low, false)] + #[case::heartbeat_normal(MetricReportType::Heartbeat, Resolution::Normal, true)] + #[case::heartbeat_high(MetricReportType::Heartbeat, Resolution::High, true)] + // Daily heartbeat cases + #[case::daily_heartbeat_off(MetricReportType::DailyHeartbeat, Resolution::Off, false)] + #[case::daily_heartbeat_low(MetricReportType::DailyHeartbeat, Resolution::Low, true)] + #[case::daily_heartbeat_normal(MetricReportType::DailyHeartbeat, Resolution::Normal, true)] + #[case::daily_heartbeat_high(MetricReportType::DailyHeartbeat, Resolution::High, true)] + // Session cases + #[case::session_off( + MetricReportType::Session(SessionName::from_str("test").unwrap()), + Resolution::Off, + false + )] + #[case::session_low( + MetricReportType::Session(SessionName::from_str("test").unwrap()), + Resolution::Low, + false + )] + #[case::session_normal( + MetricReportType::Session(SessionName::from_str("test").unwrap()), + Resolution::Normal, + true + )] + #[case::session_high( + MetricReportType::Session(SessionName::from_str("test").unwrap()), + Resolution::High, + true + )] + fn uploading_metric_reports( + #[case] report_type: MetricReportType, + #[case] resolution: Resolution, + #[case] should_upload: bool, + _setup_logger: (), + client: MockNetworkClient, + mut mar_fixture: MarCollectorFixture, + ) { + let duration = Duration::from_secs(1); + let metrics: HashMap = vec![( + MetricStringKey::from_str("foo").unwrap(), + MetricValue::Number(1.0), + )] + .into_iter() + .collect(); + + mar_fixture.create_metric_report_entry(metrics, duration, report_type); + + let sampling_config = Sampling { + debugging_resolution: Resolution::Off, + logging_resolution: Resolution::Off, + monitoring_resolution: resolution, + }; + let expected_files = should_upload.then(|| vec!["/manifest.json"]); + upload_and_verify(mar_fixture, client, sampling_config, expected_files); + } + + fn upload_and_verify( + mar_fixture: MarCollectorFixture, + mut client: MockNetworkClient, + sampling_config: Sampling, + expected_files: Option>, + ) { + if let Some(expected_files) = expected_files { + client + .expect_upload_mar_file::>() + .withf(move |buf_reader| { + let zip_encoder = buf_reader.get_ref(); + assert_mar_content_matches(zip_encoder, expected_files.clone()) + }) + .once() + .returning(|_| Ok(())); + } + collect_and_upload( + &mar_fixture.mar_staging, + &client, + usize::MAX, + sampling_config, + ) + .unwrap(); + } + + #[fixture] + fn client() -> MockNetworkClient { + MockNetworkClient::default() + } + + #[fixture] + fn mar_fixture() -> MarCollectorFixture { + MarCollectorFixture::new() + } +} diff --git a/memfaultd/src/memfaultd.rs b/memfaultd/src/memfaultd.rs new file mode 100644 index 0000000..3a7840b --- /dev/null +++ b/memfaultd/src/memfaultd.rs @@ -0,0 +1,534 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::process::Command; +use std::sync::Arc; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Mutex, +}; +use std::thread::{sleep, spawn}; +use std::time::Duration; +use std::{fs::create_dir_all, time::Instant}; + +use eyre::Result; +use eyre::{eyre, Context}; +use log::{error, info, trace, warn}; + +use crate::metrics::{ + BatteryMonitor, BatteryReadingHandler, ConnectivityMonitor, MetricReportType, + PeriodicMetricReportDumper, ReportSyncEventHandler, SessionEventHandler, + SystemMetricsCollector, +}; + +use crate::{ + config::Config, + mar::upload::collect_and_upload, + metrics::{ + core_metrics::{METRIC_MF_SYNC_FAILURE, METRIC_MF_SYNC_SUCCESS}, + CrashFreeIntervalTracker, MetricReportManager, StatsDServer, + }, +}; +use crate::{http_server::HttpHandler, util::UpdateStatus}; +use crate::{ + http_server::HttpServer, + network::{NetworkClientImpl, NetworkConfig}, +}; +use crate::{ + mar::MarExportHandler, + util::{ + can_connect::TcpConnectionChecker, + task::{loop_with_exponential_error_backoff, LoopContinuation}, + }, +}; +use crate::{mar::MarStagingCleaner, service_manager::get_service_manager}; +use crate::{reboot::RebootReasonTracker, util::disk_size::DiskSize}; + +#[cfg(feature = "collectd")] +use crate::collectd::CollectdHandler; + +#[cfg(feature = "logging")] +use crate::{ + fluent_bit::{FluentBitConfig, FluentBitConnectionHandler}, + logs::{CompletedLog, FluentBitAdapter, HeadroomLimiter, LogCollector, LogCollectorConfig}, + mar::{MarEntryBuilder, Metadata}, + util::disk_size::get_disk_space, +}; + +const CONFIG_REFRESH_INTERVAL: Duration = Duration::from_secs(60 * 120); +const DAILY_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(60 * 60 * 24); + +#[derive(PartialEq, Eq)] +pub enum MemfaultLoopResult { + Terminate, + Relaunch, +} + +pub fn memfaultd_loop Result<()>>( + config: Config, + ready_callback: C, +) -> Result { + // Register a flag which will be set when one of these signals is received. + let term_signals = [signal_hook::consts::SIGINT, signal_hook::consts::SIGTERM]; + let term = Arc::new(AtomicBool::new(false)); + for signal in term_signals { + signal_hook::flag::register(signal, Arc::clone(&term))?; + } + + // This flag will be set when we get the SIGHUP signal to reload (currently reload = restart) + let reload = Arc::new(AtomicBool::new(false)); + signal_hook::flag::register(signal_hook::consts::SIGHUP, Arc::clone(&reload))?; + + // Register a flag to be set when we are woken up by SIGUSR1 + let force_sync = Arc::new(AtomicBool::new(false)); + signal_hook::flag::register(signal_hook::consts::SIGUSR1, Arc::clone(&force_sync))?; + + // Load configuration and device information. This has already been done by the C code but + // we are preparing for a future where there is no more C code. + let client = NetworkClientImpl::new(NetworkConfig::from(&config)) + .wrap_err(eyre!("Unable to prepare network client"))?; + + let service_manager = get_service_manager(); + + // Make sure the MAR staging area exists + create_dir_all(config.mar_staging_path()).wrap_err_with(|| { + eyre!( + "Unable to create MAR staging area {}", + &config.mar_staging_path().display(), + ) + })?; + + let mar_cleaner = Arc::new(MarStagingCleaner::new( + &config.mar_staging_path(), + config.tmp_dir_max_size(), + config.tmp_dir_min_headroom(), + config.mar_entry_max_age(), + )); + + // List of tasks to run before syncing with server + let mut sync_tasks: Vec Result<()>>> = vec![]; + // List of tasks to run before shutting down + let mut shutdown_tasks: Vec Result<()>>> = vec![]; + + // List of http handlers + #[allow(unused_mut, /* reason = "Can be unused when some features are disabled." */)] + let mut http_handlers: Vec> = + vec![Box::new(MarExportHandler::new(config.mar_staging_path()))]; + + // Metric store + let metric_report_manager = match config.session_configs() { + Some(session_configs) => Arc::new(Mutex::new( + MetricReportManager::new_with_session_configs(session_configs), + )), + None => Arc::new(Mutex::new(MetricReportManager::new())), + }; + + let battery_monitor = Arc::new(Mutex::new(BatteryMonitor::::new( + metric_report_manager.clone(), + ))); + let battery_reading_handler = BatteryReadingHandler::new( + config.config_file.enable_data_collection, + battery_monitor.clone(), + ); + http_handlers.push(Box::new(battery_reading_handler)); + + let report_sync_event_handler = ReportSyncEventHandler::new( + config.config_file.enable_data_collection, + metric_report_manager.clone(), + ); + http_handlers.push(Box::new(report_sync_event_handler)); + + let session_event_handler = SessionEventHandler::new( + config.config_file.enable_data_collection, + metric_report_manager.clone(), + config.mar_staging_path(), + NetworkConfig::from(&config), + ); + http_handlers.push(Box::new(session_event_handler)); + + #[cfg(feature = "collectd")] + { + let collectd_handler = CollectdHandler::new( + config.config_file.enable_data_collection, + config.builtin_system_metric_collection_enabled(), + metric_report_manager.clone(), + ); + http_handlers.push(Box::new(collectd_handler)); + } + + // Start a thread to dump the metrics precisely every heartbeat interval + { + let net_config = NetworkConfig::from(&config); + let mar_staging_path = config.mar_staging_path(); + let heartbeat_interval = config.config_file.heartbeat_interval; + let metric_report_manager = metric_report_manager.clone(); + spawn(move || { + let periodic_metric_report_dumper = PeriodicMetricReportDumper::new( + mar_staging_path, + net_config, + metric_report_manager, + heartbeat_interval, + MetricReportType::Heartbeat, + ); + + periodic_metric_report_dumper.start(); + }); + } + + // Start statsd server + if config.statsd_server_enabled() { + let statsd_server = StatsDServer::new(); + if let Ok(bind_address) = config.statsd_server_address() { + if let Err(e) = statsd_server.start(bind_address, metric_report_manager.clone()) { + warn!("Couldn't start StatsD server: {}", e); + }; + } + } + + // Start system metric collector thread + if config.builtin_system_metric_collection_enabled() { + let metric_report_manager = metric_report_manager.clone(); + let poll_interval = config.system_metric_poll_interval(); + let system_metric_config = config.system_metric_config(); + spawn(move || { + let mut sys_metric_collector = SystemMetricsCollector::new(system_metric_config); + sys_metric_collector.run(poll_interval, metric_report_manager) + }); + } + + // Start a thread to dump metrics every 24 hours + if config.config_file.metrics.enable_daily_heartbeats { + let net_config = NetworkConfig::from(&config); + let mar_staging_path = config.mar_staging_path(); + let metric_report_manager = metric_report_manager.clone(); + spawn(move || { + let periodic_metric_report_dumper = PeriodicMetricReportDumper::new( + mar_staging_path, + net_config, + metric_report_manager, + DAILY_HEARTBEAT_INTERVAL, + MetricReportType::DailyHeartbeat, + ); + + periodic_metric_report_dumper.start(); + }); + } + + // Start a thread to update battery metrics + // periodically if enabled by configuration + if config.battery_monitor_periodic_update_enabled() { + let battery_monitor_interval = config.battery_monitor_interval(); + let battery_info_command_str = config.battery_monitor_battery_info_command().to_string(); + spawn(move || { + let mut next_battery_interval = Instant::now() + battery_monitor_interval; + loop { + while Instant::now() < next_battery_interval { + sleep(next_battery_interval - Instant::now()); + } + next_battery_interval += battery_monitor_interval; + let battery_info_command = Command::new(&battery_info_command_str); + if let Err(e) = battery_monitor + .lock() + .unwrap() + .update_via_command(battery_info_command) + { + warn!("Error updating battery monitor metrics: {}", e); + } + } + }); + } + // Connected time monitor is only enabled if config is defined + if let Some(connectivity_monitor_config) = config.connectivity_monitor_config() { + let mut connectivity_monitor = ConnectivityMonitor::::new( + connectivity_monitor_config, + metric_report_manager.clone(), + ); + spawn(move || { + let mut next_connectivity_reading_time = + Instant::now() + connectivity_monitor.interval_seconds(); + loop { + while Instant::now() < next_connectivity_reading_time { + sleep(next_connectivity_reading_time - Instant::now()); + } + next_connectivity_reading_time += connectivity_monitor.interval_seconds(); + if let Err(e) = connectivity_monitor.update_connected_time() { + warn!("Failed to update connected time metrics: {}", e); + } + } + }); + } + // Schedule a task to dump the metrics when a sync is forced + { + let net_config = NetworkConfig::from(&config); + let mar_staging_path = config.mar_staging_path(); + + let heartbeat_report_manager = metric_report_manager.clone(); + sync_tasks.push(Box::new(move |forced| match forced { + true => MetricReportManager::dump_report_to_mar_entry( + &heartbeat_report_manager, + &mar_staging_path, + &net_config, + &MetricReportType::Heartbeat, + ), + false => Ok(()), + })); + + let daily_heartbeat_report_manager = metric_report_manager.clone(); + let net_config = NetworkConfig::from(&config); + let mar_staging_path = config.mar_staging_path(); + sync_tasks.push(Box::new(move |forced| match forced { + true => { + trace!("Dumping daily heartbeat metrics"); + MetricReportManager::dump_report_to_mar_entry( + &daily_heartbeat_report_manager, + &mar_staging_path, + &net_config, + &MetricReportType::DailyHeartbeat, + ) + } + false => Ok(()), + })); + } + // Schedule a task to dump the metrics when we are shutting down + { + let net_config = NetworkConfig::from(&config); + let mar_staging_path = config.mar_staging_path(); + + let metric_report_manager = metric_report_manager.clone(); + shutdown_tasks.push(Box::new(move || { + MetricReportManager::dump_metric_reports( + &metric_report_manager, + &mar_staging_path, + &net_config, + ) + })); + } + // Schedule a task to compute operational and crashfree hours + { + let metric_report_manager = metric_report_manager.clone(); + + let mut crashfree_tracker = CrashFreeIntervalTracker::::new_hourly(); + http_handlers.push(crashfree_tracker.http_handler()); + spawn(move || { + let interval = Duration::from_secs(60); + loop { + let metrics = crashfree_tracker.wait_and_update(interval); + let mut store = metric_report_manager.lock().unwrap(); + trace!("Crashfree hours metrics: {:?}", metrics); + for m in metrics { + if let Err(e) = store.add_metric(m) { + warn!("Unable to add crashfree metric: {}", e); + } + } + } + }); + } + + #[cfg(feature = "logging")] + { + use crate::config::LogSource; + #[cfg(feature = "systemd")] + use crate::logs::journald_provider::start_journald_provider; + use crate::logs::log_entry::LogEntry; + use log::debug; + + let fluent_bit_config = FluentBitConfig::from(&config); + if config.config_file.enable_data_collection { + let log_source = config.config_file.logs.source; + let log_receiver: Box + Send> = match log_source { + LogSource::FluentBit => { + let (_, fluent_bit_receiver) = + FluentBitConnectionHandler::start(fluent_bit_config)?; + Box::new(FluentBitAdapter::new( + fluent_bit_receiver, + &config.config_file.fluent_bit.extra_fluentd_attributes, + )) + } + #[cfg(feature = "systemd")] + LogSource::Journald => Box::new(start_journald_provider(config.tmp_dir())), + }; + + let mar_cleaner = mar_cleaner.clone(); + + let network_config = NetworkConfig::from(&config); + let mar_staging_path = config.mar_staging_path(); + let on_log_completion = move |CompletedLog { + path, + cid, + next_cid, + compression, + }| + -> Result<()> { + // Prepare the MAR entry + let file_name = path + .file_name() + .ok_or(eyre!("Logfile should be a file."))? + .to_str() + .ok_or(eyre!("Invalid log filename."))? + .to_owned(); + let mar_builder = MarEntryBuilder::new(&mar_staging_path)? + .set_metadata(Metadata::new_log(file_name, cid, next_cid, compression)) + .add_attachment(path); + + mar_cleaner.clean(mar_builder.estimated_entry_size())?; + + // Move the log in the mar_staging area and add a manifest + let mar_entry = mar_builder.save(&network_config)?; + debug!( + "Logfile (cid: {}) saved as MAR entry: {}", + cid, + mar_entry.path.display() + ); + + Ok(()) + }; + let log_config = LogCollectorConfig::from(&config); + let headroom_limiter = { + let tmp_folder = log_config.log_tmp_path.clone(); + HeadroomLimiter::new(config.tmp_dir_min_headroom(), move || { + get_disk_space(&tmp_folder) + }) + }; + let mut log_collector = LogCollector::open( + log_config, + on_log_completion, + headroom_limiter, + metric_report_manager.clone(), + )?; + log_collector.spawn_collect_from(log_receiver); + + let crash_log_handler = log_collector.crash_log_handler(); + http_handlers.push(Box::new(crash_log_handler)); + + sync_tasks.push(Box::new(move |forced_sync| { + // Check if we have received a signal to force-sync and reset the flag. + if forced_sync { + trace!("Flushing logs"); + log_collector.flush_logs()?; + } else { + // If not force-flushing - we still want to make sure this file + // did not get too old. + log_collector.rotate_if_needed()?; + } + Ok(()) + })); + } else { + FluentBitConnectionHandler::start_null(fluent_bit_config)?; + } + } + + let reboot_tracker = RebootReasonTracker::new(&config, &service_manager); + if let Err(e) = reboot_tracker.track_reboot() { + error!("Unable to track reboot reason: {:#}", e); + } + + // Start the http server + let mut http_server = HttpServer::new(http_handlers); + http_server.start(config.config_file.http_server.bind_address)?; + + // Run the ready callback (creates the PID file) + ready_callback()?; + + let mut last_device_config_refresh = Option::::None; + + // If upload_interval is zero, we are only uploading on manual syncs. + let forced_sync_only = config.config_file.upload_interval.is_zero(); + // If we are only uploading on manual syncs, we still need to run the mar cleaner periodically. In + // this case set the the upload interval to 15 minutes. + let upload_interval = if forced_sync_only { + Duration::from_secs(60 * 15) + } else { + config.config_file.upload_interval + }; + loop_with_exponential_error_backoff( + || { + // Reset the forced sync flag before doing any work so we can detect + // if it's set again while we run and RerunImmediately. + let forced = force_sync.swap(false, Ordering::Relaxed); + let enable_data_collection = config.config_file.enable_data_collection; + + // Refresh device config if needed. In cases where we are only syncing on demand, we + // short-circuit this check. + if enable_data_collection + && (!forced_sync_only + && (last_device_config_refresh.is_none() + || last_device_config_refresh.unwrap() + CONFIG_REFRESH_INTERVAL + < Instant::now()) + || forced) + { + // Refresh device config from the server + match config.refresh_device_config(&client) { + Err(e) => { + warn!("Unable to refresh device config: {}", e); + // We continue processing the pending uploads on errors. + // We expect rate limiting errors here. + } + Ok(UpdateStatus::Updated) => { + info!("Device config updated"); + last_device_config_refresh = Some(Instant::now()) + } + Ok(UpdateStatus::Unchanged) => { + trace!("Device config unchanged"); + last_device_config_refresh = Some(Instant::now()) + } + } + } + + for task in &mut sync_tasks { + if let Err(e) = task(forced) { + warn!("{:#}", e); + } + } + + mar_cleaner.clean(DiskSize::ZERO).unwrap(); + + if enable_data_collection && !forced_sync_only || forced { + trace!("Collect MAR entries..."); + let result = collect_and_upload( + &config.mar_staging_path(), + &client, + config.config_file.mar.mar_file_max_size, + config.sampling(), + ); + let _metric_result = match result { + Ok(0) => Ok(()), + Ok(_count) => metric_report_manager + .lock() + .unwrap() + .increment_counter(METRIC_MF_SYNC_SUCCESS), + Err(_) => metric_report_manager + .lock() + .unwrap() + .increment_counter(METRIC_MF_SYNC_FAILURE), + }; + return result.map(|_| ()); + } + Ok(()) + }, + || match ( + term.load(Ordering::Relaxed) || reload.load(Ordering::Relaxed), + force_sync.load(Ordering::Relaxed), + ) { + // Stop when we receive a term signal + (true, _) => LoopContinuation::Stop, + // If we received a SIGUSR1 signal while we were in the loop, rerun immediately. + (false, true) => LoopContinuation::RerunImmediately, + // Otherwise, keep running normally + (false, false) => LoopContinuation::KeepRunning, + }, + upload_interval, + Duration::new(60, 0), + ); + info!("Memfaultd shutting down..."); + for task in &mut shutdown_tasks { + if let Err(e) = task() { + warn!("Error while shutting down: {}", e); + } + } + + if reload.load(Ordering::Relaxed) { + Ok(MemfaultLoopResult::Relaunch) + } else { + Ok(MemfaultLoopResult::Terminate) + } +} diff --git a/memfaultd/src/metrics/battery/battery_monitor.rs b/memfaultd/src/metrics/battery/battery_monitor.rs new file mode 100644 index 0000000..bbb84af --- /dev/null +++ b/memfaultd/src/metrics/battery/battery_monitor.rs @@ -0,0 +1,365 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use chrono::{DateTime, Utc}; + +use std::ops::Sub; +use std::process::Command; +use std::str::FromStr; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use eyre::{eyre, ErrReport, Result}; + +use crate::metrics::{ + core_metrics::{ + METRIC_BATTERY_DISCHARGE_DURATION_MS, METRIC_BATTERY_SOC_PCT, METRIC_BATTERY_SOC_PCT_DROP, + }, + KeyedMetricReading, MetricReading, MetricReportManager, MetricStringKey, +}; +use crate::util::time_measure::TimeMeasure; + +// These states are based off the valid values for +// sys/class/power_supply//status +// Read more here: +// https://www.kernel.org/doc/Documentation/ABI/testing/sysfs-class-power +enum ChargingState { + Charging, + Discharging, + Full, + Unknown, + NotCharging, + Invalid, +} + +// A single reading that describes +// the state of the device's battery +pub struct BatteryMonitorReading { + battery_soc_pct: f64, + battery_charging_state: ChargingState, +} + +impl BatteryMonitorReading { + fn new(battery_soc_pct: f64, battery_charging_state: ChargingState) -> BatteryMonitorReading { + BatteryMonitorReading { + battery_soc_pct, + battery_charging_state, + } + } +} + +impl FromStr for BatteryMonitorReading { + type Err = ErrReport; + + fn from_str(s: &str) -> Result { + if let Some((state_str, pct_str)) = s.trim().split_once(':') { + let charging_state = match state_str { + "Charging" => ChargingState::Charging, + "Discharging" => ChargingState::Discharging, + "Full" => ChargingState::Full, + "Not charging" => ChargingState::NotCharging, + "Unknown" => ChargingState::Unknown, + _ => ChargingState::Invalid, + }; + let pct = pct_str.parse::(); + match (charging_state, pct) { + (ChargingState::Invalid, _) => Err(eyre!("Invalid charging state: {}", state_str)), + (_, Err(e)) => Err(eyre!("Couldn't parse battery percentage: {}", e)), + (charging_state, Ok(p)) => { + if (0.0..=100.0).contains(&p) { + Ok(BatteryMonitorReading::new(p, charging_state)) + } else { + Err(eyre!( + "Battery SOC percentage value {} is not in the range [0.0, 100.0]!", + p + )) + } + } + } + } else { + Err(eyre!( + "Invalid output from command configured via `battery_percentage_command`" + )) + } + } +} + +// Since some of the battery metrics recorded +// are calculated based on both the current and previous reading +// (such as battery_soc_pct_drop), this struct needs to +// store the previous battery percentage as well as when that +// perecentage was recorded. +pub struct BatteryMonitor { + previous_reading: Option, + last_reading_time: T, + heartbeat_manager: Arc>, +} + +impl BatteryMonitor +where + T: TimeMeasure + Copy + Ord + Sub, +{ + pub fn new(heartbeat_manager: Arc>) -> Self { + Self { + previous_reading: None, + last_reading_time: T::now(), + heartbeat_manager, + } + } + + // Writes new values for battery_discharge_duration_ms, battery_soc_pct_drop, + // and battery_soc_pct to the in memory metric store + fn update_metrics( + &mut self, + battery_monitor_reading: BatteryMonitorReading, + reading_time: T, + wall_time: DateTime, + ) -> Result<()> { + let reading_duration = reading_time.since(&self.last_reading_time); + match ( + &battery_monitor_reading.battery_charging_state, + &self.previous_reading, + ) { + // Update battery discharge metrics only when there is a previous + // reading and both the previous AND current + // charging state are Discharging + ( + ChargingState::Discharging, + Some(BatteryMonitorReading { + battery_soc_pct: previous_soc_pct, + battery_charging_state: ChargingState::Discharging, + }), + ) => { + let soc_pct = battery_monitor_reading.battery_soc_pct; + let soc_pct_discharged = + (previous_soc_pct - battery_monitor_reading.battery_soc_pct).max(0.0); + + let mut heartbeat_manager = self.heartbeat_manager.lock().expect("Mutex Poisoned"); + heartbeat_manager.add_to_counter( + METRIC_BATTERY_DISCHARGE_DURATION_MS, + reading_duration.as_millis() as f64, + )?; + + heartbeat_manager + .add_to_counter(METRIC_BATTERY_SOC_PCT_DROP, soc_pct_discharged)?; + + let battery_soc_pct_key = MetricStringKey::from_str(METRIC_BATTERY_SOC_PCT) + .unwrap_or_else(|_| panic!("Invalid metric name: {}", METRIC_BATTERY_SOC_PCT)); + heartbeat_manager.add_metric(KeyedMetricReading::new( + battery_soc_pct_key, + MetricReading::TimeWeightedAverage { + value: soc_pct, + timestamp: wall_time, + interval: chrono::Duration::from_std(reading_duration)?, + }, + ))?; + } + // In all other cases only update the SoC percent + _ => { + let soc_pct = battery_monitor_reading.battery_soc_pct; + + let mut heartbeat_manager = self.heartbeat_manager.lock().expect("Mutex Poisoned"); + + // Add 0.0 to these counters so if the device is charging + // for the full heartbeat duration these metrics are still + // populated + heartbeat_manager.add_to_counter(METRIC_BATTERY_DISCHARGE_DURATION_MS, 0.0)?; + heartbeat_manager.add_to_counter(METRIC_BATTERY_SOC_PCT_DROP, 0.0)?; + + let battery_soc_pct_key = MetricStringKey::from_str(METRIC_BATTERY_SOC_PCT) + .unwrap_or_else(|_| panic!("Invalid metric name: {}", METRIC_BATTERY_SOC_PCT)); + heartbeat_manager.add_metric(KeyedMetricReading::new( + battery_soc_pct_key, + MetricReading::TimeWeightedAverage { + value: soc_pct, + timestamp: wall_time, + interval: chrono::Duration::from_std(reading_duration)?, + }, + ))?; + } + } + + self.previous_reading = Some(battery_monitor_reading); + self.last_reading_time = reading_time; + + Ok(()) + } + + pub fn update_via_command(&mut self, mut battery_info_command: Command) -> Result<()> { + let battery_info_output = battery_info_command.output()?; + if !battery_info_output.status.success() { + Err(eyre!( + "Failed to execute {}. Battery percentage was not captured.", + battery_info_command.get_program().to_string_lossy() + )) + } else { + let output_string = String::from_utf8(battery_info_output.stdout)?; + let battery_monitor_reading = BatteryMonitorReading::from_str(&output_string)?; + self.add_new_reading(battery_monitor_reading)?; + Ok(()) + } + } + + pub fn add_new_reading( + &mut self, + battery_monitor_reading: BatteryMonitorReading, + ) -> Result<()> { + self.update_metrics(battery_monitor_reading, T::now(), Utc::now())?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + + use super::*; + use crate::metrics::MetricValue; + use crate::test_utils::TestInstant; + use rstest::rstest; + + #[rstest] + #[case("Charging:80", true)] + #[case("Discharging:80", true)] + #[case("Not charging:80", true)] + #[case("Isn't charging:80", false)] + #[case("Charging:EIGHTY", false)] + #[case("Charging:42.5", true)] + #[case("Charging:42.five", false)] + #[case("Charging:42.3.5", false)] + #[case("Charging:-1", false)] + #[case("Discharging:100.1", false)] + #[case("Discharging:100.0", true)] + #[case("Discharging:0.0", true)] + #[case("Discharging:-0.1", false)] + #[case("Full:100.0", true)] + #[case("Unknown:80", true)] + fn test_parse(#[case] cmd_output: &str, #[case] is_ok: bool) { + assert_eq!(BatteryMonitorReading::from_str(cmd_output).is_ok(), is_ok); + } + + #[rstest] + // Single reading results in expected metrics + #[case(vec![BatteryMonitorReading::new(90.0, ChargingState::Charging)], 30, 90.0, 0.0, 0.0)] + #[case(vec![BatteryMonitorReading::new(90.0, ChargingState::Charging), BatteryMonitorReading::new(100.0, ChargingState::Charging)], 30, 95.0, 0.0, 0.0)] + // Battery discharges between readings + #[case(vec![BatteryMonitorReading::new(90.0, ChargingState::Discharging), BatteryMonitorReading::new(85.0, ChargingState::Discharging)], 30, 87.5, 5.0, 30000.0)] + // Battery is discharging, then charging, then discharging again + #[case(vec![BatteryMonitorReading::new(90.0, ChargingState::Discharging), + BatteryMonitorReading::new(85.0, ChargingState::Discharging), + BatteryMonitorReading::new(90.0, ChargingState::Charging), + BatteryMonitorReading::new(90.0, ChargingState::Discharging), + BatteryMonitorReading::new(80.0, ChargingState::Discharging)], + 30, + 87.0, + 15.0, + 60000.0)] + // Continuous discharge + #[case(vec![BatteryMonitorReading::new(90.0, ChargingState::Discharging), + BatteryMonitorReading::new(80.0, ChargingState::Discharging), + BatteryMonitorReading::new(70.0, ChargingState::Discharging), + BatteryMonitorReading::new(60.0, ChargingState::Discharging)], + 30, + 75.0, + 30.0, + 90000.0)] + // Continuous charge + #[case(vec![BatteryMonitorReading::new(60.0, ChargingState::Charging), + BatteryMonitorReading::new(70.0, ChargingState::Charging), + BatteryMonitorReading::new(80.0, ChargingState::Charging), + BatteryMonitorReading::new(90.0, ChargingState::Charging)], + 30, + 75.0, + 0.0, + 0.0)] + // Battery was charged in between monitoring calls + #[case(vec![BatteryMonitorReading::new(60.0, ChargingState::Discharging), + BatteryMonitorReading::new(80.0, ChargingState::Discharging),], + 30, + 70.0, + 0.0, + 30000.0)] + // Discharge then charge to full + #[case(vec![BatteryMonitorReading::new(90.0, ChargingState::Discharging), + BatteryMonitorReading::new(80.0, ChargingState::Discharging), + BatteryMonitorReading::new(70.0, ChargingState::Discharging), + BatteryMonitorReading::new(80.0, ChargingState::Charging), + BatteryMonitorReading::new(100.0, ChargingState::Full)], + 30, + 84.0, + 20.0, + 60000.0)] + // Check unknown and not charging states + #[case(vec![BatteryMonitorReading::new(60.0, ChargingState::Charging), + BatteryMonitorReading::new(70.0, ChargingState::Charging), + BatteryMonitorReading::new(80.0, ChargingState::Unknown), + BatteryMonitorReading::new(90.0, ChargingState::NotCharging)], + 30, + 75.0, + 0.0, + 0.0)] + // Check measurements with 0 seconds between + #[case(vec![BatteryMonitorReading::new(80.0, ChargingState::Charging), + BatteryMonitorReading::new(85.0, ChargingState::NotCharging), + BatteryMonitorReading::new(90.0, ChargingState::NotCharging)], + 0, + f64::NAN, + 0.0, + 0.0)] + fn test_update_metrics_soc_pct( + #[case] battery_monitor_readings: Vec, + #[case] seconds_between_readings: u64, + #[case] expected_soc_pct: f64, + #[case] expected_soc_pct_discharge: f64, + #[case] expected_discharge_duration: f64, + ) { + let now = TestInstant::now(); + let heartbeat_manager = Arc::new(Mutex::new(MetricReportManager::new())); + let mut battery_monitor = BatteryMonitor { + heartbeat_manager, + last_reading_time: now, + previous_reading: None, + }; + + let mut ts = Utc::now(); + for reading in battery_monitor_readings { + TestInstant::sleep(Duration::from_secs(seconds_between_readings)); + ts += chrono::Duration::seconds(seconds_between_readings as i64); + battery_monitor + .update_metrics(reading, TestInstant::now(), ts) + .unwrap(); + } + let metrics = battery_monitor + .heartbeat_manager + .lock() + .unwrap() + .take_heartbeat_metrics(); + let soc_pct_key = METRIC_BATTERY_SOC_PCT.parse::().unwrap(); + + match metrics.get(&soc_pct_key).unwrap() { + MetricValue::Number(e) => { + if expected_soc_pct.is_finite() { + assert_eq!(*e, expected_soc_pct); + } else { + assert!(e.is_nan()); + } + } + _ => panic!("This test only expects number metric values!"), + } + + let soc_pct_discharge_key = METRIC_BATTERY_SOC_PCT_DROP + .parse::() + .unwrap(); + match metrics.get(&soc_pct_discharge_key).unwrap() { + MetricValue::Number(e) => assert_eq!(*e, expected_soc_pct_discharge), + _ => panic!("This test only expects number metric values!"), + } + + let soc_discharge_duration_key = METRIC_BATTERY_DISCHARGE_DURATION_MS + .parse::() + .unwrap(); + match metrics.get(&soc_discharge_duration_key).unwrap() { + MetricValue::Number(e) => assert_eq!(*e, expected_discharge_duration), + _ => panic!("This test only expects number metric values!"), + } + } +} diff --git a/memfaultd/src/metrics/battery/battery_reading_handler.rs b/memfaultd/src/metrics/battery/battery_reading_handler.rs new file mode 100644 index 0000000..ba9d177 --- /dev/null +++ b/memfaultd/src/metrics/battery/battery_reading_handler.rs @@ -0,0 +1,188 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + io::Read, + ops::Sub, + str::{from_utf8, FromStr}, + sync::{Arc, Mutex}, + time::Duration, +}; + +use eyre::Result; +use tiny_http::{Method, Request, Response}; + +use crate::util::time_measure::TimeMeasure; +use crate::{ + http_server::{HttpHandler, HttpHandlerResult}, + metrics::{BatteryMonitor, BatteryMonitorReading}, +}; + +/// A server that listens for battery reading pushes and stores them in memory. +#[derive(Clone)] +pub struct BatteryReadingHandler { + data_collection_enabled: bool, + battery_monitor: Arc>>, +} + +impl BatteryReadingHandler +where + T: TimeMeasure + Copy + Ord + Sub + Send + Sync, +{ + pub fn new( + data_collection_enabled: bool, + battery_monitor: Arc>>, + ) -> Self { + Self { + data_collection_enabled, + battery_monitor, + } + } + + fn parse_request(stream: &mut dyn Read) -> Result { + let mut buf = vec![]; + stream.read_to_end(&mut buf)?; + let reading = BatteryMonitorReading::from_str(from_utf8(&buf)?)?; + Ok(reading) + } +} + +impl HttpHandler for BatteryReadingHandler +where + T: TimeMeasure + Copy + Ord + Sub + Send + Sync, +{ + fn handle_request(&self, request: &mut Request) -> HttpHandlerResult { + if request.url() != "/v1/battery/add_reading" || *request.method() != Method::Post { + return HttpHandlerResult::NotHandled; + } + if self.data_collection_enabled { + match Self::parse_request(request.as_reader()) { + Ok(reading) => { + match self + .battery_monitor + .lock() + .unwrap() + .add_new_reading(reading) + { + Ok(()) => HttpHandlerResult::Response(Response::empty(200).boxed()), + Err(e) => HttpHandlerResult::Error(format!( + "Failed to add battery reading to metrics: {:#}", + e + )), + } + } + Err(e) => HttpHandlerResult::Error(format!( + "Failed to parse battery reading string: {}", + e + )), + } + } else { + HttpHandlerResult::Response(Response::empty(200).boxed()) + } + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, + time::Duration, + }; + + use insta::assert_json_snapshot; + use rstest::rstest; + use tiny_http::{Method, TestRequest}; + + use crate::test_utils::TestInstant; + use crate::{ + http_server::{HttpHandler, HttpHandlerResult}, + metrics::{BatteryMonitor, MetricReportManager}, + }; + + use super::BatteryReadingHandler; + #[rstest] + fn handle_push() { + let heartbeat_manager = Arc::new(Mutex::new(MetricReportManager::new())); + let handler = BatteryReadingHandler::new( + true, + Arc::new(Mutex::new(BatteryMonitor::::new( + heartbeat_manager.clone(), + ))), + ); + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/battery/add_reading") + .with_body("Charging:80"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + let metrics = heartbeat_manager.lock().unwrap().take_heartbeat_metrics(); + + // Need to sort the map so the JSON string is consistent + let sorted_metrics: BTreeMap<_, _> = metrics.iter().collect(); + + assert_json_snapshot!(&sorted_metrics); + } + + // Need to include a test_name string parameter here due to + // a known issue using insta and rstest together: + // https://github.com/la10736/rstest/issues/183 + #[rstest] + #[case(vec!["Charging:80", "Charging:90", "Full:100", "Discharging:95", "Discharging:85"], 30, "charging_then_discharging")] + #[case(vec!["Full:100", "Discharging:90", "Discharging:50", "Not charging:50", "Discharging:30", "Discharging:10", "Charging:50"], 30, "nonconsecutive_discharges")] + #[case(vec!["Charging:90", "Charging:92.465", "Unknown:91.78", "Discharging:90", "Discharging:80"], 30, "non_integer_percentages")] + fn handle_push_of_multiple_readings( + #[case] readings: Vec<&'static str>, + #[case] seconds_between_readings: u64, + #[case] test_name: &str, + ) { + let heartbeat_manager = Arc::new(Mutex::new(MetricReportManager::new())); + let handler = BatteryReadingHandler::new( + true, + Arc::new(Mutex::new(BatteryMonitor::::new( + heartbeat_manager.clone(), + ))), + ); + for reading in readings { + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/battery/add_reading") + .with_body(reading); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + TestInstant::sleep(Duration::from_secs(seconds_between_readings)); + } + + let metrics = heartbeat_manager.lock().unwrap().take_heartbeat_metrics(); + + // Need to sort the map so the JSON string is consistent + let sorted_metrics: BTreeMap<_, _> = metrics.iter().collect(); + + // Set battery_soc_pct to 0.0 to avoid flakiness due to it being weighted by wall time + assert_json_snapshot!(test_name, &sorted_metrics, {".battery_soc_pct" => 0.0 }); + } + + #[rstest] + fn errors_when_body_is_invalid() { + let heartbeat_manager = Arc::new(Mutex::new(MetricReportManager::new())); + let handler = BatteryReadingHandler::::new( + true, + Arc::new(Mutex::new(BatteryMonitor::::new( + heartbeat_manager, + ))), + ); + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/battery/add_reading") + .with_body("{\"state\": \"Charging\", \"percent\":80}"); + matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Error(_) + ); + } +} diff --git a/memfaultd/src/metrics/battery/mod.rs b/memfaultd/src/metrics/battery/mod.rs new file mode 100644 index 0000000..e78f15a --- /dev/null +++ b/memfaultd/src/metrics/battery/mod.rs @@ -0,0 +1,9 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +mod battery_reading_handler; +pub use battery_reading_handler::BatteryReadingHandler; + +mod battery_monitor; +pub use battery_monitor::BatteryMonitor; +pub use battery_monitor::BatteryMonitorReading; diff --git a/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__charging_then_discharging.snap b/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__charging_then_discharging.snap new file mode 100644 index 0000000..b792236 --- /dev/null +++ b/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__charging_then_discharging.snap @@ -0,0 +1,9 @@ +--- +source: memfaultd/src/metrics/battery/battery_reading_handler.rs +expression: "&sorted_metrics" +--- +{ + "battery_discharge_duration_ms": 30000.0, + "battery_soc_pct": 0.0, + "battery_soc_pct_drop": 10.0 +} diff --git a/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__handle_push.snap b/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__handle_push.snap new file mode 100644 index 0000000..1d4cf88 --- /dev/null +++ b/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__handle_push.snap @@ -0,0 +1,9 @@ +--- +source: memfaultd/src/metrics/battery/battery_reading_handler.rs +expression: "serde_json::to_string_pretty(&sorted_metrics).expect(\"metric_store should be serializable\")" +--- +{ + "battery_discharge_duration_ms": 0.0, + "battery_soc_pct": null, + "battery_soc_pct_drop": 0.0 +} diff --git a/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__ignores_data_when_data_collection_is_off.snap b/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__ignores_data_when_data_collection_is_off.snap new file mode 100644 index 0000000..4d04193 --- /dev/null +++ b/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__ignores_data_when_data_collection_is_off.snap @@ -0,0 +1,5 @@ +--- +source: memfaultd/src/metrics/battery/battery_reading_handler.rs +expression: "serde_json::to_string_pretty(&metrics).expect(\"metric_store should be serializable\")" +--- +{} diff --git a/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__non_integer_percentages.snap b/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__non_integer_percentages.snap new file mode 100644 index 0000000..b792236 --- /dev/null +++ b/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__non_integer_percentages.snap @@ -0,0 +1,9 @@ +--- +source: memfaultd/src/metrics/battery/battery_reading_handler.rs +expression: "&sorted_metrics" +--- +{ + "battery_discharge_duration_ms": 30000.0, + "battery_soc_pct": 0.0, + "battery_soc_pct_drop": 10.0 +} diff --git a/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__nonconsecutive_discharges.snap b/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__nonconsecutive_discharges.snap new file mode 100644 index 0000000..07dd8ff --- /dev/null +++ b/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__nonconsecutive_discharges.snap @@ -0,0 +1,9 @@ +--- +source: memfaultd/src/metrics/battery/battery_reading_handler.rs +expression: "&sorted_metrics" +--- +{ + "battery_discharge_duration_ms": 60000.0, + "battery_soc_pct": 0.0, + "battery_soc_pct_drop": 60.0 +} diff --git a/memfaultd/src/metrics/connectivity/connectivity_monitor.rs b/memfaultd/src/metrics/connectivity/connectivity_monitor.rs new file mode 100644 index 0000000..79e0081 --- /dev/null +++ b/memfaultd/src/metrics/connectivity/connectivity_monitor.rs @@ -0,0 +1,217 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + ops::Sub, + sync::{Arc, Mutex}, + time::Duration, +}; + +use eyre::Result; + +use crate::{ + config::{ConnectivityMonitorConfig, ConnectivityMonitorTarget}, + metrics::{ + core_metrics::{METRIC_CONNECTED_TIME, METRIC_EXPECTED_CONNECTED_TIME}, + MetricReportManager, + }, + util::{can_connect::CanConnect, time_measure::TimeMeasure}, +}; + +pub struct ConnectivityMonitor { + targets: Vec, + interval: Duration, + last_checked_at: Option, + heartbeat_manager: Arc>, + connection_checker: U, +} + +impl ConnectivityMonitor +where + T: TimeMeasure + Copy + Ord + Sub, + U: CanConnect, +{ + pub fn new( + config: &ConnectivityMonitorConfig, + heartbeat_manager: Arc>, + ) -> Self { + Self { + targets: config.targets.clone(), + interval: config.interval_seconds, + last_checked_at: None, + heartbeat_manager, + connection_checker: U::new(config.timeout_seconds), + } + } + + fn is_connected(&self) -> bool { + self.targets + .iter() + .any(|ConnectivityMonitorTarget { host, port, .. }| { + self.connection_checker.can_connect(host, *port).is_ok() + }) + } + + pub fn update_connected_time(&mut self) -> Result<()> { + let now = T::now(); + let since_last_reading = self.last_checked_at.unwrap_or(now).elapsed(); + let connected_duration = if self.is_connected() { + since_last_reading + } else { + Duration::ZERO + }; + + let mut store = self.heartbeat_manager.lock().expect("Mutex Poisoned"); + store.add_to_counter(METRIC_CONNECTED_TIME, connected_duration.as_millis() as f64)?; + store.add_to_counter( + METRIC_EXPECTED_CONNECTED_TIME, + since_last_reading.as_millis() as f64, + )?; + + self.last_checked_at = Some(now); + + Ok(()) + } + + pub fn interval_seconds(&self) -> Duration { + self.interval + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::BTreeMap, + net::IpAddr, + str::FromStr, + sync::{Arc, Mutex}, + time::Duration, + }; + + use insta::assert_json_snapshot; + use rstest::rstest; + + use super::ConnectivityMonitor; + use crate::test_utils::{TestConnectionChecker, TestInstant}; + use crate::{ + config::{ConnectionCheckProtocol, ConnectivityMonitorConfig, ConnectivityMonitorTarget}, + metrics::MetricReportManager, + }; + + #[rstest] + fn test_while_connected() { + let heartbeat_manager = Arc::new(Mutex::new(MetricReportManager::new())); + let config = ConnectivityMonitorConfig { + targets: vec![ConnectivityMonitorTarget { + host: IpAddr::from_str("8.8.8.8").unwrap(), + port: 443, + protocol: ConnectionCheckProtocol::Tcp, + }], + interval_seconds: Duration::from_secs(15), + timeout_seconds: Duration::from_secs(10), + }; + let mut connectivity_monitor = + ConnectivityMonitor::::new( + &config, + heartbeat_manager.clone(), + ); + + TestConnectionChecker::connect(); + + connectivity_monitor + .update_connected_time() + .expect("Couldn't update connected time monitor!"); + + TestInstant::sleep(Duration::from_secs(30)); + connectivity_monitor + .update_connected_time() + .expect("Couldn't update connected time monitor!"); + + let metrics = heartbeat_manager.lock().unwrap().take_heartbeat_metrics(); + + // Need to sort the map so the JSON string is consistent + let sorted_metrics: BTreeMap<_, _> = metrics.iter().collect(); + + assert_json_snapshot!(sorted_metrics); + } + + #[rstest] + fn test_half_connected_half_disconnected() { + let heartbeat_manager = Arc::new(Mutex::new(MetricReportManager::new())); + let config = ConnectivityMonitorConfig { + targets: vec![ConnectivityMonitorTarget { + host: IpAddr::from_str("8.8.8.8").unwrap(), + port: 443, + protocol: ConnectionCheckProtocol::Tcp, + }], + interval_seconds: Duration::from_secs(15), + timeout_seconds: Duration::from_secs(10), + }; + let mut connectivity_monitor = + ConnectivityMonitor::::new( + &config, + heartbeat_manager.clone(), + ); + + TestConnectionChecker::connect(); + + // Initial reading + connectivity_monitor.update_connected_time().unwrap(); + + TestInstant::sleep(Duration::from_secs(30)); + connectivity_monitor + .update_connected_time() + .expect("Couldn't update connected time monitor!"); + + TestConnectionChecker::disconnect(); + + TestInstant::sleep(Duration::from_secs(30)); + connectivity_monitor + .update_connected_time() + .expect("Couldn't update connected time monitor!"); + let metrics = heartbeat_manager.lock().unwrap().take_heartbeat_metrics(); + + // Need to sort the map so the JSON string is consistent + let sorted_metrics: BTreeMap<_, _> = metrics.iter().collect(); + assert_json_snapshot!(sorted_metrics); + } + + #[rstest] + fn test_fully_disconnected() { + let heartbeat_manager = Arc::new(Mutex::new(MetricReportManager::new())); + let config = ConnectivityMonitorConfig { + targets: vec![ConnectivityMonitorTarget { + host: IpAddr::from_str("8.8.8.8").unwrap(), + port: 443, + protocol: ConnectionCheckProtocol::Tcp, + }], + interval_seconds: Duration::from_secs(15), + timeout_seconds: Duration::from_secs(10), + }; + let mut connectivity_monitor = + ConnectivityMonitor::::new( + &config, + heartbeat_manager.clone(), + ); + + TestConnectionChecker::disconnect(); + + // Initial reading + connectivity_monitor.update_connected_time().unwrap(); + + TestInstant::sleep(Duration::from_secs(30)); + connectivity_monitor + .update_connected_time() + .expect("Couldn't update connected time monitor!"); + TestInstant::sleep(Duration::from_secs(30)); + connectivity_monitor + .update_connected_time() + .expect("Couldn't update connected time monitor!"); + let metrics = heartbeat_manager.lock().unwrap().take_heartbeat_metrics(); + + // Need to sort the map so the JSON string is consistent + let sorted_metrics: BTreeMap<_, _> = metrics.iter().collect(); + + assert_json_snapshot!(sorted_metrics); + } +} diff --git a/memfaultd/src/metrics/connectivity/mod.rs b/memfaultd/src/metrics/connectivity/mod.rs new file mode 100644 index 0000000..6f4aa52 --- /dev/null +++ b/memfaultd/src/metrics/connectivity/mod.rs @@ -0,0 +1,8 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +mod connectivity_monitor; +pub use connectivity_monitor::ConnectivityMonitor; + +mod report_sync_event_handler; +pub use report_sync_event_handler::ReportSyncEventHandler; diff --git a/memfaultd/src/metrics/connectivity/report_sync_event_handler.rs b/memfaultd/src/metrics/connectivity/report_sync_event_handler.rs new file mode 100644 index 0000000..52e1cba --- /dev/null +++ b/memfaultd/src/metrics/connectivity/report_sync_event_handler.rs @@ -0,0 +1,150 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::sync::{Arc, Mutex}; + +use log::warn; +use tiny_http::{Method, Request, Response}; + +use crate::{ + http_server::{HttpHandler, HttpHandlerResult}, + metrics::{ + core_metrics::{METRIC_SYNC_FAILURE, METRIC_SYNC_SUCCESS}, + MetricReportManager, + }, +}; + +/// A server that listens for collectd JSON pushes and stores them in memory. +#[derive(Clone)] +pub struct ReportSyncEventHandler { + data_collection_enabled: bool, + metrics_store: Arc>, +} + +impl ReportSyncEventHandler { + pub fn new( + data_collection_enabled: bool, + metrics_store: Arc>, + ) -> Self { + Self { + data_collection_enabled, + metrics_store, + } + } +} + +impl HttpHandler for ReportSyncEventHandler { + fn handle_request(&self, request: &mut Request) -> HttpHandlerResult { + if (request.url() != "/v1/sync/success" && request.url() != "/v1/sync/failure") + || *request.method() != Method::Post + { + return HttpHandlerResult::NotHandled; + } + if self.data_collection_enabled { + if request.url() == "/v1/sync/success" { + let mut metrics_store = self.metrics_store.lock().unwrap(); + if let Err(e) = metrics_store.increment_counter(METRIC_SYNC_SUCCESS) { + warn!("Couldn't increment sync_success counter: {:#}", e); + } + } else if request.url() == "/v1/sync/failure" { + let mut metrics_store = self.metrics_store.lock().unwrap(); + if let Err(e) = metrics_store.increment_counter(METRIC_SYNC_FAILURE) { + warn!("Couldn't increment sync_failure counter: {:#}", e); + } + } + } + HttpHandlerResult::Response(Response::empty(200).boxed()) + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, + }; + + use insta::assert_json_snapshot; + use rstest::{fixture, rstest}; + use tiny_http::{Method, TestRequest}; + + use crate::{ + http_server::{HttpHandler, HttpHandlerResult}, + metrics::MetricReportManager, + }; + + use super::ReportSyncEventHandler; + + #[rstest] + fn handle_sync_success(handler: ReportSyncEventHandler) { + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/sync/success"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + let metrics = handler + .metrics_store + .lock() + .unwrap() + .take_heartbeat_metrics(); + assert_json_snapshot!(&metrics); + } + + #[rstest] + fn handle_sync_failure(handler: ReportSyncEventHandler) { + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/sync/failure"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + let metrics = handler + .metrics_store + .lock() + .unwrap() + .take_heartbeat_metrics(); + assert_json_snapshot!(&metrics); + } + + #[rstest] + fn handle_multiple_sync_events(handler: ReportSyncEventHandler) { + for _ in 0..10 { + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/sync/failure"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + } + + for _ in 0..90 { + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/sync/success"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + } + let metrics = handler + .metrics_store + .lock() + .unwrap() + .take_heartbeat_metrics(); + // Need to sort the map so the JSON string is consistent + let sorted_metrics: BTreeMap<_, _> = metrics.iter().collect(); + + assert_json_snapshot!(&sorted_metrics); + } + + #[fixture] + fn handler() -> ReportSyncEventHandler { + ReportSyncEventHandler::new(true, Arc::new(Mutex::new(MetricReportManager::new()))) + } +} diff --git a/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__connectivity_monitor__tests__fully_disconnected.snap b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__connectivity_monitor__tests__fully_disconnected.snap new file mode 100644 index 0000000..70b69e3 --- /dev/null +++ b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__connectivity_monitor__tests__fully_disconnected.snap @@ -0,0 +1,8 @@ +--- +source: memfaultd/src/metrics/connectivity/connectivity_monitor.rs +expression: sorted_metrics +--- +{ + "connectivity_connected_time_ms": 0.0, + "connectivity_expected_time_ms": 60000.0 +} diff --git a/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__connectivity_monitor__tests__half_connected_half_disconnected.snap b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__connectivity_monitor__tests__half_connected_half_disconnected.snap new file mode 100644 index 0000000..e411ed1 --- /dev/null +++ b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__connectivity_monitor__tests__half_connected_half_disconnected.snap @@ -0,0 +1,8 @@ +--- +source: memfaultd/src/metrics/connectivity/connectivity_monitor.rs +expression: sorted_metrics +--- +{ + "connectivity_connected_time_ms": 30000.0, + "connectivity_expected_time_ms": 60000.0 +} diff --git a/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__connectivity_monitor__tests__while_connected.snap b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__connectivity_monitor__tests__while_connected.snap new file mode 100644 index 0000000..87fa147 --- /dev/null +++ b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__connectivity_monitor__tests__while_connected.snap @@ -0,0 +1,8 @@ +--- +source: memfaultd/src/metrics/connectivity/connectivity_monitor.rs +expression: sorted_metrics +--- +{ + "connectivity_connected_time_ms": 30000.0, + "connectivity_expected_time_ms": 30000.0 +} diff --git a/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__report_sync_event_handler__tests__handle_multiple_sync_events.snap b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__report_sync_event_handler__tests__handle_multiple_sync_events.snap new file mode 100644 index 0000000..83861ef --- /dev/null +++ b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__report_sync_event_handler__tests__handle_multiple_sync_events.snap @@ -0,0 +1,8 @@ +--- +source: memfaultd/src/metrics/connectivity/report_sync_event_handler.rs +expression: "serde_json::to_string_pretty(&sorted_metrics).expect(\"metric_store should be serializable\")" +--- +{ + "sync_failure": 10.0, + "sync_successful": 90.0 +} diff --git a/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__report_sync_event_handler__tests__handle_sync_failure.snap b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__report_sync_event_handler__tests__handle_sync_failure.snap new file mode 100644 index 0000000..d6079b6 --- /dev/null +++ b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__report_sync_event_handler__tests__handle_sync_failure.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/metrics/connectivity/report_sync_event_handler.rs +expression: "serde_json::to_string_pretty(&metrics).expect(\"metric_store should be serializable\")" +--- +{ + "sync_failure": 1.0 +} diff --git a/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__report_sync_event_handler__tests__handle_sync_success.snap b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__report_sync_event_handler__tests__handle_sync_success.snap new file mode 100644 index 0000000..9e75119 --- /dev/null +++ b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__report_sync_event_handler__tests__handle_sync_success.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/metrics/connectivity/report_sync_event_handler.rs +expression: "serde_json::to_string_pretty(&metrics).expect(\"metric_store should be serializable\")" +--- +{ + "sync_successful": 1.0 +} diff --git a/memfaultd/src/metrics/core_metrics.rs b/memfaultd/src/metrics/core_metrics.rs new file mode 100644 index 0000000..408f96c --- /dev/null +++ b/memfaultd/src/metrics/core_metrics.rs @@ -0,0 +1,20 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +// Connectivity metrics +pub const METRIC_MF_SYNC_SUCCESS: &str = "sync_memfault_successful"; +pub const METRIC_MF_SYNC_FAILURE: &str = "sync_memfault_failure"; +pub const METRIC_CONNECTED_TIME: &str = "connectivity_connected_time_ms"; +pub const METRIC_EXPECTED_CONNECTED_TIME: &str = "connectivity_expected_time_ms"; +pub const METRIC_SYNC_SUCCESS: &str = "sync_successful"; +pub const METRIC_SYNC_FAILURE: &str = "sync_failure"; + +// Stability metrics +pub const METRIC_OPERATIONAL_HOURS: &str = "operational_hours"; +pub const METRIC_OPERATIONAL_CRASHFREE_HOURS: &str = "operational_crashfree_hours"; +pub const METRIC_OPERATIONAL_CRASHES: &str = "operational_crashes"; + +// Battery metrics +pub const METRIC_BATTERY_DISCHARGE_DURATION_MS: &str = "battery_discharge_duration_ms"; +pub const METRIC_BATTERY_SOC_PCT_DROP: &str = "battery_soc_pct_drop"; +pub const METRIC_BATTERY_SOC_PCT: &str = "battery_soc_pct"; diff --git a/memfaultd/src/metrics/crashfree_interval.rs b/memfaultd/src/metrics/crashfree_interval.rs new file mode 100644 index 0000000..a574b25 --- /dev/null +++ b/memfaultd/src/metrics/crashfree_interval.rs @@ -0,0 +1,413 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + cmp::max, + iter::once, + sync::mpsc::{channel, Receiver, Sender}, + time::Duration, +}; + +use chrono::Utc; +use tiny_http::{Method, Request, Response}; + +use crate::{ + http_server::{HttpHandler, HttpHandlerResult}, + metrics::{ + core_metrics::{ + METRIC_OPERATIONAL_CRASHES, METRIC_OPERATIONAL_CRASHFREE_HOURS, + METRIC_OPERATIONAL_HOURS, + }, + MetricReading, + }, + util::time_measure::TimeMeasure, +}; + +use super::{KeyedMetricReading, MetricStringKey}; + +pub struct CrashFreeIntervalTracker { + last_interval_mark: T, + last_crashfree_interval_mark: T, + crash_count: u32, + sender: Sender, + receiver: Receiver, + interval: Duration, + elapsed_intervals_key: MetricStringKey, + crashfree_intervals_key: MetricStringKey, + crash_count_key: MetricStringKey, +} + +#[derive(Debug, PartialEq, Eq)] +struct TimeMod { + count: u32, + mark: T, +} + +impl CrashFreeIntervalTracker +where + T: TimeMeasure + Copy + Ord + std::ops::Add + Send + Sync + 'static, +{ + pub fn new( + interval: Duration, + elapsed_intervals_key: MetricStringKey, + crashfree_intervals_key: MetricStringKey, + crash_count_key: MetricStringKey, + ) -> Self { + let (sender, receiver) = channel(); + Self { + last_crashfree_interval_mark: T::now(), + last_interval_mark: T::now(), + sender, + receiver, + crash_count: 0, + interval, + elapsed_intervals_key, + crashfree_intervals_key, + crash_count_key, + } + } + + /// Returns a tracker with an hourly interval + pub fn new_hourly() -> Self { + Self::new( + Duration::from_secs(3600), + METRIC_OPERATIONAL_HOURS.parse().unwrap(), + METRIC_OPERATIONAL_CRASHFREE_HOURS.parse().unwrap(), + METRIC_OPERATIONAL_CRASHES.parse().unwrap(), + ) + } + + /// Wait for the next crash or update the metrics if the wait duration has passed. + /// + /// This allows us to have instant updates on crashes and hourly updates on the metrics, but + /// also allows us to periodically update the metrics so that we don't have to wait for a crash. + pub fn wait_and_update(&mut self, wait_duration: Duration) -> Vec { + if let Ok(crash_ts) = self.receiver.recv_timeout(wait_duration) { + // Drain the receiver to get all crashes that happened since the last update + self.receiver + .try_iter() + .chain(once(crash_ts)) + .for_each(|ts| { + self.crash_count += 1; + self.last_crashfree_interval_mark = max(self.last_crashfree_interval_mark, ts); + }); + } + + // Since timing out just means no crashes occurred in the `wait_duration`, + // update even when the receiver times out. + self.update() + } + + fn update(&mut self) -> Vec { + let TimeMod { + count: count_op_interval, + mark: last_counted_op_interval, + } = Self::full_interval_elapsed_since(self.interval, &self.last_interval_mark); + let TimeMod { + count: count_crashfree_interval, + mark: last_counted_crashfree_interval, + } = Self::full_interval_elapsed_since(self.interval, &self.last_crashfree_interval_mark); + + self.last_interval_mark = last_counted_op_interval; + self.last_crashfree_interval_mark = last_counted_crashfree_interval; + + let crashes = self.crash_count; + self.crash_count = 0; + + let metrics_ts = Utc::now(); + vec![ + KeyedMetricReading::new( + self.elapsed_intervals_key.clone(), + MetricReading::Counter { + value: count_op_interval as f64, + timestamp: metrics_ts, + }, + ), + KeyedMetricReading::new( + self.crashfree_intervals_key.clone(), + MetricReading::Counter { + value: count_crashfree_interval as f64, + timestamp: metrics_ts, + }, + ), + KeyedMetricReading::new( + self.crash_count_key.clone(), + MetricReading::Counter { + value: crashes as f64, + timestamp: metrics_ts, + }, + ), + ] + } + + pub fn http_handler(&mut self) -> Box { + Box::new(CrashFreeIntervalHttpHandler { + channel: self.sender.clone(), + }) + } + + pub fn capture_crash(&self) { + self.sender + .send(T::now()) + .expect("Failed to send crash timestamp"); + } + + /// Count how many `interval` have elapsed since `since`. + /// + /// This returns the number of intervals that have elapsed since `since`, and the timestamp of the end of the last interval + /// that was counted. This is the value you should pass as `since` next time you call this function. + /// + /// See unit test for examples. + fn full_interval_elapsed_since(interval: Duration, since: &T) -> TimeMod { + let now = T::now(); + if *since > now { + return TimeMod { + count: 0, + mark: T::now(), + }; + } + + let duration = now.since(since); + let count_interval_elapsed = (duration.as_nanos() / interval.as_nanos()) as u32; + TimeMod { + count: count_interval_elapsed, + mark: since.add(interval * count_interval_elapsed), + } + } +} + +struct CrashFreeIntervalHttpHandler { + channel: Sender, +} + +impl HttpHandler for CrashFreeIntervalHttpHandler +where + T: TimeMeasure + Copy + Ord + std::ops::Add + Send + Sync, +{ + fn handle_request(&self, request: &mut Request) -> HttpHandlerResult { + if request.url() == "/v1/crash/report" && request.method() == &Method::Post { + self.channel.send(T::now()).unwrap(); + HttpHandlerResult::Response(Response::from_string("OK").boxed()) + } else { + HttpHandlerResult::NotHandled + } + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use rstest::rstest; + + use crate::{ + metrics::{ + crashfree_interval::{ + METRIC_OPERATIONAL_CRASHES, METRIC_OPERATIONAL_CRASHFREE_HOURS, + METRIC_OPERATIONAL_HOURS, + }, + KeyedMetricReading, MetricReading, + }, + test_utils::TestInstant, + }; + + use super::CrashFreeIntervalTracker; + use super::TimeMod; + + #[rstest] + fn test_counting_intervals() { + use std::time::Duration; + + // move the clock forward so we can go backwards below + TestInstant::sleep(Duration::from_secs(3600)); + let now = TestInstant::now(); + + let d10 = Duration::from_secs(10); + assert_eq!( + CrashFreeIntervalTracker::full_interval_elapsed_since(d10, &now), + TimeMod { + count: 0, + mark: now + } + ); + assert_eq!( + CrashFreeIntervalTracker::full_interval_elapsed_since( + d10, + &(now - Duration::from_secs(10)) + ), + TimeMod { + count: 1, + mark: now + } + ); + assert_eq!( + CrashFreeIntervalTracker::full_interval_elapsed_since( + d10, + &(now - Duration::from_secs(25)) + ), + TimeMod { + count: 2, + mark: now - Duration::from_secs(5) + } + ); + } + + #[rstest] + fn test_counting_hours() { + let mut crashfree_tracker = CrashFreeIntervalTracker::::new_hourly(); + + TestInstant::sleep(Duration::from_secs(7200)); + + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 2, + 2, + ); + } + + #[rstest] + fn test_counting_minutes() { + let mut crashfree_tracker = CrashFreeIntervalTracker::::new( + Duration::from_secs(60), + METRIC_OPERATIONAL_HOURS.parse().unwrap(), + METRIC_OPERATIONAL_CRASHFREE_HOURS.parse().unwrap(), + METRIC_OPERATIONAL_CRASHES.parse().unwrap(), + ); + + TestInstant::sleep(Duration::from_secs(3600)); + crashfree_tracker.capture_crash(); + TestInstant::sleep(Duration::from_secs(3600)); + + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 120, + 60, + ); + } + + #[rstest] + fn test_30min_heartbeat() { + let mut crashfree_tracker = CrashFreeIntervalTracker::::new_hourly(); + + TestInstant::sleep(Duration::from_secs(1800)); + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 0, + 0, + ); + + TestInstant::sleep(Duration::from_secs(1800)); + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 1, + 1, + ); + } + + #[rstest] + fn test_30min_heartbeat_with_crash() { + let mut crashfree_tracker = CrashFreeIntervalTracker::::new_hourly(); + + TestInstant::sleep(Duration::from_secs(1800)); + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 0, + 0, + ); + + // Crash at t0 + 30min + crashfree_tracker.capture_crash(); + + // After 30' we should be ready to mark an operational hour + TestInstant::sleep(Duration::from_secs(1800)); + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 1, + 0, + ); + + // After another 30' we should be ready to mark another crashfree hour + TestInstant::sleep(Duration::from_secs(1800)); + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 0, + 1, + ); + + // After another 30' we should be ready to mark another operational hour + TestInstant::sleep(Duration::from_secs(1800)); + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 1, + 0, + ); + } + + #[rstest] + fn test_180min_heartbeat_with_one_crash() { + let mut crashfree_tracker = CrashFreeIntervalTracker::::new_hourly(); + + // Basic test + TestInstant::sleep(Duration::from_secs(3600 * 3)); + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 3, + 3, + ); + + // Crash at interval + 170' + TestInstant::sleep(Duration::from_secs(170 * 60)); + crashfree_tracker.capture_crash(); + + // Another 10' to the heartbeat mark + // We will count 0 operational hour here. That is a consequence of the heartbeat being larger than the hour + // To avoid this bug, we need to make sure we call the `update` at least once per hour! + TestInstant::sleep(Duration::from_secs(10 * 60)); + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 3, + 0, + ); + + // However, doing the crash at interval +10' then waiting for 170' will record 2 crashfree hours + TestInstant::sleep(Duration::from_secs(10 * 60)); + crashfree_tracker.capture_crash(); + TestInstant::sleep(Duration::from_secs(170 * 60)); + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 3, + 2, + ); + } + + fn assert_operational_metrics( + metrics: Vec, + expected_op_hours: u32, + expected_crashfree_hours: u32, + ) { + assert_eq!(metrics.len(), 3); + let op_hours = metrics + .iter() + .find(|m| m.name.as_str() == METRIC_OPERATIONAL_HOURS) + .unwrap(); + let crash_free_hours = metrics + .iter() + .find(|m| m.name.as_str() == METRIC_OPERATIONAL_CRASHFREE_HOURS) + .unwrap(); + + let op_hours_value = match op_hours.value { + MetricReading::Counter { value, .. } => value, + _ => panic!("Unexpected metric type"), + }; + + let crashfree_hours_value = match crash_free_hours.value { + MetricReading::Counter { value, .. } => value, + _ => panic!("Unexpected metric type"), + }; + + assert_eq!( + (op_hours_value as u32, crashfree_hours_value as u32), + (expected_op_hours, expected_crashfree_hours) + ); + } +} diff --git a/memfaultd/src/metrics/metric_reading.rs b/memfaultd/src/metrics/metric_reading.rs new file mode 100644 index 0000000..df0f51d --- /dev/null +++ b/memfaultd/src/metrics/metric_reading.rs @@ -0,0 +1,255 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::str::FromStr; + +use chrono::{Duration, Utc}; +use eyre::{eyre, ErrReport}; +use nom::{ + branch::alt, + character::complete::char, + combinator::value, + number::complete::double, + sequence::separated_pair, + Finish, + {bytes::complete::tag, IResult}, +}; +use serde::{Deserialize, Serialize}; + +use super::{MetricStringKey, MetricTimestamp}; + +use crate::util::serialization::float_to_duration; + +/// A typed value and timestamp pair that represents +/// an individual reading value for a metric. This type does +/// not have a notion of which key it is associated it +/// and is purely the "value" in a metric reading. +/// For the full metric reading type that includes the key, +/// use KeyedMetricReading. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum MetricReading { + /// TimeWeightedAverage readings will be aggregated based on + /// the time the reading was captured over. + TimeWeightedAverage { + value: f64, + timestamp: MetricTimestamp, + /// Time period considered for this reading. This is only used to give a "time-weight" to the first + /// value in the series. For future values we will use the time + /// difference we measure between the two points + /// In doubt, it's safe to use Duration::from_secs(0) here. This means the first value will be ignored. + #[serde(with = "float_to_duration")] + interval: Duration, + }, + /// A non-decreasing monotonic sum. Within a metric report, Counter readings are summed together. + Counter { + value: f64, + timestamp: MetricTimestamp, + }, + /// Gauges are absolute values. We keep the latest value collected during a metric report. + Gauge { + value: f64, + timestamp: MetricTimestamp, + }, + /// Histogram readings are averaged together by dividing the sum of the values + /// by the number of readings over the duration of a metric report. + Histogram { + value: f64, + timestamp: MetricTimestamp, + }, + /// ReportTags are string values associated with the MetricReport they are captured in. + /// We keep the latest value collected during a metric report for a key and drop the older + /// ones. + ReportTag { + value: String, + timestamp: MetricTimestamp, + }, +} + +impl MetricReading { + /// Parse a metric reading in the format f64|. + /// The timestamp will be set to Utc::now(). + /// This is the suffix of a full StatsD reading of the following format: + /// : + /// + /// Examples of valid readings: + /// 64|h + /// 100.0|c + /// -89.5|g + fn parse(input: &str) -> IResult<&str, MetricReading> { + let (remaining, (value, statsd_type)) = + separated_pair(double, tag("|"), StatsDMetricType::parse)(input)?; + let timestamp = Utc::now(); + match statsd_type { + StatsDMetricType::Histogram => { + Ok((remaining, MetricReading::Histogram { value, timestamp })) + } + StatsDMetricType::Counter => { + Ok((remaining, MetricReading::Counter { value, timestamp })) + } + StatsDMetricType::Timer => Ok((remaining, MetricReading::Counter { value, timestamp })), + StatsDMetricType::Gauge => Ok((remaining, MetricReading::Gauge { value, timestamp })), + } + } +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct KeyedMetricReading { + pub name: MetricStringKey, + pub value: MetricReading, +} + +impl KeyedMetricReading { + pub fn new(name: MetricStringKey, value: MetricReading) -> Self { + Self { name, value } + } + + pub fn new_gauge(name: MetricStringKey, value: f64) -> Self { + Self { + name, + value: MetricReading::Gauge { + value, + timestamp: Utc::now(), + }, + } + } + + pub fn new_histogram(name: MetricStringKey, value: f64) -> Self { + Self { + name, + value: MetricReading::Histogram { + value, + timestamp: Utc::now(), + }, + } + } + + /// Construct a KeyedMetricReading from a string in the StatsD format + /// | + /// + /// Examples of valid keyed metric readings: + /// testCounter:1|c + /// test_counter:1.0|c + /// test_histo:100|h + /// test_gauge:1.7|g + /// cpu3_idle:100.9898|g + pub fn from_statsd_str(s: &str) -> Result { + match Self::parse_statsd(s).finish() { + Ok((_, reading)) => Ok(reading), + Err(e) => Err(eyre!( + "Failed to parse string \"{}\" as a StatsD metric reading: {}", + s, + e + )), + } + } + + /// Helper that handles `nom` details for parsing a StatsD string as + /// a KeyedMetricReading + fn parse_statsd(input: &str) -> IResult<&str, KeyedMetricReading> { + let (remaining, (name, value)) = + separated_pair(MetricStringKey::parse, tag(":"), MetricReading::parse)(input)?; + Ok((remaining, KeyedMetricReading { name, value })) + } + + /// Deserialize a string in the form = to a Gauge metric reading + /// + /// Currently deserialization to a KeyedMetricReading with any other type of MetricReading + /// as its value is not supported + fn from_arg_str(s: &str) -> Result { + let (key, value_str) = s.split_once('=').ok_or(eyre!( + "Attached metric reading should be specified as KEY=VALUE" + ))?; + + // Let's ensure the key is valid first: + let metric_key = MetricStringKey::from_str(key).map_err(|e| eyre!(e))?; + if let Ok(value) = f64::from_str(value_str) { + let reading = MetricReading::Gauge { + value, + timestamp: Utc::now(), + }; + Ok(KeyedMetricReading::new(metric_key, reading)) + } else { + let reading = MetricReading::ReportTag { + value: value_str.to_string(), + timestamp: Utc::now(), + }; + Ok(KeyedMetricReading::new(metric_key, reading)) + } + } +} + +impl FromStr for KeyedMetricReading { + type Err = ErrReport; + + fn from_str(s: &str) -> Result { + match Self::from_statsd_str(s) { + Ok(reading) => Ok(reading), + Err(_) => match Self::from_arg_str(s) { + Ok(reading) => Ok(reading), + Err(e) => Err(eyre!("Couldn't parse \"{}\" as a Gauge metric: {}", s, e)), + }, + } + } +} + +#[derive(Debug, Clone)] +enum StatsDMetricType { + Counter, + Histogram, + Gauge, + Timer, +} + +impl StatsDMetricType { + /// Parse a StatsDMetricType, which must be one of 'c', 'g', or 'h' + /// + /// 'c' indicates a Counter reading + /// 'g' indicates a Gauge reading + /// 'h' indicates a Histogram reading + fn parse(input: &str) -> IResult<&str, StatsDMetricType> { + alt(( + value(StatsDMetricType::Counter, char('c')), + value(StatsDMetricType::Histogram, char('h')), + value(StatsDMetricType::Gauge, char('g')), + value(StatsDMetricType::Timer, tag("ms")), + ))(input) + } +} + +#[cfg(test)] +mod tests { + + use super::*; + use crate::test_utils::setup_logger; + use rstest::rstest; + + #[rstest] + #[case("testCounter=1")] + #[case("hello=world")] + #[case("float=100.0")] + fn parse_valid_arg_reading(#[case] reading_str: &str, _setup_logger: ()) { + assert!(KeyedMetricReading::from_str(reading_str).is_ok()) + } + + #[rstest] + #[case("testCounter:1|c")] + #[case("test_counter:1.0|c")] + #[case("test_histo:100|h")] + #[case("test_gauge:1.7|g")] + #[case("cpu3_idle:100.9898|g")] + #[case("some_negative_gauge:-87.55|g")] + #[case("test_timer:3600000|ms")] + fn parse_valid_statsd_reading(#[case] reading_str: &str, _setup_logger: ()) { + assert!(KeyedMetricReading::from_str(reading_str).is_ok()) + } + + #[rstest] + #[case("test Counter:1|c")] + #[case("{test_counter:1.0|c}")] + #[case("\"test_counter\":1.0|c}")] + #[case("test_gauge:\"string-value\"|g")] + #[case("test_gauge:string-value|g")] + fn fail_on_invalid_statsd_reading(#[case] reading_str: &str, _setup_logger: ()) { + assert!(KeyedMetricReading::from_str(reading_str).is_err()) + } +} diff --git a/memfaultd/src/metrics/metric_report.rs b/memfaultd/src/metrics/metric_report.rs new file mode 100644 index 0000000..86e8579 --- /dev/null +++ b/memfaultd/src/metrics/metric_report.rs @@ -0,0 +1,337 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use chrono::Utc; +use eyre::{eyre, Result}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{HashMap, HashSet}, + path::Path, + str::FromStr, + time::{Duration, Instant}, +}; + +use crate::{ + mar::{MarEntryBuilder, Metadata}, + metrics::{ + core_metrics::{ + METRIC_BATTERY_DISCHARGE_DURATION_MS, METRIC_BATTERY_SOC_PCT_DROP, + METRIC_CONNECTED_TIME, METRIC_EXPECTED_CONNECTED_TIME, METRIC_MF_SYNC_FAILURE, + METRIC_MF_SYNC_SUCCESS, METRIC_OPERATIONAL_CRASHES, METRIC_SYNC_FAILURE, + METRIC_SYNC_SUCCESS, + }, + metric_reading::KeyedMetricReading, + timeseries::{Counter, Gauge, Histogram, TimeSeries, TimeWeightedAverage}, + MetricReading, MetricStringKey, MetricValue, SessionName, + }, +}; + +use super::timeseries::ReportTag; + +pub enum CapturedMetrics { + All, + Metrics(HashSet), +} + +pub const HEARTBEAT_REPORT_TYPE: &str = "heartbeat"; +pub const DAILY_HEARTBEAT_REPORT_TYPE: &str = "daily-heartbeat"; + +const SESSION_CORE_METRICS: &[&str; 9] = &[ + METRIC_MF_SYNC_FAILURE, + METRIC_MF_SYNC_SUCCESS, + METRIC_BATTERY_DISCHARGE_DURATION_MS, + METRIC_BATTERY_SOC_PCT_DROP, + METRIC_CONNECTED_TIME, + METRIC_EXPECTED_CONNECTED_TIME, + METRIC_SYNC_FAILURE, + METRIC_SYNC_SUCCESS, + METRIC_OPERATIONAL_CRASHES, +]; + +#[derive(Serialize, Deserialize, Clone, PartialEq, Eq)] +pub enum MetricReportType { + #[serde(rename = "heartbeat")] + Heartbeat, + #[serde(rename = "session")] + Session(SessionName), + #[serde(rename = "daily-heartbeat")] + DailyHeartbeat, +} + +impl MetricReportType { + pub fn as_str(&self) -> &str { + match self { + MetricReportType::Heartbeat => HEARTBEAT_REPORT_TYPE, + MetricReportType::Session(session_name) => session_name.as_str(), + MetricReportType::DailyHeartbeat => DAILY_HEARTBEAT_REPORT_TYPE, + } + } +} + +pub struct MetricReport { + /// In-memory metric store for this report + metrics: HashMap>, + /// Point in time when capture of metrics currently in + /// report's metric store began + start: Instant, + /// Configuration of which metrics this report should capture + captured_metrics: CapturedMetrics, + /// Indicates whether this is a heartbeat metric report or + /// session metric report (with session name) + report_type: MetricReportType, +} + +struct MetricReportSnapshot { + duration: Duration, + metrics: HashMap, +} + +impl MetricReport { + pub fn new( + report_type: MetricReportType, + configured_captured_metrics: CapturedMetrics, + ) -> Self { + // Always include session core metrics regardless of configuration + let captured_metrics = match configured_captured_metrics { + CapturedMetrics::All => CapturedMetrics::All, + CapturedMetrics::Metrics(metrics) => { + let mut merged_set = SESSION_CORE_METRICS + .iter() + .map(|core_metric_key| { + // This expect should never be hit as SESSION_CORE_METRICS is + // an array of static strings + MetricStringKey::from_str(core_metric_key) + .expect("Invalid Metric Key in SESSION_CORE_METRICS") + }) + .collect::>(); + merged_set.extend(metrics); + CapturedMetrics::Metrics(merged_set) + } + }; + + Self { + metrics: HashMap::new(), + start: Instant::now(), + captured_metrics, + report_type, + } + } + + /// Creates a heartbeat report that captures all metrics + pub fn new_heartbeat() -> Self { + MetricReport::new(MetricReportType::Heartbeat, CapturedMetrics::All) + } + + /// Creates a daily heartbeat report that captures all metrics + pub fn new_daily_heartbeat() -> Self { + MetricReport::new(MetricReportType::DailyHeartbeat, CapturedMetrics::All) + } + + fn is_captured(&self, metric_key: &MetricStringKey) -> bool { + match &self.captured_metrics { + CapturedMetrics::Metrics(metric_keys) => metric_keys.contains(metric_key), + CapturedMetrics::All => true, + } + } + + /// Adds a metric reading to the report's internal + /// metric store if the report captures that metric, + /// otherwise no-op + pub fn add_metric(&mut self, m: KeyedMetricReading) -> Result<()> { + if self.is_captured(&m.name) { + match self.metrics.entry(m.name) { + std::collections::hash_map::Entry::Occupied(mut o) => { + let state = o.get_mut(); + if let Err(e) = (*state).aggregate(&m.value) { + *state = Self::select_aggregate_for(&m.value)?; + log::warn!( + "New value for metric {} is incompatible ({}). Resetting timeseries.", + o.key(), + e + ); + } + } + std::collections::hash_map::Entry::Vacant(v) => { + let timeseries = Self::select_aggregate_for(&m.value)?; + v.insert(timeseries); + } + }; + } + Ok(()) + } + + /// Increment a counter metric by 1 + pub fn increment_counter(&mut self, name: &str) -> Result<()> { + self.add_to_counter(name, 1.0) + } + + pub fn add_to_counter(&mut self, name: &str, value: f64) -> Result<()> { + match name.parse::() { + Ok(metric_name) => self.add_metric(KeyedMetricReading::new( + metric_name, + MetricReading::Counter { + value, + timestamp: Utc::now(), + }, + )), + Err(e) => Err(eyre!("Invalid metric name: {} - {}", name, e)), + } + } + + /// Return all the metrics in memory for this report and resets its store. + pub fn take_metrics(&mut self) -> HashMap { + self.take_metric_report_snapshot().metrics + } + + fn take_metric_report_snapshot(&mut self) -> MetricReportSnapshot { + let duration = std::mem::replace(&mut self.start, Instant::now()).elapsed(); + let metrics = std::mem::take(&mut self.metrics) + .into_iter() + .map(|(name, state)| (name, state.value())) + .collect(); + + MetricReportSnapshot { duration, metrics } + } + + /// Create one metric report MAR entry with all the metrics in the store. + /// + /// All data will be timestamped with current time measured by CollectionTime::now(), effectively + /// disregarding the collectd timestamps. + pub fn prepare_metric_report( + &mut self, + mar_staging_area: &Path, + ) -> Result>> { + let snapshot = self.take_metric_report_snapshot(); + + if snapshot.metrics.is_empty() { + return Ok(None); + } + + Ok(Some(MarEntryBuilder::new(mar_staging_area)?.set_metadata( + Metadata::new_metric_report( + snapshot.metrics, + snapshot.duration, + self.report_type.clone(), + ), + ))) + } + + fn select_aggregate_for(event: &MetricReading) -> Result> { + match event { + MetricReading::Histogram { .. } => Ok(Box::new(Histogram::new(event)?)), + MetricReading::Counter { .. } => Ok(Box::new(Counter::new(event)?)), + MetricReading::Gauge { .. } => Ok(Box::new(Gauge::new(event)?)), + MetricReading::TimeWeightedAverage { .. } => { + Ok(Box::new(TimeWeightedAverage::new(event)?)) + } + MetricReading::ReportTag { .. } => Ok(Box::new(ReportTag::new(event)?)), + } + } + + pub fn report_type(&self) -> &MetricReportType { + &self.report_type + } +} + +#[cfg(test)] +mod tests { + use tempfile::TempDir; + + use std::collections::BTreeMap; + + use super::*; + use crate::test_utils::in_histograms; + use std::str::FromStr; + + use insta::assert_json_snapshot; + use rstest::rstest; + + #[rstest] + #[case(in_histograms(vec![("foo", 1.0), ("bar", 2.0), ("baz", 3.0)]), "heartbeat_report_1")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 2.0), ("foo", 3.0)]), "heartbeat_report_2")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 1.0)]), "heartbeat_report_3")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 2.0)]), "heartbeat_report_4")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 2.0), ("foo", 2.0)]), "heartbeat_report_5")] + fn test_aggregate_metrics( + #[case] metrics: impl Iterator, + #[case] test_name: &str, + ) { + let mut metric_report = MetricReport::new_heartbeat(); + + for m in metrics { + metric_report.add_metric(m).unwrap(); + } + let sorted_metrics: BTreeMap<_, _> = metric_report.take_metrics().into_iter().collect(); + assert_json_snapshot!(test_name, sorted_metrics); + } + + #[rstest] + #[case(in_histograms(vec![("foo", 1.0), ("bar", 2.0), ("baz", 3.0)]), "session_report_1")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 2.0), ("foo", 3.0)]), "session_report_2")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 1.0)]), "session_report_3")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 2.0)]), "session_report_4")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 2.0), ("baz", 2.0), ("bar", 3.0)]), "session_report_5")] + fn test_aggregate_metrics_session( + #[case] metrics: impl Iterator, + #[case] test_name: &str, + ) { + let mut metric_report = MetricReport::new( + MetricReportType::Session(SessionName::from_str("foo_only").unwrap()), + CapturedMetrics::Metrics( + [ + MetricStringKey::from_str("foo").unwrap(), + MetricStringKey::from_str("baz").unwrap(), + ] + .into_iter() + .collect(), + ), + ); + + for m in metrics { + metric_report.add_metric(m).unwrap(); + } + let sorted_metrics: BTreeMap<_, _> = metric_report.take_metrics().into_iter().collect(); + assert_json_snapshot!(test_name, sorted_metrics); + } + + /// Core metrics should always be captured by sessions even if they are not + /// configured to do so + #[rstest] + fn test_aggregate_core_metrics_session() { + let mut metric_report = MetricReport::new( + MetricReportType::Session(SessionName::from_str("foo_only").unwrap()), + CapturedMetrics::Metrics( + [ + MetricStringKey::from_str("foo").unwrap(), + MetricStringKey::from_str("baz").unwrap(), + ] + .into_iter() + .collect(), + ), + ); + + let metrics = in_histograms( + SESSION_CORE_METRICS + .map(|metric_name: &'static str| (metric_name, 100.0)) + .to_vec(), + ); + + for m in metrics { + metric_report.add_metric(m).unwrap(); + } + let sorted_metrics: BTreeMap<_, _> = metric_report.take_metrics().into_iter().collect(); + assert_json_snapshot!(sorted_metrics); + } + + #[rstest] + fn test_empty_after_write() { + let mut metric_report = MetricReport::new_heartbeat(); + for m in in_histograms(vec![("foo", 1.0), ("bar", 2.0), ("baz", 3.0)]) { + metric_report.add_metric(m).unwrap(); + } + + let tempdir = TempDir::new().unwrap(); + let _ = metric_report.prepare_metric_report(tempdir.path()); + assert_eq!(metric_report.take_metrics().len(), 0); + } +} diff --git a/memfaultd/src/metrics/metric_report_manager.rs b/memfaultd/src/metrics/metric_report_manager.rs new file mode 100644 index 0000000..79e07c9 --- /dev/null +++ b/memfaultd/src/metrics/metric_report_manager.rs @@ -0,0 +1,464 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::{eyre, Result}; +use log::{debug, error}; +use std::{ + collections::{hash_map::Entry, HashMap}, + path::Path, + sync::{Arc, Mutex}, +}; + +use super::{ + core_metrics::METRIC_OPERATIONAL_CRASHES, metric_reading::KeyedMetricReading, + metric_report::CapturedMetrics, SessionName, +}; +use crate::{ + config::SessionConfig, + mar::{MarEntryBuilder, Metadata}, + metrics::{MetricReport, MetricReportType, MetricStringKey, MetricValue}, + network::NetworkConfig, +}; + +pub struct MetricReportManager { + heartbeat: MetricReport, + daily_heartbeat: MetricReport, + sessions: HashMap, + session_configs: Vec, +} + +impl MetricReportManager { + /// Creates a MetricReportManager with no sessions + /// configured + pub fn new() -> Self { + Self { + heartbeat: MetricReport::new_heartbeat(), + daily_heartbeat: MetricReport::new_daily_heartbeat(), + sessions: HashMap::new(), + session_configs: vec![], + } + } + + pub fn new_with_session_configs(session_configs: &[SessionConfig]) -> Self { + Self { + heartbeat: MetricReport::new_heartbeat(), + daily_heartbeat: MetricReport::new_daily_heartbeat(), + sessions: HashMap::new(), + session_configs: session_configs.to_vec(), + } + } + + /// Starts a session of the specified session name. + /// Fails if the session name provided is not configured. + /// If there is already a session with that name ongoing, + /// this is a no-op + pub fn start_session(&mut self, session_name: SessionName) -> Result<()> { + let report_type = MetricReportType::Session(session_name.clone()); + let captured_metric_keys = self.captured_metric_keys_for_report(&report_type)?; + + if let Entry::Vacant(e) = self.sessions.entry(session_name) { + let session = e.insert(MetricReport::new(report_type, captured_metric_keys)); + // Make sure we always include the operational_crashes counter in every session report. + session.add_to_counter(METRIC_OPERATIONAL_CRASHES, 0.0)?; + } + Ok(()) + } + + /// Returns the metrics the provided session name is configured to capture + fn captured_metric_keys_for_report( + &self, + report_type: &MetricReportType, + ) -> Result { + match report_type { + MetricReportType::Heartbeat => Ok(CapturedMetrics::All), + MetricReportType::DailyHeartbeat => Ok(CapturedMetrics::All), + MetricReportType::Session(session_name) => self + .session_configs + .iter() + .find(|&session_config| session_config.name == *session_name) + .map(|config| { + CapturedMetrics::Metrics(config.captured_metrics.clone().into_iter().collect()) + }) + .ok_or_else(|| eyre!("No configuration for session named {} found!", session_name)), + } + } + + /// Returns an iterator over all ongoing metric reports + fn report_iter(&mut self) -> impl Iterator { + self.sessions + .values_mut() + .chain([&mut self.heartbeat, &mut self.daily_heartbeat]) + } + + /// Adds a metric reading to all ongoing metric reports + /// that capture that metric + pub fn add_metric(&mut self, m: KeyedMetricReading) -> Result<()> { + self.report_iter() + .try_for_each(|report| report.add_metric(m.clone())) + } + + /// Increment a counter metric by 1 + pub fn increment_counter(&mut self, name: &str) -> Result<()> { + self.report_iter() + .try_for_each(|report| report.increment_counter(name)) + } + + /// Increment a counter by a specified amount + pub fn add_to_counter(&mut self, name: &str, value: f64) -> Result<()> { + self.report_iter() + .try_for_each(|report| report.add_to_counter(name, value)) + } + + /// Adds a metric reading to a specific metric report + pub fn add_metric_to_report( + &mut self, + report_type: &MetricReportType, + m: KeyedMetricReading, + ) -> Result<()> { + match report_type { + MetricReportType::Heartbeat => self.heartbeat.add_metric(m), + MetricReportType::DailyHeartbeat => self.daily_heartbeat.add_metric(m), + MetricReportType::Session(session_name) => self + .sessions + .get_mut(session_name) + .ok_or_else(|| eyre!("No ongoing session with name {}", session_name)) + .and_then(|session_report| session_report.add_metric(m)), + } + } + + /// Return all the metrics in memory and resets the + /// store for the periodic heartbeat report. + pub fn take_heartbeat_metrics(&mut self) -> HashMap { + self.heartbeat.take_metrics() + } + + /// Return all the metrics in memory and resets the store + /// for a specified session. + pub fn take_session_metrics( + &mut self, + session_name: &SessionName, + ) -> Result> { + self.sessions + .get_mut(session_name) + .ok_or_else(|| eyre!("No ongoing session with name {}", session_name)) + .map(|session_report| session_report.take_metrics()) + } + + /// Dump the metrics to a MAR entry. + /// + /// This takes a &Arc> and will minimize lock time. + /// This will empty the metrics store. + /// When used with a heartbeat metric report type, the heartbeat + /// will be reset. + /// When used with a session report type, the session will end and + /// be removed from the MetricReportManager's internal sessions HashMap. + pub fn dump_report_to_mar_entry( + metric_report_manager: &Arc>, + mar_staging_area: &Path, + network_config: &NetworkConfig, + report_type: &MetricReportType, + ) -> Result<()> { + let mar_builder = match report_type { + MetricReportType::Heartbeat => metric_report_manager + .lock() + .expect("Mutex Poisoned!") + .heartbeat + .prepare_metric_report(mar_staging_area)?, + MetricReportType::DailyHeartbeat => metric_report_manager + .lock() + .expect("Mutex Poisoned!") + .daily_heartbeat + .prepare_metric_report(mar_staging_area)?, + MetricReportType::Session(session_name) => { + match metric_report_manager + .lock() + .expect("Mutex Poisoned!") + .sessions + .remove(session_name) + { + Some(mut report) => report.prepare_metric_report(mar_staging_area)?, + None => return Err(eyre!("No metric report found for {}", session_name)), + } + } + }; + + // Save to disk after releasing the lock + if let Some(mar_builder) = mar_builder { + let mar_entry = mar_builder + .save(network_config) + .map_err(|e| eyre!("Error building MAR entry: {}", e))?; + debug!( + "Generated MAR entry from metrics: {}", + mar_entry.path.display() + ); + } else { + debug!( + "Skipping generating metrics entry. No metrics in store for: {}", + report_type.as_str() + ) + } + Ok(()) + } + + fn prepare_all_metric_reports( + &mut self, + mar_staging_area: &Path, + ) -> Vec> { + self.report_iter() + .filter_map(|report| { + if let Ok(builder) = report.prepare_metric_report(mar_staging_area) { + builder.or_else(|| { + debug!( + "Skipping generating metrics entry. No metrics in store for: {}", + report.report_type().as_str() + ); + None + }) + } else { + debug!( + "Failed to prepare metric report for: {}", + report.report_type().as_str() + ); + None + } + }) + .collect() + } + + /// Ends all ongoing MetricReports and dumps them as MARs to disk. + /// + /// MetricReports with the MetricReportType specified with + /// exclude_report_types are excluded from this operation + /// entirely + pub fn dump_metric_reports( + metric_report_manager: &Arc>, + mar_staging_area: &Path, + network_config: &NetworkConfig, + ) -> Result<()> { + let mar_builders = metric_report_manager + .lock() + .expect("Mutex poisoned") + .prepare_all_metric_reports(mar_staging_area); + + for mar_builder in mar_builders { + match mar_builder.save(network_config) { + Ok(mar_entry) => debug!( + "Generated MAR entry from metrics: {}", + mar_entry.path.display() + ), + Err(e) => error!("Error building MAR entry: {}", e), + } + } + + Ok(()) + } +} + +impl Default for MetricReportManager { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use tempfile::TempDir; + + use super::*; + use crate::test_utils::in_histograms; + use insta::assert_json_snapshot; + use rstest::rstest; + use std::str::FromStr; + + #[rstest] + #[case(in_histograms(vec![("foo", 1.0), ("bar", 2.0), ("baz", 3.0)]), "heartbeat_report_1")] + #[case(in_histograms(vec![("foo", 1.0), ("foo",2.0), ("foo", 3.0)]), "heartbeat_report_2")] + #[case(in_histograms(vec![("foo", 1.0), ("foo",1.0)]), "heartbeat_report_3")] + #[case(in_histograms(vec![("foo", 1.0), ("foo",2.0)]), "heartbeat_report_4")] + #[case(in_histograms(vec![("foo", 1.0), ("foo",2.0), ("foo", 2.0)]), "heartbeat_report_5")] + fn test_heartbeat_report( + #[case] metrics: impl Iterator, + #[case] test_name: &str, + ) { + let mut metric_report_manager = MetricReportManager::new(); + for m in metrics { + metric_report_manager + .add_metric(m) + .expect("Failed to add metric reading"); + } + + let tempdir = TempDir::new().unwrap(); + let builder = metric_report_manager + .heartbeat + .prepare_metric_report(tempdir.path()) + .unwrap(); + assert_json_snapshot!(test_name, builder.unwrap().get_metadata(), {".metadata.duration_ms" => 0}); + } + + #[rstest] + fn test_unconfigured_session_name_fails() { + let mut metric_report_manager = MetricReportManager::new(); + assert!(metric_report_manager + .start_session(SessionName::from_str("test-session").unwrap()) + .is_err()) + } + + #[rstest] + #[case(in_histograms(vec![("foo", 1.0), ("bar", 2.0), ("baz", 3.0)]), "heartbeat_and_sessions_report_1")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 2.0), ("foo", 3.0)]), "heartbeat_and_sessions_report_2")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 1.0)]), "heartbeat_and_sessions_report_3")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 2.0), ("baz", 1.0), ("baz", 2.0)]), "heartbeat_and_sessions_report_4")] + #[case(in_histograms(vec![("foo", 1.0), ("bar", 2.0), ("foo", 2.0)]), "heartbeat_and_sessions_report_5")] + fn test_heartbeat_and_session_reports( + #[case] metrics: impl Iterator, + #[case] test_name: &str, + ) { + let session_a_name = SessionName::from_str("test-session-some-metrics").unwrap(); + let session_b_name = SessionName::from_str("test-session-all-metrics").unwrap(); + let session_configs = vec![ + SessionConfig { + name: session_a_name.clone(), + captured_metrics: vec![ + MetricStringKey::from_str("foo").unwrap(), + MetricStringKey::from_str("bar").unwrap(), + ], + }, + SessionConfig { + name: session_b_name.clone(), + captured_metrics: vec![ + MetricStringKey::from_str("foo").unwrap(), + MetricStringKey::from_str("bar").unwrap(), + MetricStringKey::from_str("baz").unwrap(), + ], + }, + ]; + + let mut metric_report_manager = + MetricReportManager::new_with_session_configs(&session_configs); + + assert!(metric_report_manager.start_session(session_a_name).is_ok()); + assert!(metric_report_manager.start_session(session_b_name).is_ok()); + + for m in metrics { + metric_report_manager + .add_metric(m) + .expect("Failed to add metric reading"); + } + + let tempdir = TempDir::new().unwrap(); + // Verify heartbeat report + let snapshot_name = format!("{}.{}", test_name, "heartbeat"); + assert_report_snapshot( + &mut metric_report_manager.heartbeat, + &snapshot_name, + &tempdir, + ); + + // Verify daily heartbeat report + let snapshot_name = format!("{}.{}", test_name, "daily_heartbeat"); + assert_report_snapshot( + &mut metric_report_manager.daily_heartbeat, + &snapshot_name, + &tempdir, + ); + + for (session_name, mut metric_report) in metric_report_manager.sessions { + let snapshot_name = format!("{}.{}", test_name, session_name); + assert_report_snapshot(&mut metric_report, &snapshot_name, &tempdir); + } + } + + #[rstest] + fn test_start_session_twice() { + let session_name = SessionName::from_str("test-session-start-twice").unwrap(); + let session_configs = vec![SessionConfig { + name: session_name.clone(), + captured_metrics: vec![ + MetricStringKey::from_str("foo").unwrap(), + MetricStringKey::from_str("bar").unwrap(), + ], + }]; + + let mut metric_report_manager = + MetricReportManager::new_with_session_configs(&session_configs); + + let metrics_a = in_histograms(vec![("foo", 1.0), ("bar", 2.0)]); + assert!(metric_report_manager + .start_session(session_name.clone()) + .is_ok()); + for m in metrics_a { + metric_report_manager + .add_metric(m) + .expect("Failed to add metric reading"); + } + + // Final metric report should aggregate both metrics_a and + // metrics_b as the session should not be restarted + // by the second start_session + let metrics_b = in_histograms(vec![("foo", 9.0), ("bar", 5.0)]); + assert!(metric_report_manager + .start_session(session_name.clone()) + .is_ok()); + for m in metrics_b { + metric_report_manager + .add_metric(m) + .expect("Failed to add metric reading"); + } + + let tempdir = TempDir::new().unwrap(); + let builder = metric_report_manager + .sessions + .get_mut(&session_name) + .unwrap() + .prepare_metric_report(tempdir.path()) + .unwrap(); + + assert_json_snapshot!(builder.unwrap().get_metadata(), {".metadata.duration_ms" => 0}); + } + + #[rstest] + fn test_prepare_all_prepares_sessions() { + let session_name = SessionName::from_str("test-session").unwrap(); + let session_configs = vec![SessionConfig { + name: session_name.clone(), + captured_metrics: vec![ + MetricStringKey::from_str("foo").unwrap(), + MetricStringKey::from_str("bar").unwrap(), + ], + }]; + + let mut metric_report_manager = + MetricReportManager::new_with_session_configs(&session_configs); + + let metrics = in_histograms(vec![("foo", 5.0), ("bar", 3.5)]); + assert!(metric_report_manager.start_session(session_name).is_ok()); + for m in metrics { + metric_report_manager + .add_metric(m) + .expect("Failed to add metric reading"); + } + + let tempdir = TempDir::new().unwrap(); + let builders = metric_report_manager.prepare_all_metric_reports(tempdir.path()); + + // 3 MAR builders should be created for "heartbeat", "daily-heartbeat", and "test-session" + // Note this only works because report_iter() with only 1 session is deterministic + for builder in builders { + match builder.get_metadata() { + Metadata::LinuxMetricReport { report_type, .. } => { + assert_json_snapshot!(report_type.as_str(), builder.get_metadata(), {".metadata.duration_ms" => 0}) + } + _ => panic!("Invalid MAR builder"), + } + } + } + + fn assert_report_snapshot( + metric_report: &mut MetricReport, + snapshot_name: &str, + tempdir: &TempDir, + ) { + let builder = metric_report.prepare_metric_report(tempdir.path()).unwrap(); + assert_json_snapshot!(snapshot_name, builder.unwrap().get_metadata(), {".metadata.duration_ms" => 0}); + } +} diff --git a/memfaultd/src/metrics/metric_string_key.rs b/memfaultd/src/metrics/metric_string_key.rs new file mode 100644 index 0000000..9d48030 --- /dev/null +++ b/memfaultd/src/metrics/metric_string_key.rs @@ -0,0 +1,115 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use nom::{ + combinator::map_res, + error::ParseError, + {AsChar, IResult, InputTakeAtPosition}, +}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::fmt::{Debug, Formatter}; +use std::str::FromStr; +use std::{borrow::Cow, fmt::Display}; + +/// Struct containing a valid metric / attribute key. +#[derive(Clone, PartialEq, Eq, Hash)] +#[repr(transparent)] +pub struct MetricStringKey { + inner: String, +} + +impl MetricStringKey { + pub fn as_str(&self) -> &str { + &self.inner + } + + pub fn metric_string_key_parser>(input: T) -> IResult + where + T: InputTakeAtPosition, + ::Item: AsChar, + { + input.split_at_position_complete(|item| { + let c = item.as_char(); + !(c.is_alphanumeric() || c == '_' || c == '/' || c == '.') + }) + } + + pub fn parse(input: &str) -> IResult<&str, Self> { + map_res(Self::metric_string_key_parser, Self::from_str)(input) + } +} + +impl Debug for MetricStringKey { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + std::fmt::Debug::fmt(&self.inner, f) + } +} + +impl FromStr for MetricStringKey { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + if !(1..=128).contains(&s.len()) { + return Err("Invalid key: must be between 1 and 128 characters"); + } + if !s.is_ascii() { + return Err("Invalid key: must be ASCII"); + } + Ok(Self { + inner: s.to_string(), + }) + } +} + +impl Display for MetricStringKey { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(&self.inner, f) + } +} + +impl Ord for MetricStringKey { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.inner.cmp(&other.inner) + } +} +impl PartialOrd for MetricStringKey { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.inner.cmp(&other.inner)) + } +} + +impl Serialize for MetricStringKey { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> Deserialize<'de> for MetricStringKey { + fn deserialize>(deserializer: D) -> Result { + let s: Cow = Deserialize::deserialize(deserializer)?; + let key: MetricStringKey = str::parse(&s).map_err(serde::de::Error::custom)?; + Ok(key) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::*; + + #[rstest] + #[case("", "Invalid key: must be between 1 and 128 characters")] + #[case("\u{1F4A9}", "Invalid key: must be ASCII")] + fn validation_errors(#[case] input: &str, #[case] expected: &str) { + let result: Result = str::parse(input); + assert_eq!(result.err().unwrap(), expected); + } + + #[rstest] + #[case("foo")] + #[case("weird valid.key-123$")] + fn parsed_ok(#[case] input: &str) { + let result: Result = str::parse(input); + assert_eq!(result.ok().unwrap().as_str(), input); + } +} diff --git a/memfaultd/src/metrics/metric_value.rs b/memfaultd/src/metrics/metric_value.rs new file mode 100644 index 0000000..ceb4724 --- /dev/null +++ b/memfaultd/src/metrics/metric_value.rs @@ -0,0 +1,24 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::Result; +use serde::{Deserialize, Serialize, Serializer}; + +#[derive(Clone, Debug, PartialEq, Deserialize)] +#[serde(untagged)] +pub enum MetricValue { + Number(f64), + String(String), +} + +impl Serialize for MetricValue { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self { + MetricValue::Number(v) => serializer.serialize_f64(*v), + MetricValue::String(v) => serializer.serialize_str(v.as_str()), + } + } +} diff --git a/memfaultd/src/metrics/mod.rs b/memfaultd/src/metrics/mod.rs new file mode 100644 index 0000000..8261daa --- /dev/null +++ b/memfaultd/src/metrics/mod.rs @@ -0,0 +1,55 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use chrono::{DateTime, Utc}; + +mod battery; +pub use battery::BatteryMonitor; +pub use battery::BatteryMonitorReading; +pub use battery::BatteryReadingHandler; + +mod connectivity; +pub use connectivity::ConnectivityMonitor; +pub use connectivity::ReportSyncEventHandler; + +mod metric_string_key; +pub use metric_string_key::MetricStringKey; + +mod metric_report; +pub use metric_report::MetricReport; +pub use metric_report::MetricReportType; + +mod metric_report_manager; +pub use metric_report_manager::MetricReportManager; + +mod metric_reading; +pub use metric_reading::KeyedMetricReading; +pub use metric_reading::MetricReading; + +mod timeseries; + +mod metric_value; +pub use metric_value::MetricValue; + +pub type MetricTimestamp = DateTime; + +mod periodic_metric_report; +pub use periodic_metric_report::PeriodicMetricReportDumper; + +mod crashfree_interval; +pub use crashfree_interval::CrashFreeIntervalTracker; + +mod session_name; +pub use session_name::SessionName; + +mod session_event_handler; +pub use session_event_handler::SessionEventHandler; + +pub mod core_metrics; + +pub mod statsd_server; +pub use statsd_server::StatsDServer; + +mod system_metrics; +pub use system_metrics::SystemMetricsCollector; +pub use system_metrics::BUILTIN_SYSTEM_METRIC_NAMESPACES; diff --git a/memfaultd/src/metrics/periodic_metric_report.rs b/memfaultd/src/metrics/periodic_metric_report.rs new file mode 100644 index 0000000..3707ba1 --- /dev/null +++ b/memfaultd/src/metrics/periodic_metric_report.rs @@ -0,0 +1,131 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + path::PathBuf, + sync::{Arc, Mutex}, + thread::sleep, + time::{Duration, Instant}, +}; + +use log::warn; + +use crate::network::NetworkConfig; + +use super::{MetricReportManager, MetricReportType}; + +/// A runner that periodically dumps metrics to a MAR entry +/// +/// This runner will periodically dump metrics of a given type to a MAR entry +/// in the background. The runner will sleep between dumps. +pub struct PeriodicMetricReportDumper { + metric_report_manager: Arc>, + net_config: NetworkConfig, + report_interval: Duration, + mar_staging_path: PathBuf, + report_type: MetricReportType, +} + +impl PeriodicMetricReportDumper { + pub fn new( + mar_staging_path: PathBuf, + net_config: NetworkConfig, + metric_report_manager: Arc>, + report_interval: Duration, + report_type: MetricReportType, + ) -> Self { + Self { + metric_report_manager, + net_config, + report_interval, + mar_staging_path, + report_type, + } + } + + pub fn start(&self) { + let mut next_report = Instant::now() + self.report_interval; + loop { + while Instant::now() < next_report { + sleep(next_report - Instant::now()); + } + + self.run_once(&mut next_report); + } + } + + fn run_once(&self, next_report: &mut Instant) { + *next_report += self.report_interval; + if let Err(e) = MetricReportManager::dump_report_to_mar_entry( + &self.metric_report_manager, + &self.mar_staging_path, + &self.net_config, + &self.report_type, + ) { + warn!("Unable to dump metrics: {}", e); + } + } +} + +#[cfg(test)] +mod test { + use tempfile::tempdir; + + use super::*; + + use crate::mar::manifest::Metadata; + use crate::test_utils::in_histograms; + + #[test] + fn test_happy_path_metric_report() { + let report_manager = Arc::new(Mutex::new(MetricReportManager::new())); + let tempdir = tempdir().unwrap(); + let mar_staging_path = tempdir.path().to_owned(); + let readings = in_histograms(vec![("hello", 10.0), ("mad", 20.0)]).collect::>(); + + { + let mut report_manager = report_manager.lock().unwrap(); + + for reading in &readings { + report_manager.add_metric(reading.clone()).unwrap(); + } + } + + let report_interval = Duration::from_secs(1); + let runner = PeriodicMetricReportDumper::new( + mar_staging_path.clone(), + NetworkConfig::test_fixture(), + report_manager, + report_interval, + MetricReportType::Heartbeat, + ); + let mut next_report = Instant::now(); + let report_start = next_report; + runner.run_once(&mut next_report); + + let entries = crate::mar::MarEntry::iterate_from_container(&mar_staging_path) + .unwrap() + .map(Result::unwrap) + .collect::>(); + + assert_eq!(next_report.duration_since(report_start), report_interval); + assert_eq!(entries.len(), 1); + let metadata = &entries[0].manifest.metadata; + match metadata { + Metadata::LinuxMetricReport { + metrics, + report_type, + .. + } => { + assert_eq!(metrics.len(), 2); + + matches!(report_type, &MetricReportType::Heartbeat); + + for reading in readings { + assert!(metrics.contains_key(&reading.name)); + } + } + _ => panic!("Unexpected metadata"), + } + } +} diff --git a/memfaultd/src/metrics/session_event_handler.rs b/memfaultd/src/metrics/session_event_handler.rs new file mode 100644 index 0000000..26b2756 --- /dev/null +++ b/memfaultd/src/metrics/session_event_handler.rs @@ -0,0 +1,447 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + io::Read, + path::PathBuf, + str::{from_utf8, FromStr}, + sync::{Arc, Mutex}, +}; + +use eyre::{eyre, Result}; +use log::warn; +use tiny_http::{Method, Request, Response}; + +use crate::{ + http_server::{HttpHandler, HttpHandlerResult, SessionRequest}, + metrics::{MetricReportManager, MetricReportType, SessionName}, + network::NetworkConfig, +}; + +use super::KeyedMetricReading; + +/// A server that listens for session management requests +#[derive(Clone)] +pub struct SessionEventHandler { + data_collection_enabled: bool, + metrics_store: Arc>, + mar_staging_path: PathBuf, + network_config: NetworkConfig, +} + +impl SessionEventHandler { + pub fn new( + data_collection_enabled: bool, + metrics_store: Arc>, + mar_staging_path: PathBuf, + network_config: NetworkConfig, + ) -> Self { + Self { + data_collection_enabled, + metrics_store, + mar_staging_path, + network_config, + } + } + + fn parse_request(stream: &mut dyn Read) -> Result { + let mut buf = vec![]; + stream.read_to_end(&mut buf)?; + let body = from_utf8(&buf)?; + match serde_json::from_str(body) { + Ok(request_body) => Ok(request_body), + // Fall back to legacy API, SessionName as raw str in body (no JSON) + Err(e) => { + // If the request doesn't match either the JSON API or legacy API, + // include JSON API parse error in response (as that is currently + // the standard API) + Ok(SessionRequest::new_without_readings( + SessionName::from_str(body) + .map_err(|_| eyre!("Couldn't parse request: {}", e))?, + )) + } + } + } + + fn add_metric_readings_to_session( + session_name: &SessionName, + metric_reports: Arc>, + metric_readings: Vec, + ) -> Result<()> { + let mut metric_reports = metric_reports.lock().expect("Mutex poisoned!"); + for metric_reading in metric_readings { + metric_reports.add_metric_to_report( + &MetricReportType::Session(session_name.clone()), + metric_reading, + )? + } + + Ok(()) + } +} + +impl HttpHandler for SessionEventHandler { + fn handle_request(&self, request: &mut Request) -> HttpHandlerResult { + if (request.url() != "/v1/session/start" && request.url() != "/v1/session/end") + || *request.method() != Method::Post + { + return HttpHandlerResult::NotHandled; + } + + if self.data_collection_enabled { + match Self::parse_request(request.as_reader()) { + Ok(SessionRequest { + session_name, + readings, + }) => { + if request.url() == "/v1/session/start" { + if let Err(e) = self + .metrics_store + .lock() + .expect("Mutex poisoned") + .start_session(session_name.clone()) + { + return HttpHandlerResult::Error(format!( + "Failed to start session: {:?}", + e + )); + } + } + + // Add additional metric readings after the session has started + // (if starting) but before it has ended (if ending) + if !readings.is_empty() { + if let Err(e) = Self::add_metric_readings_to_session( + &session_name, + self.metrics_store.clone(), + readings, + ) { + warn!("Failed to add metrics to session report: {}", e); + } + } + + if request.url() == "/v1/session/end" { + if let Err(e) = MetricReportManager::dump_report_to_mar_entry( + &self.metrics_store, + &self.mar_staging_path, + &self.network_config, + &MetricReportType::Session(session_name), + ) { + return HttpHandlerResult::Error(format!( + "Failed to end session: {:?}", + e + )); + } + } + } + Err(e) => { + warn!("Failed to parse session request: {:?}", e); + return HttpHandlerResult::Error(format!( + "Failed to parse session request: {:?}", + e + )); + } + } + } + HttpHandlerResult::Response(Response::empty(200).boxed()) + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::BTreeMap, + path::Path, + str::FromStr, + sync::{Arc, Mutex}, + }; + + use insta::assert_json_snapshot; + use rstest::{fixture, rstest}; + use tempfile::TempDir; + use tiny_http::{Method, TestRequest}; + + use crate::{ + config::SessionConfig, + http_server::{HttpHandler, HttpHandlerResult}, + mar::manifest::{Manifest, Metadata}, + metrics::{MetricReportManager, MetricStringKey, SessionName}, + network::NetworkConfig, + test_utils::in_histograms, + }; + + use super::*; + use crate::test_utils::setup_logger; + + #[rstest] + fn test_start_without_stop_session(handler: SessionEventHandler) { + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/session/start") + .with_body("test-session"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + let mut metric_report_manager = handler.metrics_store.lock().unwrap(); + let readings = in_histograms(vec![("foo", 1.0), ("bar", 2.0), ("not-captured", 3.0)]); + + for reading in readings { + metric_report_manager + .add_metric(reading) + .expect("Failed to add metric reading"); + } + + let metrics: BTreeMap<_, _> = metric_report_manager + .take_session_metrics(&SessionName::from_str("test-session").unwrap()) + .unwrap() + .into_iter() + .collect(); + + assert_json_snapshot!(metrics); + } + + #[rstest] + fn test_start_with_metrics(_setup_logger: (), handler: SessionEventHandler) { + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/session/start") + .with_body("{\"session_name\": \"test-session\", + \"readings\": + [ + {\"name\": \"foo\", \"value\": {\"Gauge\": {\"value\": 1.0, \"timestamp\": \"2024-01-01 00:00:00 UTC\"}}}, + {\"name\": \"bar\", \"value\": {\"Gauge\": {\"value\": 4.0, \"timestamp\": \"2024-01-01 00:00:00 UTC\"}}}, + {\"name\": \"baz\", \"value\": {\"ReportTag\": {\"value\": \"test-tag\", \"timestamp\": \"2024-01-01 00:00:00 UTC\"}}} + ] + }"); + let response = handler.handle_request(&mut r.into()); + assert!(matches!(response, HttpHandlerResult::Response(_))); + + let mut metric_report_manager = handler.metrics_store.lock().unwrap(); + let metrics: BTreeMap<_, _> = metric_report_manager + .take_session_metrics(&SessionName::from_str("test-session").unwrap()) + .unwrap() + .into_iter() + .collect(); + + assert_json_snapshot!(metrics); + } + + #[rstest] + fn test_end_with_metrics(_setup_logger: ()) { + let session_config = SessionConfig { + name: SessionName::from_str("test-session").unwrap(), + captured_metrics: vec![ + MetricStringKey::from_str("foo").unwrap(), + MetricStringKey::from_str("bar").unwrap(), + ], + }; + + let tempdir = TempDir::new().unwrap(); + let handler = SessionEventHandler::new( + true, + Arc::new(Mutex::new(MetricReportManager::new_with_session_configs( + &[session_config], + ))), + tempdir.path().to_path_buf(), + NetworkConfig::test_fixture(), + ); + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/session/start") + .with_body("test-session"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/session/end") + .with_body("{\"session_name\": \"test-session\", + \"readings\": + [ + {\"name\": \"foo\", \"value\": {\"Gauge\": {\"value\": 1.0, \"timestamp\": \"2024-01-01 00:00:00 UTC\"}}}, + {\"name\": \"bar\", \"value\": {\"Gauge\": {\"value\": 3.0, \"timestamp\": \"2024-01-01 00:00:00 UTC\"}}} + ] + }"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + verify_dumped_metric_report(&handler.mar_staging_path, "end_with_metrics") + } + + #[rstest] + fn test_start_twice_without_stop_session(_setup_logger: (), handler: SessionEventHandler) { + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/session/start") + .with_body("test-session"); + + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + { + let mut metric_report_manager = handler.metrics_store.lock().unwrap(); + let readings = + in_histograms(vec![("foo", 10.0), ("bar", 20.0), ("not-captured", 30.0)]); + + for reading in readings { + metric_report_manager + .add_metric(reading) + .expect("Failed to add metric reading"); + } + } + + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/session/start") + .with_body("test-session"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + let mut metric_report_manager = handler.metrics_store.lock().unwrap(); + let readings = in_histograms(vec![("foo", 1.0), ("bar", 2.0), ("not-captured", 3.0)]); + + for reading in readings { + metric_report_manager + .add_metric(reading) + .expect("Failed to add metric reading"); + } + + let metrics: BTreeMap<_, _> = metric_report_manager + .take_session_metrics(&SessionName::from_str("test-session").unwrap()) + .unwrap() + .into_iter() + .collect(); + + assert_json_snapshot!(metrics); + } + + #[rstest] + fn test_start_then_stop_session(_setup_logger: ()) { + let session_config = SessionConfig { + name: SessionName::from_str("test-session").unwrap(), + captured_metrics: vec![ + MetricStringKey::from_str("foo").unwrap(), + MetricStringKey::from_str("bar").unwrap(), + MetricStringKey::from_str("baz").unwrap(), + ], + }; + + let tempdir = TempDir::new().unwrap(); + let handler = SessionEventHandler::new( + true, + Arc::new(Mutex::new(MetricReportManager::new_with_session_configs( + &[session_config], + ))), + tempdir.path().to_path_buf(), + NetworkConfig::test_fixture(), + ); + + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/session/start") + .with_body("{\"session_name\": \"test-session\", \"readings\": []}"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + { + let mut metric_report_manager = handler.metrics_store.lock().unwrap(); + let readings = in_histograms(vec![("bar", 20.0), ("not-captured", 30.0)]); + + for reading in readings { + metric_report_manager + .add_metric(reading) + .expect("Failed to add metric reading"); + } + } + + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/session/end") + .with_body("{\"session_name\": \"test-session\", + \"readings\": + [ + {\"name\": \"foo\", \"value\": {\"Gauge\": {\"value\": 100, \"timestamp\": \"2024-01-01 00:00:00 UTC\"}}}, + {\"name\": \"baz\", \"value\": {\"ReportTag\": {\"value\": \"test-tag\", \"timestamp\": \"2024-01-01 00:00:00 UTC\"}}} + ] + }"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + verify_dumped_metric_report(&handler.mar_staging_path, "start_then_stop"); + let mut metric_report_manager = handler.metrics_store.lock().unwrap(); + + // Should error as session should have been removed from MetricReportManager + // after it was ended + assert!(metric_report_manager + .take_session_metrics(&SessionName::from_str("test-session").unwrap()) + .is_err()); + } + + #[rstest] + fn test_stop_without_start_session(handler: SessionEventHandler) { + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/session/end") + .with_body("test-session"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Error(_) + )); + } + + /// Creates a SessionEventHandler whose metric store is configured with + /// a "test-session" session that captures the "foo" and "bar" metrics + #[fixture] + fn handler() -> SessionEventHandler { + let session_config = SessionConfig { + name: SessionName::from_str("test-session").unwrap(), + captured_metrics: vec![ + MetricStringKey::from_str("foo").unwrap(), + MetricStringKey::from_str("bar").unwrap(), + MetricStringKey::from_str("baz").unwrap(), + ], + }; + + SessionEventHandler::new( + true, + Arc::new(Mutex::new(MetricReportManager::new_with_session_configs( + &[session_config], + ))), + TempDir::new().unwrap().path().to_path_buf(), + NetworkConfig::test_fixture(), + ) + } + + fn verify_dumped_metric_report(mar_staging_path: &Path, test_name: &str) { + let mar_dir = std::fs::read_dir(mar_staging_path) + .expect("Failed to read temp dir") + .filter_map(|entry| entry.ok()) + .collect::>(); + + // There should only be an entry for the reboot reason + assert_eq!(mar_dir.len(), 1); + + let mar_manifest = mar_dir[0].path().join("manifest.json"); + let manifest_string = std::fs::read_to_string(mar_manifest).unwrap(); + let manifest: Manifest = serde_json::from_str(&manifest_string).unwrap(); + + if let Metadata::LinuxMetricReport { .. } = manifest.metadata { + assert_json_snapshot!(test_name, manifest.metadata); + } else { + panic!("Unexpected metadata type"); + } + } +} diff --git a/memfaultd/src/metrics/session_name.rs b/memfaultd/src/metrics/session_name.rs new file mode 100644 index 0000000..5e6465a --- /dev/null +++ b/memfaultd/src/metrics/session_name.rs @@ -0,0 +1,94 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::fmt::{Debug, Formatter}; +use std::str::FromStr; +use std::{borrow::Cow, fmt::Display}; + +use eyre::{eyre, ErrReport, Result}; + +use crate::metrics::metric_report::{DAILY_HEARTBEAT_REPORT_TYPE, HEARTBEAT_REPORT_TYPE}; +use crate::util::patterns::alphanum_slug_is_valid_and_starts_alpha; +const RESERVED_SESSION_NAMES: &[&str; 2] = &[HEARTBEAT_REPORT_TYPE, DAILY_HEARTBEAT_REPORT_TYPE]; + +/// Struct containing a valid session name +#[derive(Clone, PartialEq, Eq, Hash, Ord, PartialOrd)] +#[repr(transparent)] +pub struct SessionName { + inner: String, +} + +impl SessionName { + pub fn as_str(&self) -> &str { + &self.inner + } +} + +impl Debug for SessionName { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + std::fmt::Debug::fmt(&self.inner, f) + } +} + +impl FromStr for SessionName { + type Err = ErrReport; + + fn from_str(s: &str) -> Result { + match alphanum_slug_is_valid_and_starts_alpha(s, 64) { + Ok(()) => { + if RESERVED_SESSION_NAMES.contains(&s) { + Err(eyre!("Cannot use reserved session name: {}", s)) + } else { + Ok(Self { + inner: s.to_string(), + }) + } + } + Err(e) => Err(eyre!("Invalid session name {}: {}", s, e)), + } + } +} + +impl Display for SessionName { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(&self.inner, f) + } +} + +impl Serialize for SessionName { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> Deserialize<'de> for SessionName { + fn deserialize>(deserializer: D) -> Result { + let s: Cow = Deserialize::deserialize(deserializer)?; + let name: SessionName = str::parse(&s).map_err(serde::de::Error::custom)?; + Ok(name) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::*; + + #[rstest] + #[case("")] + #[case("\u{1F4A9}")] + #[case("Wi-fi Connected")] + #[case("heartbeat")] + #[case("daily-heartbeat")] + fn validation_errors(#[case] input: &str) { + assert!(SessionName::from_str(input).is_err()) + } + + #[rstest] + #[case("foo")] + #[case("valid_session-name")] + fn parsed_ok(#[case] input: &str) { + assert!(SessionName::from_str(input).is_ok()) + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__aggregate_core_metrics_session.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__aggregate_core_metrics_session.snap new file mode 100644 index 0000000..6095daf --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__aggregate_core_metrics_session.snap @@ -0,0 +1,15 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "battery_discharge_duration_ms": 100.0, + "battery_soc_pct_drop": 100.0, + "connectivity_connected_time_ms": 100.0, + "connectivity_expected_time_ms": 100.0, + "operational_crashes": 100.0, + "sync_failure": 100.0, + "sync_memfault_failure": 100.0, + "sync_memfault_successful": 100.0, + "sync_successful": 100.0 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_1.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_1.snap new file mode 100644 index 0000000..b66b75d --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_1.snap @@ -0,0 +1,9 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "bar": 2.0, + "baz": 3.0, + "foo": 1.0 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_2.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_2.snap new file mode 100644 index 0000000..4420475 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_2.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "foo": 2.0 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_3.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_3.snap new file mode 100644 index 0000000..9a3a9ca --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_3.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "foo": 1.0 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_4.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_4.snap new file mode 100644 index 0000000..de0e20b --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_4.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "foo": 1.5 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_5.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_5.snap new file mode 100644 index 0000000..74f3f37 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_5.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "foo": 1.6666666666666667 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_1.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_1.snap new file mode 100644 index 0000000..089dbde --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_1.snap @@ -0,0 +1,8 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "baz": 3.0, + "foo": 1.0 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_2.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_2.snap new file mode 100644 index 0000000..4420475 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_2.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "foo": 2.0 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_3.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_3.snap new file mode 100644 index 0000000..9a3a9ca --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_3.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "foo": 1.0 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_4.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_4.snap new file mode 100644 index 0000000..de0e20b --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_4.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "foo": 1.5 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_5.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_5.snap new file mode 100644 index 0000000..468b5d0 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_5.snap @@ -0,0 +1,8 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "baz": 2.0, + "foo": 1.5 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__daily-heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__daily-heartbeat.snap new file mode 100644 index 0000000..2dba7e8 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__daily-heartbeat.snap @@ -0,0 +1,15 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 3.5, + "foo": 5.0 + }, + "duration_ms": 0, + "report_type": "daily-heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat.snap new file mode 100644 index 0000000..49f4490 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat.snap @@ -0,0 +1,15 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 3.5, + "foo": 5.0 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.daily_heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.daily_heartbeat.snap new file mode 100644 index 0000000..7d20519 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.daily_heartbeat.snap @@ -0,0 +1,16 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 2.0, + "baz": 3.0, + "foo": 1.0 + }, + "duration_ms": 0, + "report_type": "daily-heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.heartbeat.snap new file mode 100644 index 0000000..c5d86b3 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.heartbeat.snap @@ -0,0 +1,16 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 2.0, + "baz": 3.0, + "foo": 1.0 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.test-session-all-metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.test-session-all-metrics.snap new file mode 100644 index 0000000..249b685 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.test-session-all-metrics.snap @@ -0,0 +1,19 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 2.0, + "baz": 3.0, + "foo": 1.0, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-all-metrics" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.test-session-some-metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.test-session-some-metrics.snap new file mode 100644 index 0000000..3dfcdca --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.test-session-some-metrics.snap @@ -0,0 +1,18 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 2.0, + "foo": 1.0, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-some-metrics" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.daily_heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.daily_heartbeat.snap new file mode 100644 index 0000000..c3c95fc --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.daily_heartbeat.snap @@ -0,0 +1,14 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 2.0 + }, + "duration_ms": 0, + "report_type": "daily-heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.heartbeat.snap new file mode 100644 index 0000000..5dca47d --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.heartbeat.snap @@ -0,0 +1,14 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 2.0 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.test-session-all-metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.test-session-all-metrics.snap new file mode 100644 index 0000000..a18ecb4 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.test-session-all-metrics.snap @@ -0,0 +1,17 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 2.0, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-all-metrics" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.test-session-some-metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.test-session-some-metrics.snap new file mode 100644 index 0000000..8d5b783 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.test-session-some-metrics.snap @@ -0,0 +1,17 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 2.0, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-some-metrics" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.daily_heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.daily_heartbeat.snap new file mode 100644 index 0000000..dfbc85a --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.daily_heartbeat.snap @@ -0,0 +1,14 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 1.0 + }, + "duration_ms": 0, + "report_type": "daily-heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.heartbeat.snap new file mode 100644 index 0000000..4499c05 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.heartbeat.snap @@ -0,0 +1,14 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 1.0 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.test-session-all-metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.test-session-all-metrics.snap new file mode 100644 index 0000000..86dab3c --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.test-session-all-metrics.snap @@ -0,0 +1,17 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 1.0, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-all-metrics" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.test-session-some-metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.test-session-some-metrics.snap new file mode 100644 index 0000000..f7fb507 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.test-session-some-metrics.snap @@ -0,0 +1,17 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 1.0, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-some-metrics" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.daily_heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.daily_heartbeat.snap new file mode 100644 index 0000000..07efa1e --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.daily_heartbeat.snap @@ -0,0 +1,15 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "baz": 1.5, + "foo": 1.5 + }, + "duration_ms": 0, + "report_type": "daily-heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.heartbeat.snap new file mode 100644 index 0000000..31599bd --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.heartbeat.snap @@ -0,0 +1,15 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "baz": 1.5, + "foo": 1.5 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.test-session-all-metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.test-session-all-metrics.snap new file mode 100644 index 0000000..4843681 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.test-session-all-metrics.snap @@ -0,0 +1,18 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "baz": 1.5, + "foo": 1.5, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-all-metrics" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.test-session-some-metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.test-session-some-metrics.snap new file mode 100644 index 0000000..5d9d0fd --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.test-session-some-metrics.snap @@ -0,0 +1,17 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 1.5, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-some-metrics" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.daily_heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.daily_heartbeat.snap new file mode 100644 index 0000000..5c86285 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.daily_heartbeat.snap @@ -0,0 +1,15 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 2.0, + "foo": 1.5 + }, + "duration_ms": 0, + "report_type": "daily-heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.heartbeat.snap new file mode 100644 index 0000000..e690567 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.heartbeat.snap @@ -0,0 +1,15 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 2.0, + "foo": 1.5 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.test-session-all-metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.test-session-all-metrics.snap new file mode 100644 index 0000000..4229d9e --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.test-session-all-metrics.snap @@ -0,0 +1,18 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 2.0, + "foo": 1.5, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-all-metrics" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.test-session-some-metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.test-session-some-metrics.snap new file mode 100644 index 0000000..b29a3ce --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.test-session-some-metrics.snap @@ -0,0 +1,18 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 2.0, + "foo": 1.5, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-some-metrics" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_1.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_1.snap new file mode 100644 index 0000000..c5d86b3 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_1.snap @@ -0,0 +1,16 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 2.0, + "baz": 3.0, + "foo": 1.0 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_2.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_2.snap new file mode 100644 index 0000000..5dca47d --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_2.snap @@ -0,0 +1,14 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 2.0 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_3.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_3.snap new file mode 100644 index 0000000..4499c05 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_3.snap @@ -0,0 +1,14 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 1.0 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_4.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_4.snap new file mode 100644 index 0000000..be2df83 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_4.snap @@ -0,0 +1,14 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 1.5 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_5.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_5.snap new file mode 100644 index 0000000..b783ecc --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_5.snap @@ -0,0 +1,14 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 1.6666666666666667 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__start_session_twice.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__start_session_twice.snap new file mode 100644 index 0000000..caf83fd --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__start_session_twice.snap @@ -0,0 +1,18 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 3.5, + "foo": 5.0, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-start-twice" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__test-session.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__test-session.snap new file mode 100644 index 0000000..bb5d7b8 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__test-session.snap @@ -0,0 +1,18 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 3.5, + "foo": 5.0, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__end_with_metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__end_with_metrics.snap new file mode 100644 index 0000000..58f2ccd --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__end_with_metrics.snap @@ -0,0 +1,18 @@ +--- +source: memfaultd/src/metrics/session_event_handler.rs +expression: manifest.metadata +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 3.0, + "foo": 1.0, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_then_stop.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_then_stop.snap new file mode 100644 index 0000000..266ebc6 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_then_stop.snap @@ -0,0 +1,19 @@ +--- +source: memfaultd/src/metrics/session_event_handler.rs +expression: manifest.metadata +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 20.0, + "baz": "test-tag", + "foo": 100.0, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_twice_without_stop_session.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_twice_without_stop_session.snap new file mode 100644 index 0000000..bb3f755 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_twice_without_stop_session.snap @@ -0,0 +1,9 @@ +--- +source: memfaultd/src/metrics/session_event_handler.rs +expression: metrics +--- +{ + "bar": 11.0, + "foo": 5.5, + "operational_crashes": 0.0 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_with_metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_with_metrics.snap new file mode 100644 index 0000000..3e8e135 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_with_metrics.snap @@ -0,0 +1,10 @@ +--- +source: memfaultd/src/metrics/session_event_handler.rs +expression: metrics +--- +{ + "bar": 4.0, + "baz": "test-tag", + "foo": 1.0, + "operational_crashes": 0.0 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_without_stop_session.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_without_stop_session.snap new file mode 100644 index 0000000..0536e61 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_without_stop_session.snap @@ -0,0 +1,9 @@ +--- +source: memfaultd/src/metrics/session_event_handler.rs +expression: metrics +--- +{ + "bar": 2.0, + "foo": 1.0, + "operational_crashes": 0.0 +} diff --git a/memfaultd/src/metrics/statsd_server/mod.rs b/memfaultd/src/metrics/statsd_server/mod.rs new file mode 100644 index 0000000..198cb36 --- /dev/null +++ b/memfaultd/src/metrics/statsd_server/mod.rs @@ -0,0 +1,126 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::net::SocketAddr; +use std::net::UdpSocket; +use std::sync::{Arc, Mutex}; +use std::thread::spawn; + +use eyre::Result; +use log::warn; + +use crate::metrics::{KeyedMetricReading, MetricReportManager}; + +pub struct StatsDServer {} + +impl StatsDServer { + pub fn new() -> StatsDServer { + StatsDServer {} + } + + pub fn start( + &self, + listening_address: SocketAddr, + metric_report_manager: Arc>, + ) -> Result<()> { + let socket = UdpSocket::bind(listening_address)?; + spawn(move || { + loop { + // This means that packets with > 1432 bytes are NOT supported + // Clients must enforce a maximum message size of 1432 bytes or less + let mut buf = [0; 1432]; + match socket.recv(&mut buf) { + Ok(amt) => { + let message = String::from_utf8_lossy(&buf[..amt]); + Self::process_statsd_message(&message, &metric_report_manager) + } + Err(e) => warn!("Statsd server socket error: {}", e), + } + } + }); + Ok(()) + } + + fn process_statsd_message( + message: &str, + metric_report_manager: &Arc>, + ) { + // https://github.com/statsd/statsd/blob/master/docs/server.md + // From statsd spec: + // Multiple metrics can be received in a single packet if separated by the \n character. + let metric_readings = message + .trim() + .lines() + .map(KeyedMetricReading::from_statsd_str) + // Drop strings that couldn't be parsed as a KeyedMetricReading + .filter_map(|res| { + if let Err(e) = &res { + warn!("{}", e) + }; + res.ok() + }); + + for metric_reading in metric_readings { + if let Err(e) = metric_report_manager + .lock() + .expect("Mutex poisoned!") + .add_metric(metric_reading) + { + warn!("Error adding metric sent to StatsD server: {}", e); + } + } + } +} + +impl Default for StatsDServer { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod test { + use std::collections::BTreeMap; + + use insta::assert_json_snapshot; + use rstest::rstest; + + use super::*; + + #[rstest] + #[case("test_counter:1|c", "test_gauge:2.0|g", "test_simple")] + #[case("test_counter:1|c", "test_counter:1|c", "test_counter_aggregation")] + #[case( + "test_counter:1|c\ntest_gauge:2.0|g", + "test_counter:1|c\ntest_gauge:10.0|g", + "test_counter_and_gauge_aggregation" + )] + #[case( + "test_histo:100|h\ntest_another_histo:20.0|h", + "test_one_more_histo:35|h\ntest_another_histo:1000.0|h", + "test_histogram_aggregation" + )] + fn test_process_statsd_message( + #[case] statsd_message_a: &str, + #[case] statsd_message_b: &str, + #[case] test_name: &str, + ) { + let metric_report_manager = Arc::new(Mutex::new(MetricReportManager::new())); + + // Process first StatsD test message + StatsDServer::process_statsd_message(statsd_message_a, &metric_report_manager); + + // Process second StatsD test message + StatsDServer::process_statsd_message(statsd_message_b, &metric_report_manager); + + // Verify resulting metric report + let metrics: BTreeMap<_, _> = metric_report_manager + .lock() + .unwrap() + .take_heartbeat_metrics() + .into_iter() + .collect(); + + assert_json_snapshot!(test_name, metrics); + } +} diff --git a/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_counter_aggregation.snap b/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_counter_aggregation.snap new file mode 100644 index 0000000..8ebff42 --- /dev/null +++ b/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_counter_aggregation.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/metrics/statsd_server/mod.rs +expression: metrics +--- +{ + "test_counter": 2.0 +} diff --git a/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_counter_and_gauge_aggregation.snap b/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_counter_and_gauge_aggregation.snap new file mode 100644 index 0000000..194f9d0 --- /dev/null +++ b/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_counter_and_gauge_aggregation.snap @@ -0,0 +1,8 @@ +--- +source: memfaultd/src/metrics/statsd_server/mod.rs +expression: metrics +--- +{ + "test_counter": 2.0, + "test_gauge": 10.0 +} diff --git a/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_histogram_aggregation.snap b/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_histogram_aggregation.snap new file mode 100644 index 0000000..731e6e6 --- /dev/null +++ b/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_histogram_aggregation.snap @@ -0,0 +1,9 @@ +--- +source: memfaultd/src/metrics/statsd_server/mod.rs +expression: metrics +--- +{ + "test_another_histo": 510.0, + "test_histo": 100.0, + "test_one_more_histo": 35.0 +} diff --git a/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_simple.snap b/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_simple.snap new file mode 100644 index 0000000..157abc1 --- /dev/null +++ b/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_simple.snap @@ -0,0 +1,8 @@ +--- +source: memfaultd/src/metrics/statsd_server/mod.rs +expression: metrics +--- +{ + "test_counter": 1.0, + "test_gauge": 2.0 +} diff --git a/memfaultd/src/metrics/system_metrics/cpu.rs b/memfaultd/src/metrics/system_metrics/cpu.rs new file mode 100644 index 0000000..be95095 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/cpu.rs @@ -0,0 +1,304 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Collect CPU metric readings from /proc/stat +//! +//! This module parses CPU statistics from /proc/stat and +//! constructs KeyedMetricReadings based on those statistics. +//! Because the /proc/stat values are accumulations since boot, +//! a "previous reading" (stored in CpuMetricCollector) is +//! required to calculate the time each CPU core +//! has spent in each state since the last reading. +//! +//! Example /proc/stat contents: +//! cpu 326218 0 178980 36612114 6054 0 11961 0 0 0 +//! cpu0 77186 0 73689 9126238 1353 0 6352 0 0 0 +//! cpu1 83902 0 35260 9161039 1524 0 1865 0 0 0 +//! cpu2 83599 0 35323 9161010 1676 0 1875 0 0 0 +//! cpu3 81530 0 34707 9163825 1500 0 1867 0 0 0 +//! intr 95400676 0 9795 1436573 0 0 0 0 0 0 0 0 93204555 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 77883 0 530 0 0 1523 0 0 468762 0 0 97412 103573 0 70 0 0 0 0 0 +//! ctxt 9591503 +//! btime 1714309294 +//! processes 9416 +//! procs_running 1 +//! procs_blocked 0 +//! softirq 47765068 15 3173702 0 541726 82192 0 1979 41497887 0 2467567 +//! +//! Only the lines that start with "cpu" are currently +//! processed into metric readings by this module - the rest are discarded. +//! +//! See additional Linux kernel documentation on /proc/stat here: +//! https://docs.kernel.org/filesystems/proc.html#miscellaneous-kernel-statistics-in-proc-stat +use std::collections::HashMap; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::iter::zip; +use std::ops::Add; +use std::path::Path; +use std::str::FromStr; + +use chrono::Utc; +use nom::{ + bytes::complete::tag, + character::complete::{digit0, space1}, + multi::count, + number::complete::double, + sequence::{pair, preceded}, + IResult, +}; + +use crate::metrics::{ + system_metrics::SystemMetricFamilyCollector, KeyedMetricReading, MetricReading, MetricStringKey, +}; +use eyre::{eyre, ErrReport, Result}; + +const PROC_STAT_PATH: &str = "/proc/stat"; +pub const CPU_METRIC_NAMESPACE: &str = "cpu"; + +pub struct CpuMetricCollector { + last_reading_by_cpu: HashMap>, +} + +impl CpuMetricCollector { + pub fn new() -> Self { + Self { + last_reading_by_cpu: HashMap::new(), + } + } + + pub fn get_cpu_metrics(&mut self) -> Result> { + // Track if any lines in /proc/stat are parse-able + // so we can alert user if none are + let mut no_parseable_lines = true; + + let path = Path::new(PROC_STAT_PATH); + + let file = File::open(path)?; + let reader = BufReader::new(file); + + let mut cpu_metric_readings = vec![]; + + for line in reader.lines() { + // Discard errors - the assumption here is that we are only parsing + // lines that follow the specified format and expect other lines in the file to error + if let Ok((cpu_id, cpu_stats)) = Self::parse_proc_stat_line_cpu(line?.trim()) { + no_parseable_lines = false; + if let Ok(Some(mut readings)) = self.delta_since_last_reading(cpu_id, cpu_stats) { + cpu_metric_readings.append(&mut readings); + } + } + } + + // Check if we were able to parse at least one CPU metric reading + if !no_parseable_lines { + Ok(cpu_metric_readings) + } else { + Err(eyre!( + "No CPU metrics were collected from {} - is it a properly formatted /proc/stat file?", + PROC_STAT_PATH + )) + } + } + + /// Parse a cpu ID from a line of /proc/stat + /// + /// A cpu ID is the digit following the 3 character string "cpu" + /// No ID being present implies these stats are for the total + /// of all cores on the CPU + fn parse_cpu_id(input: &str) -> IResult<&str, &str> { + preceded(tag("cpu"), digit0)(input) + } + + /// Parse the CPU stats from the suffix of a /proc/stat line following the cpu ID + /// + /// 7 or more space delimited integers are expected. Values after the 7th are discarded. + fn parse_cpu_stats(input: &str) -> IResult<&str, Vec> { + count(preceded(space1, double), 7)(input) + } + + /// Parse the output of a line of /proc/stat, returning + /// a pair of the cpu ID that the parsed line corresponds + /// to and the first 7 floats listed for it + /// + /// The 7 floats represent how much time since boot the cpu has + /// spent in the "user", "nice", "system", "idle", "iowait", "irq", + /// "softirq", in that order + /// Example of a valid parse-able line: + /// cpu2 36675 176 11216 1552961 689 0 54 + fn parse_proc_stat_line_cpu(line: &str) -> Result<(String, Vec)> { + let (_remaining, (cpu_id, cpu_stats)) = + pair(Self::parse_cpu_id, Self::parse_cpu_stats)(line) + .map_err(|_e| eyre!("Failed to parse CPU stats line: {}", line))?; + Ok(("cpu".to_string().add(cpu_id), cpu_stats)) + } + + /// Calculate the time spent in each state for the + /// provided CPU core since the last reading collected + /// by the CpuMetricCollector + /// + /// Returns an Ok(None) if there is no prior reading + /// to calculate a delta from. + fn delta_since_last_reading( + &mut self, + cpu_id: String, + cpu_stats: Vec, + ) -> Result>> { + // Check to make sure there was a previous reading to calculate a delta with + if let Some(last_stats) = self + .last_reading_by_cpu + .insert(cpu_id.clone(), cpu_stats.clone()) + { + let delta = cpu_stats + .iter() + .zip(last_stats) + .map(|(current, previous)| current - previous); + + let cpu_states_with_ticks = zip( + ["user", "nice", "system", "idle", "iowait", "irq", "softirq"], + delta, + ) + .collect::>(); + + let sum: f64 = cpu_states_with_ticks.iter().map(|(_k, v)| v).sum(); + let timestamp = Utc::now(); + + let readings = cpu_states_with_ticks + .iter() + .map(|(key, value)| -> Result { + Ok(KeyedMetricReading::new( + MetricStringKey::from_str(&format!( + "{}/{}/percent/{}", + CPU_METRIC_NAMESPACE, cpu_id, key + )) + .map_err(|e| eyre!(e))?, + MetricReading::Histogram { + // Transform raw tick value to a percentage + value: 100.0 * value / sum, + timestamp, + }, + )) + }) + .collect::>>(); + match readings { + Ok(readings) => Ok(Some(readings)), + Err(e) => Err(e), + } + } else { + Ok(None) + } + } +} + +impl SystemMetricFamilyCollector for CpuMetricCollector { + fn family_name(&self) -> &'static str { + CPU_METRIC_NAMESPACE + } + + fn collect_metrics(&mut self) -> Result> { + self.get_cpu_metrics() + } +} + +#[cfg(test)] +mod test { + + use insta::{assert_json_snapshot, rounded_redaction, with_settings}; + use rstest::rstest; + + use super::*; + + #[rstest] + #[case("cpu 1000 5 0 0 2 0 0", "test_basic_line")] + #[case("cpu0 1000 5 0 0 2 0 0 0 0 0", "test_basic_line_with_extra")] + fn test_process_valid_proc_stat_line(#[case] proc_stat_line: &str, #[case] test_name: &str) { + assert_json_snapshot!(test_name, + CpuMetricCollector::parse_proc_stat_line_cpu(proc_stat_line).unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + } + + #[rstest] + #[case("cpu 1000 5 0 0 2")] + #[case("1000 5 0 0 2 0 0 0 0 0")] + #[case("processor0 1000 5 0 0 2 0 0 0 0 0")] + #[case("softirq 403453672 10204651 21667771 199 12328940 529390 0 3519783 161759969 147995 193294974")] + fn test_fails_on_invalid_proc_stat_line(#[case] proc_stat_line: &str) { + assert!(CpuMetricCollector::parse_proc_stat_line_cpu(proc_stat_line).is_err()) + } + + #[rstest] + #[case( + "cpu 1000 5 0 0 2 0 0", + "cpu 1500 20 4 1 2 0 0", + "cpu 1550 200 40 3 3 0 0", + "basic_delta" + )] + fn test_cpu_metric_collector_calcs( + #[case] proc_stat_line_a: &str, + #[case] proc_stat_line_b: &str, + #[case] proc_stat_line_c: &str, + #[case] test_name: &str, + ) { + let mut cpu_metric_collector = CpuMetricCollector::new(); + + let (cpu, stats) = CpuMetricCollector::parse_proc_stat_line_cpu(proc_stat_line_a).unwrap(); + let result_a = cpu_metric_collector.delta_since_last_reading(cpu, stats); + matches!(result_a, Ok(None)); + + let (cpu, stats) = CpuMetricCollector::parse_proc_stat_line_cpu(proc_stat_line_b).unwrap(); + let result_b = cpu_metric_collector.delta_since_last_reading(cpu, stats); + + assert!(result_b.is_ok()); + + with_settings!({sort_maps => true}, { + assert_json_snapshot!(format!("{}_{}", test_name, "a_b_metrics"), + result_b.unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + }); + + let (cpu, stats) = CpuMetricCollector::parse_proc_stat_line_cpu(proc_stat_line_c).unwrap(); + let result_c = cpu_metric_collector.delta_since_last_reading(cpu, stats); + + assert!(result_c.is_ok()); + + with_settings!({sort_maps => true}, { + assert_json_snapshot!(format!("{}_{}", test_name, "b_c_metrics"), + result_c.unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + }); + } + + #[rstest] + #[case( + "cpu1 40 20 30 10 0 0 0", + "cpu0 1500 20 4 1 2 0 0", + "cpu1 110 30 40 12 5 3 0", + "different_cores" + )] + fn test_cpu_metric_collector_different_cores( + #[case] proc_stat_line_a: &str, + #[case] proc_stat_line_b: &str, + #[case] proc_stat_line_c: &str, + #[case] test_name: &str, + ) { + let mut cpu_metric_collector = CpuMetricCollector::new(); + + let (cpu, stats) = CpuMetricCollector::parse_proc_stat_line_cpu(proc_stat_line_a).unwrap(); + let result_a = cpu_metric_collector.delta_since_last_reading(cpu, stats); + matches!(result_a, Ok(None)); + + let (cpu, stats) = CpuMetricCollector::parse_proc_stat_line_cpu(proc_stat_line_b).unwrap(); + let result_b = cpu_metric_collector.delta_since_last_reading(cpu, stats); + matches!(result_b, Ok(None)); + + let (cpu, stats) = CpuMetricCollector::parse_proc_stat_line_cpu(proc_stat_line_c).unwrap(); + let result_c = cpu_metric_collector.delta_since_last_reading(cpu, stats); + + assert!(result_c.is_ok()); + + with_settings!({sort_maps => true}, { + assert_json_snapshot!(format!("{}_{}", test_name, "a_c_metrics"), + result_c.unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + }); + } +} diff --git a/memfaultd/src/metrics/system_metrics/disk_space.rs b/memfaultd/src/metrics/system_metrics/disk_space.rs new file mode 100644 index 0000000..0110c50 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/disk_space.rs @@ -0,0 +1,341 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Collect disk space metric readings for devices listed in +//! /proc/mounts +//! +//! This module parses mounted devices and their mount points +//! from /proc/mounts and calculates how many bytes are free +//! and used on the device. +//! +use std::{ + collections::HashSet, + fs::File, + io::{BufRead, BufReader}, + path::{Path, PathBuf}, +}; + +use eyre::{eyre, Result}; +use log::warn; +use nix::sys::statvfs::statvfs; +use nom::{ + bytes::complete::take_while, + character::complete::multispace1, + sequence::{pair, preceded}, + IResult, +}; + +use serde::Serialize; + +use crate::metrics::{system_metrics::SystemMetricFamilyCollector, KeyedMetricReading}; + +pub const DISKSPACE_METRIC_NAMESPACE_LEGACY: &str = "df"; +pub const DISKSPACE_METRIC_NAMESPACE: &str = "disk_space"; +pub const PROC_MOUNTS_PATH: &str = "/proc/mounts"; + +pub struct DiskSpaceInfo { + block_size: u64, + blocks: u64, + blocks_free: u64, +} + +#[cfg_attr(test, mockall::automock)] +pub trait DiskSpaceInfoForPath { + fn disk_space_info_for_path(&self, p: &Path) -> Result; +} + +pub struct NixStatvfs {} + +impl NixStatvfs { + pub fn new() -> Self { + Self {} + } +} + +impl DiskSpaceInfoForPath for NixStatvfs { + fn disk_space_info_for_path(&self, p: &Path) -> Result { + let statfs = statvfs(p) + .map_err(|e| eyre!("Failed to get statfs info for {}: {}", p.display(), e))?; + + Ok(DiskSpaceInfo { + // Ignore unnecessary cast for these + // as it is needed on 32-bit systems. + #[allow(clippy::unnecessary_cast)] + block_size: statfs.block_size() as u64, + #[allow(clippy::unnecessary_cast)] + blocks: statfs.blocks() as u64, + #[allow(clippy::unnecessary_cast)] + blocks_free: statfs.blocks_free() as u64, + }) + } +} + +#[derive(Serialize)] +struct Mount { + device: PathBuf, + mount_point: PathBuf, +} + +pub enum DiskSpaceMetricsConfig { + Auto, + Disks(HashSet), +} + +pub struct DiskSpaceMetricCollector +where + T: DiskSpaceInfoForPath, +{ + config: DiskSpaceMetricsConfig, + mounts: Vec, + disk_space_impl: T, +} + +impl DiskSpaceMetricCollector +where + T: DiskSpaceInfoForPath, +{ + pub fn new(disk_space_impl: T, config: DiskSpaceMetricsConfig) -> Self { + Self { + config, + mounts: Vec::new(), + disk_space_impl, + } + } + fn disk_is_monitored(&self, disk: &str) -> bool { + match &self.config { + DiskSpaceMetricsConfig::Auto => disk.starts_with("/dev"), + DiskSpaceMetricsConfig::Disks(configured_disks) => configured_disks.contains(disk), + } + } + + /// Parses a line of /proc/mounts for the name of + /// the device the line corresponds to + /// + /// Example input: + /// "/dev/sda2 / ext4 rw,noatime 0 0" + /// Example output: + /// "/dev/sda2" + fn parse_proc_mounts_device(proc_mounts_line: &str) -> IResult<&str, &str> { + take_while(|c: char| !c.is_whitespace())(proc_mounts_line) + } + + /// Parses a line of /proc/mounts for the + /// mount point the line corresponds to + /// Parses /proc/mounts for a list of devices with active + /// mount points in the system + /// Example input: + /// " / ext4 rw,noatime 0 0" + /// Example output: + /// "/" + fn parse_proc_mounts_mount_point(proc_mounts_line: &str) -> IResult<&str, &str> { + preceded(multispace1, take_while(|c: char| !c.is_whitespace()))(proc_mounts_line) + } + + /// Parse a line of /proc/mounts + /// Example input: + /// "/dev/sda2 / ext4 rw,noatime 0 0" + /// Example output: + /// Mount { device: "/dev/sda2", "mount_point": "/" } + fn parse_proc_mounts_line(line: &str) -> Result { + let (_remaining, (device, mount_point)) = pair( + Self::parse_proc_mounts_device, + Self::parse_proc_mounts_mount_point, + )(line) + .map_err(|e| eyre!("Failed to parse /proc/mounts line: {}", e))?; + Ok(Mount { + device: Path::new(device).to_path_buf(), + mount_point: Path::new(mount_point).to_path_buf(), + }) + } + + /// Initialize the list of mounted devices and their mount points based + /// on the contents of /proc/mounts + pub fn initialize_mounts(&mut self, proc_mounts_path: &Path) -> Result<()> { + let file = File::open(proc_mounts_path)?; + let reader = BufReader::new(file); + + for line in reader.lines() { + // Discard errors - the assumption here is that we are only parsing + // lines that follow the specified format and expect other lines in the file to error + if let Ok(mount) = Self::parse_proc_mounts_line(line?.trim()) { + if self.disk_is_monitored(&mount.device.to_string_lossy()) { + self.mounts.push(mount); + } + } + } + Ok(()) + } + + /// For a given mounted device, construct metric readings + /// for how many bytes are used and free on the device + fn get_metrics_for_mount(&self, mount: &Mount) -> Result> { + let mount_stats = self + .disk_space_impl + .disk_space_info_for_path(mount.mount_point.as_path())?; + + let block_size = mount_stats.block_size; + let bytes_free = mount_stats.blocks_free * block_size; + let bytes_used = (mount_stats.blocks * block_size) - bytes_free; + + let disk_id = mount + .device + .file_name() + .ok_or_else(|| eyre!("Couldn't extract basename"))? + .to_string_lossy(); + + let bytes_free_reading = KeyedMetricReading::new_histogram( + format!("disk_space/{}/free_bytes", disk_id) + .as_str() + .parse() + .map_err(|e| eyre!("Couldn't parse metric key for bytes free: {}", e))?, + bytes_free as f64, + ); + + let bytes_used_reading = KeyedMetricReading::new_histogram( + format!("disk_space/{}/used_bytes", disk_id) + .as_str() + .parse() + .map_err(|e| eyre!("Couldn't parse metric key for bytes used: {}", e))?, + bytes_used as f64, + ); + + Ok(vec![bytes_free_reading, bytes_used_reading]) + } + + pub fn get_disk_space_metrics(&mut self) -> Result> { + if self.mounts.is_empty() { + self.initialize_mounts(Path::new(PROC_MOUNTS_PATH))?; + } + + let mut disk_space_readings = Vec::new(); + for mount in self.mounts.iter() { + match self.get_metrics_for_mount(mount) { + Ok(readings) => disk_space_readings.extend(readings), + Err(e) => warn!( + "Failed to calculate disk space readings for {} mounted at {}: {}", + mount.device.display(), + mount.mount_point.display(), + e + ), + } + } + + Ok(disk_space_readings) + } +} + +impl SystemMetricFamilyCollector for DiskSpaceMetricCollector +where + T: DiskSpaceInfoForPath, +{ + fn family_name(&self) -> &'static str { + DISKSPACE_METRIC_NAMESPACE + } + + fn collect_metrics(&mut self) -> Result> { + self.get_disk_space_metrics() + } +} + +#[cfg(test)] +mod test { + use std::fs::File; + use std::io::Write; + + use insta::{assert_json_snapshot, rounded_redaction}; + use rstest::rstest; + use tempfile::tempdir; + + use super::*; + + #[rstest] + fn test_process_valid_proc_mounts_line() { + let line = "/dev/sda2 /media ext4 rw,noatime 0 0"; + let mount = + DiskSpaceMetricCollector::::parse_proc_mounts_line(line) + .unwrap(); + + assert_eq!(mount.device.as_os_str().to_string_lossy(), "/dev/sda2"); + assert_eq!(mount.mount_point.as_os_str().to_string_lossy(), "/media"); + } + + #[rstest] + fn test_initialize_and_calc_disk_space_for_mounts() { + let mut mock_statfs = MockDiskSpaceInfoForPath::new(); + + mock_statfs + .expect_disk_space_info_for_path() + .times(2) + .returning(|_p| { + Ok(DiskSpaceInfo { + block_size: 4096, + blocks: 1024, + blocks_free: 286, + }) + }); + + let mut disk_space_collector = + DiskSpaceMetricCollector::new(mock_statfs, DiskSpaceMetricsConfig::Auto); + + let line = "/dev/sda2 /media ext4 rw,noatime 0 0"; + let line2 = "/dev/sda1 / ext4 rw,noatime 0 0"; + + let dir = tempdir().unwrap(); + + let mounts_file_path = dir.path().join("mounts"); + let mut mounts_file = File::create(mounts_file_path.clone()).unwrap(); + + writeln!(mounts_file, "{}", line).unwrap(); + writeln!(mounts_file, "{}", line2).unwrap(); + + assert!(disk_space_collector + .initialize_mounts(&mounts_file_path) + .is_ok()); + + disk_space_collector + .mounts + .sort_by(|a, b| a.device.cmp(&b.device)); + + assert_json_snapshot!(disk_space_collector.mounts); + + let metrics = disk_space_collector.collect_metrics().unwrap(); + + assert_json_snapshot!(metrics, + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)} + ); + + dir.close().unwrap(); + } + #[rstest] + fn test_unmonitored_disks_not_initialized() { + let mock_statfs = MockDiskSpaceInfoForPath::new(); + + let mut disk_space_collector = DiskSpaceMetricCollector::new( + mock_statfs, + DiskSpaceMetricsConfig::Disks(HashSet::from_iter(["/dev/sdc1".to_string()])), + ); + + let line = "/dev/sda2 /media ext4 rw,noatime 0 0"; + let line2 = "/dev/sda1 / ext4 rw,noatime 0 0"; + + let dir = tempdir().unwrap(); + + let mounts_file_path = dir.path().join("mounts"); + let mut mounts_file = File::create(mounts_file_path.clone()).unwrap(); + + writeln!(mounts_file, "{}", line).unwrap(); + writeln!(mounts_file, "{}", line2).unwrap(); + + assert!(disk_space_collector + .initialize_mounts(&mounts_file_path) + .is_ok()); + + disk_space_collector + .mounts + .sort_by(|a, b| a.device.cmp(&b.device)); + + assert!(disk_space_collector.mounts.is_empty()); + + dir.close().unwrap(); + } +} diff --git a/memfaultd/src/metrics/system_metrics/memory.rs b/memfaultd/src/metrics/system_metrics/memory.rs new file mode 100644 index 0000000..8642a1f --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/memory.rs @@ -0,0 +1,291 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Collect memory metric readings from /proc/meminfo +//! +//! This module parses memory statistics from /proc/meminfo and +//! constructs KeyedMetricReadings based on those statistics. +//! +//! Example /proc/meminfo contents: +//! MemTotal: 365916 kB +//! MemFree: 242276 kB +//! MemAvailable: 292088 kB +//! Buffers: 4544 kB +//! Cached: 52128 kB +//! SwapCached: 0 kB +//! Active: 21668 kB +//! Inactive: 51404 kB +//! Active(anon): 2312 kB +//! Inactive(anon): 25364 kB +//! Active(file): 19356 kB +//! Inactive(file): 26040 kB +//! Unevictable: 3072 kB +//! Mlocked: 0 kB +//! SwapTotal: 0 kB +//! SwapFree: 0 kB +//! Dirty: 0 kB +//! Writeback: 0 kB +//! AnonPages: 19488 kB +//! Mapped: 29668 kB +//! Shmem: 11264 kB +//! KReclaimable: 14028 kB +//! Slab: 32636 kB +//! SReclaimable: 14028 kB +//! +//! Only the following lines are currently processed: +//! MemTotal, MemFree, and optionally MemAvailable +//! +//! These lines are used by this module to calculate +//! free and used memory. MemFree is used in place of +//! MemAvailable if the latter is not present. +//! +//! +//! See additional Linux kernel documentation on /proc/meminfo here: +//! https://www.kernel.org/doc/html/latest/filesystems/proc.html#meminfo +use std::fs::read_to_string; +use std::path::Path; +use std::{collections::HashMap, str::FromStr}; + +use eyre::{eyre, Result}; +use nom::{ + bytes::complete::tag, + character::complete::{alpha1, multispace1}, + number::complete::double, + sequence::{delimited, terminated}, + IResult, +}; + +use crate::metrics::{ + system_metrics::SystemMetricFamilyCollector, KeyedMetricReading, MetricStringKey, +}; + +const PROC_MEMINFO_PATH: &str = "/proc/meminfo"; +pub const MEMORY_METRIC_NAMESPACE: &str = "memory"; + +pub struct MemoryMetricsCollector; + +impl MemoryMetricsCollector { + pub fn new() -> Self { + MemoryMetricsCollector {} + } + + pub fn get_memory_metrics(&self) -> Result> { + let path = Path::new(PROC_MEMINFO_PATH); + // Need to read all of /proc/meminfo at once + // as we derive used memory based on a calculation + // using multiple lines + let contents = read_to_string(path)?; + Self::parse_meminfo_memory_stats(&contents) + } + + /// Parses the key in a /proc/meminfo line + /// + /// A key is the string terminated by the `:` character + /// In the following line, "MemTotal" would be parsed as the key + /// MemTotal: 365916 kB + fn parse_meminfo_key(meminfo_line: &str) -> IResult<&str, &str> { + terminated(alpha1, tag(":"))(meminfo_line) + } + + /// Parses the kilobyte value in a /proc/meminfo line + /// + /// This value is the kilobytes used by the corresponding + /// meminfo key. The value is a number terminated by " kB". + /// In the following line, 365916.0 would be parsed by + /// this function as the kB used by "MemTotal" + /// MemTotal: 365916 kB + fn parse_meminfo_kb(meminfo_line_suffix: &str) -> IResult<&str, f64> { + delimited(multispace1, double, tag(" kB"))(meminfo_line_suffix) + } + + /// Parses a full /proc/meminfo contents and returns + /// a vector of KeyedMetricReadings + fn parse_meminfo_memory_stats(meminfo: &str) -> Result> { + let mut stats = meminfo + .trim() + .lines() + .map(|line| -> Result<(&str, f64), String> { + let (suffix, key) = Self::parse_meminfo_key(line).map_err(|e| e.to_string())?; + let (_, kb) = Self::parse_meminfo_kb(suffix).map_err(|e| e.to_string())?; + // Use bytes as unit instead of KB + Ok((key, kb * 1024.0)) + }) + .filter_map(|result| result.ok()) + .collect::>(); + + // Use the same methodology as `free` to calculate used memory. + // + // For kernels 3.14 and greater: + // MemUsed = MemTotal - MemAvailable + // + // For kernels less than 3.14 (no MemAvailable): + // MemUsed = MemTotal - MemFree + // + // See below man page for more info: + // https://man7.org/linux/man-pages/man1/free.1.html + let total = stats + .remove("MemTotal") + .ok_or_else(|| eyre!("{} is missing required value MemTotal", PROC_MEMINFO_PATH))?; + let free = stats + .remove("MemFree") + .ok_or_else(|| eyre!("{} is missing required value MemFree", PROC_MEMINFO_PATH))?; + let available = stats.remove("MemAvailable").unwrap_or(free); + + let used = total - available; + + let used_key = MetricStringKey::from_str("memory/memory/used") + .map_err(|e| eyre!("Failed to construct MetricStringKey for used memory: {}", e))?; + let free_key = MetricStringKey::from_str("memory/memory/free") + .map_err(|e| eyre!("Failed to construct MetricStringKey for used memory: {}", e))?; + + Ok(vec![ + KeyedMetricReading::new_histogram(free_key, free), + KeyedMetricReading::new_histogram(used_key, used), + ]) + } +} + +impl SystemMetricFamilyCollector for MemoryMetricsCollector { + fn collect_metrics(&mut self) -> Result> { + self.get_memory_metrics() + } + + fn family_name(&self) -> &'static str { + MEMORY_METRIC_NAMESPACE + } +} + +#[cfg(test)] +mod test { + + use insta::{assert_json_snapshot, rounded_redaction, with_settings}; + use rstest::rstest; + + use super::MemoryMetricsCollector; + + #[rstest] + #[case("MemTotal: 365916 kB", "MemTotal", 365916.0)] + #[case("MemFree: 242276 kB", "MemFree", 242276.0)] + #[case("MemAvailable: 292088 kB", "MemAvailable", 292088.0)] + #[case("Buffers: 4544 kB", "Buffers", 4544.0)] + #[case("Cached: 52128 kB", "Cached", 52128.0)] + fn test_parse_meminfo_line( + #[case] proc_meminfo_line: &str, + #[case] expected_key: &str, + #[case] expected_value: f64, + ) { + let (suffix, key) = MemoryMetricsCollector::parse_meminfo_key(proc_meminfo_line).unwrap(); + let (_, kb) = MemoryMetricsCollector::parse_meminfo_kb(suffix).unwrap(); + + assert_eq!(key, expected_key); + assert_eq!(kb, expected_value); + } + + #[rstest] + fn test_get_memory_metrics() { + let meminfo = "MemTotal: 365916 kB +MemFree: 242276 kB +MemAvailable: 292088 kB +Buffers: 4544 kB +Cached: 52128 kB +SwapCached: 0 kB +Active: 21668 kB +Inactive: 51404 kB +Active(anon): 2312 kB +Inactive(anon): 25364 kB +Active(file): 19356 kB +Inactive(file): 26040 kB +Unevictable: 3072 kB +Mlocked: 0 kB +SwapTotal: 0 kB +SwapFree: 0 kB +Dirty: 0 kB +Writeback: 0 kB +AnonPages: 19488 kB +Mapped: 29668 kB +Shmem: 11264 kB +KReclaimable: 14028 kB + "; + + with_settings!({sort_maps => true}, { + assert_json_snapshot!( + MemoryMetricsCollector::parse_meminfo_memory_stats(meminfo).unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + }); + } + + #[rstest] + fn test_get_memory_metrics_no_memavailable() { + let meminfo = "MemTotal: 365916 kB +MemFree: 242276 kB +Buffers: 4544 kB +Cached: 52128 kB +SwapCached: 0 kB +Active: 21668 kB +Inactive: 51404 kB +Active(anon): 2312 kB +Inactive(anon): 25364 kB +Active(file): 19356 kB +Inactive(file): 26040 kB +Unevictable: 3072 kB +Mlocked: 0 kB +SwapTotal: 0 kB +SwapFree: 0 kB +Dirty: 0 kB +Writeback: 0 kB +AnonPages: 19488 kB +Mapped: 29668 kB +Shmem: 11264 kB +KReclaimable: 14028 kB + "; + + with_settings!({sort_maps => true}, { + assert_json_snapshot!( + MemoryMetricsCollector::parse_meminfo_memory_stats(meminfo).unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + }); + } + + #[rstest] + fn test_fail_to_parse_bad_meminfo_line() { + assert!(MemoryMetricsCollector::parse_meminfo_key("MemFree=1080kB").is_err()); + assert!(MemoryMetricsCollector::parse_meminfo_kb("1080 mB").is_err()); + } + + #[rstest] + fn test_fail_get_metrics_with_missing_required_lines() { + // MemFree is missing + let meminfo = "MemTotal: 365916 kB +MemAvailable: 292088 kB +Buffers: 4544 kB +Cached: 52128 kB +SwapCached: 0 kB +Active: 21668 kB +Inactive: 51404 kB +Active(anon): 2312 kB +Inactive(anon): 25364 kB +Active(file): 19356 kB +Inactive(file): 26040 kB +Unevictable: 3072 kB +Mlocked: 0 kB +SwapTotal: 0 kB +SwapFree: 0 kB +Dirty: 0 kB +Writeback: 0 kB +AnonPages: 19488 kB +Mapped: 29668 kB +Shmem: 11264 kB +KReclaimable: 14028 kB + "; + assert!(MemoryMetricsCollector::parse_meminfo_memory_stats(meminfo).is_err()); + } + + #[rstest] + fn test_fail_get_metrics_with_bad_fmt() { + // Not properly formatted with newlines between each key / kB pair + let meminfo = "MemTotal: 365916 kB MemFree: 242276 kB +Buffers: 4544 kB Cached: 52128 kB Shmem: 11264 kB + "; + assert!(MemoryMetricsCollector::parse_meminfo_memory_stats(meminfo).is_err()); + } +} diff --git a/memfaultd/src/metrics/system_metrics/mod.rs b/memfaultd/src/metrics/system_metrics/mod.rs new file mode 100644 index 0000000..c616928 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/mod.rs @@ -0,0 +1,180 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + collections::HashSet, + sync::{Arc, Mutex}, + thread::sleep, + time::{Duration, Instant}, +}; + +use eyre::Result; +use log::warn; + +use crate::{ + config::SystemMetricConfig, + metrics::{KeyedMetricReading, MetricReportManager}, + util::system::{bytes_per_page, clock_ticks_per_second}, +}; + +mod cpu; +use crate::metrics::system_metrics::cpu::{CpuMetricCollector, CPU_METRIC_NAMESPACE}; + +mod thermal; +use crate::metrics::system_metrics::thermal::{ThermalMetricsCollector, THERMAL_METRIC_NAMESPACE}; + +mod memory; +use crate::metrics::system_metrics::memory::{MemoryMetricsCollector, MEMORY_METRIC_NAMESPACE}; + +mod network_interfaces; +use crate::metrics::system_metrics::network_interfaces::{ + NetworkInterfaceMetricCollector, NetworkInterfaceMetricsConfig, + NETWORK_INTERFACE_METRIC_NAMESPACE, +}; + +mod processes; +use processes::{ProcessMetricsCollector, PROCESSES_METRIC_NAMESPACE}; + +mod disk_space; +use disk_space::{ + DiskSpaceMetricCollector, DiskSpaceMetricsConfig, NixStatvfs, DISKSPACE_METRIC_NAMESPACE, + DISKSPACE_METRIC_NAMESPACE_LEGACY, +}; + +use self::processes::ProcessMetricsConfig; + +pub const BUILTIN_SYSTEM_METRIC_NAMESPACES: &[&str; 7] = &[ + CPU_METRIC_NAMESPACE, + MEMORY_METRIC_NAMESPACE, + THERMAL_METRIC_NAMESPACE, + NETWORK_INTERFACE_METRIC_NAMESPACE, + PROCESSES_METRIC_NAMESPACE, + DISKSPACE_METRIC_NAMESPACE, + // Include in list of namespaces so that + // legacy collectd from the "df" plugin + // are still filtered out + DISKSPACE_METRIC_NAMESPACE_LEGACY, +]; + +pub trait SystemMetricFamilyCollector { + fn collect_metrics(&mut self) -> Result>; + fn family_name(&self) -> &'static str; +} + +pub struct SystemMetricsCollector { + metric_family_collectors: Vec>, +} + +impl SystemMetricsCollector { + pub fn new(system_metric_config: SystemMetricConfig) -> Self { + // CPU, Memory, and Thermal metrics are captured by default + let mut metric_family_collectors: Vec> = vec![ + Box::new(CpuMetricCollector::new()), + Box::new(MemoryMetricsCollector::new()), + Box::new(ThermalMetricsCollector::new()), + ]; + + // Check if process metrics have been manually configured + match system_metric_config.processes { + Some(processes) if !processes.is_empty() => { + metric_family_collectors.push(Box::new(ProcessMetricsCollector::::new( + ProcessMetricsConfig::Processes(processes), + clock_ticks_per_second() as f64 / 1000.0, + bytes_per_page() as f64, + ))) + } + // Monitoring no processes means this collector is disabled + Some(_empty_set) => {} + None => { + metric_family_collectors.push(Box::new(ProcessMetricsCollector::::new( + ProcessMetricsConfig::Auto, + clock_ticks_per_second() as f64 / 1000.0, + bytes_per_page() as f64, + ))) + } + }; + + // Check if disk space metrics have been manually configured + match system_metric_config.disk_space { + Some(disks) if !disks.is_empty() => { + metric_family_collectors.push(Box::new(DiskSpaceMetricCollector::new( + NixStatvfs::new(), + DiskSpaceMetricsConfig::Disks(disks), + ))) + } + // Monitoring no disks means this collector is disabled + Some(_empty_set) => {} + None => metric_family_collectors.push(Box::new(DiskSpaceMetricCollector::new( + NixStatvfs::new(), + DiskSpaceMetricsConfig::Auto, + ))), + }; + + // Check if network interface metrics have been manually configured + match system_metric_config.network_interfaces { + Some(interfaces) if !interfaces.is_empty() => metric_family_collectors.push(Box::new( + NetworkInterfaceMetricCollector::::new( + NetworkInterfaceMetricsConfig::Interfaces(interfaces), + ), + )), + // Monitoring no interfaces means this collector is disabled + Some(_empty_set) => {} + None => metric_family_collectors.push(Box::new(NetworkInterfaceMetricCollector::< + Instant, + >::new( + NetworkInterfaceMetricsConfig::Auto + ))), + }; + + Self { + metric_family_collectors, + } + } + + pub fn run( + &mut self, + metric_poll_duration: Duration, + metric_report_manager: Arc>, + ) { + loop { + for collector in self.metric_family_collectors.iter_mut() { + match collector.collect_metrics() { + Ok(readings) => { + for metric_reading in readings { + if let Err(e) = metric_report_manager + .lock() + .expect("Mutex poisoned") + .add_metric(metric_reading) + { + warn!( + "Couldn't add metric reading for family \"{}\": {:?}", + collector.family_name(), + e + ) + } + } + } + Err(e) => warn!( + "Failed to collect readings for family \"{}\": {}", + collector.family_name(), + e + ), + } + } + + sleep(metric_poll_duration); + } + } +} + +impl Default for SystemMetricsCollector { + fn default() -> Self { + Self::new(SystemMetricConfig { + enable: true, + poll_interval_seconds: Duration::from_secs(10), + processes: Some(HashSet::from_iter(["memfaultd".to_string()])), + disk_space: None, + network_interfaces: None, + }) + } +} diff --git a/memfaultd/src/metrics/system_metrics/network_interfaces.rs b/memfaultd/src/metrics/system_metrics/network_interfaces.rs new file mode 100644 index 0000000..77ace6b --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/network_interfaces.rs @@ -0,0 +1,489 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Collect Network Interface metric readings from /proc/net/dev +//! +//! Example /proc/net/dev output: +//! Inter-| Receive | Transmit +//! face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed +//! lo: 2707 25 0 0 0 0 0 0 2707 25 0 0 0 0 0 0 +//! eth0: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +//! wlan0: 10919408 8592 0 0 0 0 0 0 543095 4066 0 0 0 0 0 0 +//! +//! Kernel docs: +//! https://docs.kernel.org/filesystems/proc.html#networking-info-in-proc-net +use std::collections::{HashMap, HashSet}; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::iter::zip; +use std::path::Path; +use std::str::FromStr; +use std::time::Duration; + +use chrono::Utc; +use nom::bytes::complete::tag; +use nom::character::complete::{alphanumeric1, multispace0, multispace1, u64}; +use nom::sequence::terminated; +use nom::{ + multi::count, + sequence::{pair, preceded}, + IResult, +}; + +use crate::{ + metrics::{ + system_metrics::SystemMetricFamilyCollector, KeyedMetricReading, MetricReading, + MetricStringKey, + }, + util::time_measure::TimeMeasure, +}; + +use eyre::{eyre, ErrReport, Result}; + +const PROC_NET_DEV_PATH: &str = "/proc/net/dev"; +pub const NETWORK_INTERFACE_METRIC_NAMESPACE: &str = "interface"; + +// Metric keys that are currently captured and reported +// by memfaultd. +// There is a lot of information in /proc/net/dev +// and the intention with this list is to use it +// to filter out the values read from it so that +// only the high-signal and widely-applicable metrics remain. +const NETWORK_INTERFACE_METRIC_KEYS: &[&str; 8] = &[ + "bytes_per_second/rx", + "packets_per_second/rx", + "errors_per_second/rx", + "dropped_per_second/rx", + "bytes_per_second/tx", + "packets_per_second/tx", + "errors_per_second/tx", + "dropped_per_second/tx", +]; + +pub enum NetworkInterfaceMetricsConfig { + Auto, + Interfaces(HashSet), +} + +pub struct NetworkInterfaceMetricCollector { + config: NetworkInterfaceMetricsConfig, + previous_readings_by_interface: HashMap>, +} + +#[derive(Clone)] +pub struct ProcNetDevReading { + stats: Vec, + reading_time: T, +} + +impl NetworkInterfaceMetricCollector +where + T: TimeMeasure + Copy + Ord + std::ops::Add + Send + Sync + 'static, +{ + pub fn new(config: NetworkInterfaceMetricsConfig) -> Self { + Self { + config, + previous_readings_by_interface: HashMap::new(), + } + } + + fn interface_is_monitored(&self, interface: &str) -> bool { + match &self.config { + // Ignore loopback interfaces + NetworkInterfaceMetricsConfig::Auto => !interface.starts_with("lo"), + NetworkInterfaceMetricsConfig::Interfaces(configured_interfaces) => { + configured_interfaces.contains(interface) + } + } + } + + pub fn get_network_interface_metrics(&mut self) -> Result> { + // Track if any lines in /proc/net/dev are parse-able + // so we can alert user if none are + let mut no_parseable_lines = true; + + let path = Path::new(PROC_NET_DEV_PATH); + + let file = File::open(path)?; + let reader = BufReader::new(file); + + let mut net_metric_readings = vec![]; + + for line in reader.lines() { + // Discard errors - the assumption here is that we are only parsing + // lines that follow the specified format and expect other lines in the file to error + if let Ok((interface_id, net_stats)) = Self::parse_proc_net_dev_line(line?.trim()) { + no_parseable_lines = false; + + // Ignore unmonitored interfaces + if self.interface_is_monitored(&interface_id) { + if let Ok(Some(mut readings)) = self.calculate_network_metrics( + interface_id.to_string(), + ProcNetDevReading { + stats: net_stats, + reading_time: T::now(), + }, + ) { + net_metric_readings.append(&mut readings); + } + } + } + } + + // Check if we were able to parse at least one CPU metric reading + if no_parseable_lines { + Err(eyre!( + "No network metrics were collected from {} - is it a properly formatted /proc/net/dev file?", + PROC_NET_DEV_PATH + )) + } else { + Ok(net_metric_readings) + } + } + + /// Parse a network interface name from a line of /proc/net/dev + /// The network interface may be preceded by whitespace and will + /// always be terminated with a ':' + /// in a line that is followed by 16 number values + fn parse_net_if(input: &str) -> IResult<&str, &str> { + terminated(preceded(multispace0, alphanumeric1), tag(":"))(input) + } + + /// Parse the CPU stats from the suffix of a /proc/net/dev line following the interface ID + /// + /// The first 8 values track RX traffic on the interface. The latter 8 track TX traffic. + fn parse_interface_stats(input: &str) -> IResult<&str, Vec> { + count(preceded(multispace1, u64), 16)(input) + } + + /// Parse the output of a line of /proc/net/dev, returning + /// a pair of the network interface that the parsed line corresponds + /// to and the first 7 floats listed for it + /// + /// The first 8 values track RX traffic on the interface since boot with + /// following names (in order): + /// "bytes", "packets", "errs", "drop", "fifo", "frame", "compressed" "multicast" + /// The latter 8 track TX traffic, with the following names: + /// "bytes", "packets", "errs", "drop", "fifo", "colls", "carrier", "compressed" + /// + /// Important!!: The rest of this module assumes this is the ordering of values + /// in the /proc/net/dev file + fn parse_proc_net_dev_line(line: &str) -> Result<(String, Vec)> { + let (_remaining, (interface_id, net_stats)) = + pair(Self::parse_net_if, Self::parse_interface_stats)(line) + .map_err(|e| eyre!("Failed to parse /proc/net/dev line: {}", e))?; + Ok((interface_id.to_string(), net_stats)) + } + + /// We need to account for potential rollovers in the + /// /proc/net/dev counters, handled by this function + fn counter_delta_with_overflow(current: u64, previous: u64) -> u64 { + // The only time a counter's value would be less + // that its previous value is if it rolled over + // due to overflow - drop these readings that overlap + // with an overflow + if current < previous { + // Need to detect if the counter rolled over at u32::MAX or u64::MAX + current + + ((if previous > u32::MAX as u64 { + u64::MAX + } else { + u32::MAX as u64 + }) - previous) + } else { + current - previous + } + } + + /// Calculates network metrics + fn calculate_network_metrics( + &mut self, + interface: String, + current_reading: ProcNetDevReading, + ) -> Result>> { + // Check to make sure there was a previous reading to calculate a delta with + if let Some(ProcNetDevReading { + stats: previous_net_stats, + reading_time: previous_reading_time, + }) = self + .previous_readings_by_interface + .insert(interface.clone(), current_reading.clone()) + { + let current_period_rates = + current_reading + .stats + .iter() + .zip(previous_net_stats) + .map(|(current, previous)| { + Self::counter_delta_with_overflow(*current, previous) as f64 + / (current_reading + .reading_time + .since(&previous_reading_time) + .as_secs_f64()) + }); + + let net_keys_with_stats = zip( + [ + "bytes_per_second/rx", + "packets_per_second/rx", + "errors_per_second/rx", + "dropped_per_second/rx", + "fifo/rx", + "frame/rx", + "compressed/rx", + "multicast/rx", + "bytes_per_second/tx", + "packets_per_second/tx", + "errors_per_second/tx", + "dropped_per_second/tx", + "fifo/tx", + "colls/tx", + "carrier/tx", + "compressed/tx", + ], + current_period_rates, + ) + // Filter out metrics we don't want memfaultd to include in reports like fifo and colls + .filter(|(key, _)| NETWORK_INTERFACE_METRIC_KEYS.contains(key)) + .collect::>(); + + let timestamp = Utc::now(); + let readings = net_keys_with_stats + .iter() + .map(|(key, value)| -> Result { + Ok(KeyedMetricReading::new( + MetricStringKey::from_str(&format!( + "{}/{}/{}", + NETWORK_INTERFACE_METRIC_NAMESPACE, interface, key + )) + .map_err(|e| eyre!(e))?, + MetricReading::Histogram { + value: *value, + timestamp, + }, + )) + }) + .collect::>>(); + match readings { + Ok(readings) => Ok(Some(readings)), + Err(e) => Err(e), + } + } else { + Ok(None) + } + } +} + +impl SystemMetricFamilyCollector for NetworkInterfaceMetricCollector +where + T: TimeMeasure + Copy + Ord + std::ops::Add + Send + Sync + 'static, +{ + fn family_name(&self) -> &'static str { + NETWORK_INTERFACE_METRIC_NAMESPACE + } + + fn collect_metrics(&mut self) -> Result> { + self.get_network_interface_metrics() + } +} + +#[cfg(test)] +mod test { + + use insta::{assert_json_snapshot, rounded_redaction, with_settings}; + use rstest::rstest; + + use super::*; + use crate::test_utils::TestInstant; + + #[rstest] + #[case(" eth0: 2707 25 0 0 0 0 0 0 2707 25 0 0 0 0 0 0", "eth0")] + #[case("wlan1: 2707 25 0 0 0 0 0 0 2707 25 0 0 0 0 0 0", "wlan1")] + fn test_parse_netdev_line(#[case] proc_net_dev_line: &str, #[case] test_name: &str) { + assert_json_snapshot!(test_name, + NetworkInterfaceMetricCollector::::parse_proc_net_dev_line(proc_net_dev_line).unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + } + #[rstest] + // Missing a colon after wlan0 + #[case("wlan0 2707 25 0 0 0 0 0 0 2707 25 0 0 0 0 0 0")] + // Only 15 stat values instead of 16 + #[case("wlan0: 2707 0 0 0 0 0 0 2707 25 0 0 0 0 0 0")] + fn test_fails_on_invalid_proc_net_dev_line(#[case] proc_net_dev_line: &str) { + assert!( + NetworkInterfaceMetricCollector::::parse_proc_net_dev_line( + proc_net_dev_line + ) + .is_err() + ) + } + + #[rstest] + #[case( + " eth0: 1000 25 0 0 0 0 0 0 2000 25 0 0 0 0 0 0", + " eth0: 2500 80 10 10 0 0 0 0 3000 50 0 0 0 0 0 0", + " eth0: 5000 100 15 15 0 0 0 0 5000 75 20 20 0 0 0 0", + "basic_delta" + )] + #[case( + " eth0: 4294967293 25 0 0 0 0 0 0 2000 25 0 0 0 0 0 0", + " eth0: 2498 80 10 10 0 0 0 0 3000 50 0 0 0 0 0 0", + " eth0: 5000 100 15 15 0 0 0 0 5000 75 20 20 0 0 0 0", + "with_overflow" + )] + fn test_net_if_metric_collector_calcs( + #[case] proc_net_dev_line_a: &str, + #[case] proc_net_dev_line_b: &str, + #[case] proc_net_dev_line_c: &str, + #[case] test_name: &str, + ) { + let mut net_metric_collector = NetworkInterfaceMetricCollector::::new( + NetworkInterfaceMetricsConfig::Interfaces(HashSet::from_iter(["eth0".to_string()])), + ); + + let (net_if, stats) = + NetworkInterfaceMetricCollector::::parse_proc_net_dev_line( + proc_net_dev_line_a, + ) + .unwrap(); + let reading_a = ProcNetDevReading { + stats, + reading_time: TestInstant::now(), + }; + let result_a = net_metric_collector.calculate_network_metrics(net_if, reading_a); + matches!(result_a, Ok(None)); + + TestInstant::sleep(Duration::from_secs(10)); + + let (net_if, stats) = + NetworkInterfaceMetricCollector::::parse_proc_net_dev_line( + proc_net_dev_line_b, + ) + .unwrap(); + let reading_b = ProcNetDevReading { + stats, + reading_time: TestInstant::now(), + }; + let result_b = net_metric_collector.calculate_network_metrics(net_if, reading_b); + + assert!(result_b.is_ok()); + + with_settings!({sort_maps => true}, { + assert_json_snapshot!(format!("{}_{}", test_name, "a_b_metrics"), + result_b.unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + }); + + TestInstant::sleep(Duration::from_secs(30)); + + let (net_if, stats) = + NetworkInterfaceMetricCollector::::parse_proc_net_dev_line( + proc_net_dev_line_c, + ) + .unwrap(); + let reading_c = ProcNetDevReading { + stats, + reading_time: TestInstant::now(), + }; + let result_c = net_metric_collector.calculate_network_metrics(net_if, reading_c); + + assert!(result_c.is_ok()); + + with_settings!({sort_maps => true}, { + assert_json_snapshot!(format!("{}_{}", test_name, "b_c_metrics"), + result_c.unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + }); + } + + #[rstest] + #[case( + " eth0: 1000 25 0 0 0 0 0 0 2000 25 0 0 0 0 0 0", + " eth1: 2500 80 10 10 0 0 0 0 3000 50 0 0 0 0 0 0", + " eth0: 5000 100 15 15 0 0 0 0 5000 75 20 20 0 0 0 0", + true, + "different_interfaces" + )] + fn test_net_if_metric_collector_different_if( + #[case] proc_net_dev_line_a: &str, + #[case] proc_net_dev_line_b: &str, + #[case] proc_net_dev_line_c: &str, + #[case] use_auto_config: bool, + #[case] test_name: &str, + ) { + let mut net_metric_collector = + NetworkInterfaceMetricCollector::::new(if use_auto_config { + NetworkInterfaceMetricsConfig::Auto + } else { + NetworkInterfaceMetricsConfig::Interfaces(HashSet::from_iter(["eth1".to_string()])) + }); + + let (net_if, stats) = + NetworkInterfaceMetricCollector::::parse_proc_net_dev_line( + proc_net_dev_line_a, + ) + .unwrap(); + let reading_a = ProcNetDevReading { + stats, + reading_time: TestInstant::now(), + }; + let result_a = net_metric_collector.calculate_network_metrics(net_if, reading_a); + matches!(result_a, Ok(None)); + + TestInstant::sleep(Duration::from_secs(10)); + + let (net_if, stats) = + NetworkInterfaceMetricCollector::::parse_proc_net_dev_line( + proc_net_dev_line_b, + ) + .unwrap(); + let reading_b = ProcNetDevReading { + stats, + reading_time: TestInstant::now(), + }; + let result_b = net_metric_collector.calculate_network_metrics(net_if, reading_b); + matches!(result_b, Ok(None)); + + TestInstant::sleep(Duration::from_secs(30)); + + let (net_if, stats) = + NetworkInterfaceMetricCollector::::parse_proc_net_dev_line( + proc_net_dev_line_c, + ) + .unwrap(); + let reading_c = ProcNetDevReading { + stats, + reading_time: TestInstant::now(), + }; + let result_c = net_metric_collector.calculate_network_metrics(net_if, reading_c); + + assert!(result_c.is_ok()); + + // 2 readings are required to calculate metrics (since they are rates), + // so we should only get actual metrics after processing reading_c + // (which is the second eth0 reading) + with_settings!({sort_maps => true}, { + assert_json_snapshot!(format!("{}_{}", test_name, "a_c_metrics"), + result_c.unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + }); + } + #[rstest] + #[case(vec!["eth0".to_string(), "wlan1".to_string()], "eth1", false)] + #[case(vec!["eth0".to_string(), "wlan1".to_string()], "eth0", true)] + #[case(vec!["eth0".to_string(), "wlan1".to_string()], "enp0s10", false)] + #[case(vec!["eth0".to_string(), "wlan1".to_string()], "wlan1", true)] + fn test_interface_is_monitored( + #[case] monitored_interfaces: Vec, + #[case] interface: &str, + #[case] should_be_monitored: bool, + ) { + let net_metric_collector = NetworkInterfaceMetricCollector::::new( + NetworkInterfaceMetricsConfig::Interfaces(HashSet::from_iter(monitored_interfaces)), + ); + assert_eq!( + net_metric_collector.interface_is_monitored(interface), + should_be_monitored + ) + } +} diff --git a/memfaultd/src/metrics/system_metrics/processes.rs b/memfaultd/src/metrics/system_metrics/processes.rs new file mode 100644 index 0000000..06a7eca --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/processes.rs @@ -0,0 +1,526 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Collect per-process metrics from /proc//stat +//! +//! Collects process-level metrics using the +//! /proc//stat files for processes whose +//! process name matches an item in the user-specified +//! list of processes to monitor. +//! +//! Example /proc//stat contents: +//! +//! 55270 (memfaultd) S 1 55270 55270 0 -1 4194368 825 0 0 0 155 102 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0 +//! +//! Further documentation of the /proc//stat file +//! can be found at: +//! https://man7.org/linux/man-pages/man5/proc_pid_stat.5.html +use std::{ + collections::HashMap, + collections::HashSet, + fs::{read_dir, read_to_string}, +}; + +use eyre::{eyre, Result}; +use log::warn; +use nom::character::complete::{alpha1, multispace1}; +use nom::{ + bytes::complete::{is_not, tag}, + character::complete::{space1, u64}, + multi::count, + number::complete::double, + sequence::{delimited, preceded, terminated}, + IResult, +}; + +use crate::metrics::{system_metrics::SystemMetricFamilyCollector, KeyedMetricReading}; +use crate::util::time_measure::TimeMeasure; + +const PROC_DIR: &str = "/proc/"; +pub const PROCESSES_METRIC_NAMESPACE: &str = "processes"; + +pub enum ProcessMetricsConfig { + Auto, + Processes(HashSet), +} + +#[derive(Clone, Debug)] +struct ProcessReading { + pid: u64, + name: String, + cputime_user: f64, + cputime_system: f64, + num_threads: f64, + rss: f64, + vm: f64, + pagefaults_major: f64, + pagefaults_minor: f64, + reading_time: T, +} + +pub struct ProcessMetricsCollector { + config: ProcessMetricsConfig, + processes: HashMap>, + clock_ticks_per_ms: f64, + bytes_per_page: f64, +} + +impl ProcessMetricsCollector +where + T: TimeMeasure + Copy + Send + Sync + 'static, +{ + pub fn new(config: ProcessMetricsConfig, clock_ticks_per_ms: f64, bytes_per_page: f64) -> Self { + Self { + config, + processes: HashMap::new(), + clock_ticks_per_ms, + bytes_per_page, + } + } + + fn process_is_monitored(&self, process_name: &str) -> bool { + match &self.config { + ProcessMetricsConfig::Auto => process_name == "memfaultd", + ProcessMetricsConfig::Processes(ps) => ps.contains(process_name), + } + } + + /// Parses the PID at the start of the /proc//stat + /// + /// Example input: + /// 55270 (memfaultd) S 1 55270 55270 0 -1 4194368 825 0 0 0 100 50 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + /// Example output: + /// 55270 + fn parse_pid(proc_pid_stat_line: &str) -> IResult<&str, u64> { + terminated(u64, space1)(proc_pid_stat_line) + } + + /// Parses the process name which follows the PID, delimited by ( and ), in /proc//stat + /// + /// Note this snippet from the documentation on /proc//stat: + /// + /// (2) comm %s + /// The filename of the executable, in parentheses. + /// Strings longer than TASK_COMM_LEN (16) characters + /// (including the terminating null byte) are silently + /// truncated. This is visible whether or not the + /// executable is swapped out. + /// + /// So for executables with names longer than 16 characters, the + /// name will be truncated to just the first 16 characters. + /// + /// Example input: + /// (memfaultd) S 1 55270 55270 0 -1 4194368 825 0 0 0 100 50 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + /// Example output: + /// memfaultd + fn parse_name(proc_pid_stat_line: &str) -> IResult<&str, &str> { + // This will break if there's a ')' in the process name - that seems unlikely + // enough to leave as-is for now + delimited(tag("("), is_not(")"), tag(")"))(proc_pid_stat_line) + } + + /// Parses the process state which follows the PID, delimited by ( and ) + /// + /// Example input: + /// S 1 55270 55270 0 -1 4194368 825 0 0 0 100 50 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + /// Example output: + /// S + fn parse_state(proc_pid_stat_line: &str) -> IResult<&str, &str> { + preceded(space1, alpha1)(proc_pid_stat_line) + } + + /// Parses the process state which follows the PID, delimited by ( and ) + /// + /// The following values from the resulting Vector are currently used: + /// - minfault: The number of minor faults the process has made + /// which have not required loading a memory page from + /// disk. (index 6) + /// - majfault: The number of major faults the process has made + /// which have required loading a memory page from + /// disk. (index 8) + /// - utime: Amount of time that this process has been scheduled + /// in user mode, measured in clock ticks (index 10) + /// - stime: Amount of time that this process has been scheduled + /// in kernel mode, measured in clock ticks (index 11) + /// - num_threads: Number of threads in the corresponding process (Index 16) + /// - vsize: Virtual memory size in bytes for the process (index 19) + /// - rss: number of pages the process has in real memory (index 20) + /// + /// Example input: + /// 1 55270 55270 0 -1 4194368 825 0 0 0 100 50 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + /// Example output: + /// vec![1 55270 55270 0 -1 4194368 825 0 0 0 100 50 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0] + fn parse_stats(proc_pid_stat_line: &str) -> IResult<&str, Vec> { + // There are more than 29 values in a line but we don't use any past the 29th + // (kstkesp) so don't parse past that + count(preceded(multispace1, double), 29)(proc_pid_stat_line) + } + + /// Parses the full contents of a /proc//stat file into a ProcessReading + /// + /// If the process name is not in the set of configured processes to monitor, this + /// function will stop parsing and return Ok(None) to avoid doing unnecessary work. + fn parse_process_stat(&self, proc_pid_stat_line: &str) -> Result>> { + let (after_pid, pid) = Self::parse_pid(proc_pid_stat_line) + .map_err(|_e| eyre!("Failed to parse PID for process"))?; + let (after_name, name) = + Self::parse_name(after_pid).map_err(|_e| eyre!("Failed to parse process name"))?; + + // Don't bother continuing to parse processes that aren't monitored + if self.process_is_monitored(name) { + let (after_state, _) = Self::parse_state(after_name) + .map_err(|_e| eyre!("Failed to parse process state for {}", name))?; + let (_, stats) = Self::parse_stats(after_state) + .map_err(|_e| eyre!("Failed to parse process stats for {}", name))?; + + let pagefaults_minor = *stats + .get(6) + .ok_or(eyre!("Failed to read pagefaults_minor"))?; + let pagefaults_major = *stats + .get(8) + .ok_or(eyre!("Failed to read pagefaults_major"))?; + + let cputime_user = *stats.get(10).ok_or(eyre!("Failed to read cputime_user"))?; + let cputime_system = *stats.get(11).ok_or(eyre!("Failed to read cputime_user"))?; + + let num_threads = *stats.get(16).ok_or(eyre!("Failed to read num_threads"))?; + + let vm = *stats.get(19).ok_or(eyre!("Failed to read vm"))?; + + // RSS is provided as the number of pages used by the process, we need + // to multiply by the system-specific bytes per page to get a value in bytes + let rss = *stats.get(20).ok_or(eyre!("Failed to read rss"))? * self.bytes_per_page; + + Ok(Some(ProcessReading { + pid, + name: name.to_string(), + cputime_user, + cputime_system, + num_threads, + rss, + pagefaults_major, + pagefaults_minor, + vm, + reading_time: T::now(), + })) + } else { + Ok(None) + } + } + + fn calculate_metric_readings( + &self, + previous: ProcessReading, + current: ProcessReading, + ) -> Result> { + let rss_reading = KeyedMetricReading::new_histogram( + format!("processes/{}/rss_bytes", current.name) + .as_str() + .parse() + .map_err(|e| eyre!("Couldn't parse metric key: {}", e))?, + current.rss, + ); + + let vm_reading = KeyedMetricReading::new_histogram( + format!("processes/{}/vm_bytes", current.name) + .as_str() + .parse() + .map_err(|e| eyre!("Couldn't parse metric key: {}", e))?, + current.vm, + ); + + let num_threads_reading = KeyedMetricReading::new_histogram( + format!("processes/{}/num_threads", current.name) + .as_str() + .parse() + .map_err(|e| eyre!("Couldn't parse metric key: {}", e))?, + current.num_threads, + ); + + // The values from /proc//stat are monotonic counters of jiffies since the + // process was started. We need to calculate the difference since the last reading, + // divide by the amount of jiffies in a millisecond (to get milliseconds spent in + // the given state), then divide by the total number of milliseconds since the + // previous reading in order to give us the % of time this process caused + // the CPU to spend in the user and system states. + let cputime_user_pct = (((current.cputime_user - previous.cputime_user) + / self.clock_ticks_per_ms) + / (current + .reading_time + .since(&previous.reading_time) + .as_millis() as f64)) + * 100.0; + + let cputime_sys_pct = (((current.cputime_system - previous.cputime_system) + / self.clock_ticks_per_ms) + / (current + .reading_time + .since(&previous.reading_time) + .as_millis() as f64)) + * 100.0; + + let utime_reading = KeyedMetricReading::new_histogram( + format!("processes/{}/cpu/percent/user", current.name) + .as_str() + .parse() + .map_err(|e| eyre!("Couldn't parse metric key: {}", e))?, + cputime_user_pct, + ); + let stime_reading = KeyedMetricReading::new_histogram( + format!("processes/{}/cpu/percent/system", current.name) + .as_str() + .parse() + .map_err(|e| eyre!("Couldn't parse metric key: {}", e))?, + cputime_sys_pct, + ); + + let pagefaults_minor_reading = KeyedMetricReading::new_histogram( + format!("processes/{}/pagefaults/minor", current.name) + .as_str() + .parse() + .map_err(|e| eyre!("Couldn't parse metric key: {}", e))?, + current.pagefaults_minor - previous.pagefaults_minor, + ); + let pagefaults_major_reading = KeyedMetricReading::new_histogram( + format!("processes/{}/pagefaults/major", current.name) + .as_str() + .parse() + .map_err(|e| eyre!("Couldn't parse metric key: {}", e))?, + current.pagefaults_major - previous.pagefaults_major, + ); + + Ok(vec![ + rss_reading, + vm_reading, + num_threads_reading, + stime_reading, + utime_reading, + pagefaults_minor_reading, + pagefaults_major_reading, + ]) + } + + // To facilitate unit testing, make the process directory path an arg + fn read_process_metrics_from_dir(&mut self, proc_dir: &str) -> Result> { + let process_readings: Vec<_> = read_dir(proc_dir)? + .filter_map(|entry| entry.map(|e| e.path()).ok()) + // Filter out non-numeric directories (since these won't be PIDs) + .filter(|path| match path.file_name() { + Some(p) => p.to_string_lossy().chars().all(|c| c.is_numeric()), + None => false, + }) + // Append "/stat" to the path since this is the file we want to read + // for a given PID's directory + .filter_map(|path| read_to_string(path.join("stat")).ok()) + .filter_map(|proc_pid_stat_contents| { + self.parse_process_stat(&proc_pid_stat_contents).ok() + }) + .flatten() + .collect(); + + let mut process_metric_readings = vec![]; + for current_reading in process_readings { + // A previous reading is required to calculate CPU time %s, as + // /proc//stat only has monotonic counters that track + // time spent in CPU states. Without a delta over a known period + // of time, we can't know if the contents of the counter are relevant + // to the current sampling window or not. + // + // For simplicity, only return any metrics for a PID when there + // is a previous reading for that PID in the `processes` map. + if let Some(previous_reading) = self + .processes + .insert(current_reading.pid, current_reading.clone()) + { + match self.calculate_metric_readings(previous_reading, current_reading.clone()) { + Ok(metric_readings) => process_metric_readings.extend(metric_readings), + Err(e) => warn!( + "Couldn't calculate metric readings for process {} (PID {}): {}", + current_reading.name, current_reading.pid, e + ), + } + } + } + + Ok(process_metric_readings) + } + + pub fn get_process_metrics(&mut self) -> Result> { + self.read_process_metrics_from_dir(PROC_DIR) + } +} + +impl SystemMetricFamilyCollector for ProcessMetricsCollector +where + T: TimeMeasure + Copy + Send + Sync + 'static, +{ + fn family_name(&self) -> &'static str { + PROCESSES_METRIC_NAMESPACE + } + + fn collect_metrics(&mut self) -> Result> { + self.get_process_metrics() + } +} + +#[cfg(test)] +mod tests { + use std::{ + fs::{create_dir, remove_file, File}, + io::Write, + time::Duration, + }; + use tempfile::tempdir; + + use insta::{assert_json_snapshot, rounded_redaction, with_settings}; + use rstest::rstest; + + use super::*; + use crate::test_utils::TestInstant; + + #[rstest] + fn test_parse_single_line() { + let collector = ProcessMetricsCollector::::new( + ProcessMetricsConfig::Processes(HashSet::from_iter(["memfaultd".to_string()])), + 100.0, + 4096.0, + ); + + let line = "55270 (memfaultd) S 1 55270 55270 0 -1 4194368 825 0 0 0 155 102 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0"; + assert!( + ProcessMetricsCollector::::parse_process_stat(&collector, line).is_ok() + ); + } + + #[rstest] + #[case( + "55270 (memfaultd) S 1 55270 55270 0 -1 4194368 825 0 0 0 100 50 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + "55270 (memfaultd) S 1 55270 55270 0 -1 4194368 875 0 10 0 1100 550 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + "simple_cpu_delta", + )] + fn test_collect_metrics(#[case] line1: &str, #[case] line2: &str, #[case] test_name: &str) { + let collector = ProcessMetricsCollector::::new( + ProcessMetricsConfig::Processes(HashSet::from_iter(["memfaultd".to_string()])), + 100.0, + 4096.0, + ); + + let first_reading = + ProcessMetricsCollector::::parse_process_stat(&collector, line1) + .unwrap() + .unwrap(); + + TestInstant::sleep(Duration::from_secs(10)); + + let second_reading = + ProcessMetricsCollector::::parse_process_stat(&collector, line2) + .unwrap() + .unwrap(); + + let process_metric_readings = + collector.calculate_metric_readings(first_reading, second_reading); + with_settings!({sort_maps => true}, { + assert_json_snapshot!(format!("{}_{}", test_name, "metrics"), + process_metric_readings.unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + }); + } + + #[rstest] + #[case( + "55270 (memfaultd) S 1 55270 55270 0 -1 4194368 825 0 0 0 100 50 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + "24071 (systemd) S 1 24071 24071 0 -1 4194560 1580 2275 0 0 12 2 0 1 20 0 1 0 1465472 19828736 2784 18446744073709551615 1 1 0 0 0 0 671173123 4096 0 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + "55270 (memfaultd) S 1 55270 55270 0 -1 4194368 845 0 16 0 1100 550 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + "24071 (systemd) S 1 24071 24071 0 -1 4194560 1580 2275 0 0 100 30 0 1 20 0 1 0 1465472 19828736 2784 18446744073709551615 1 1 0 0 0 0 671173123 4096 0 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + false, + )] + #[case( + "55270 (memfaultd) S 1 55270 55270 0 -1 4194368 825 0 0 0 100 50 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + "24071 (systemd) S 1 24071 24071 0 -1 4194560 1580 2275 0 0 12 2 0 1 20 0 1 0 1465472 19828736 2784 18446744073709551615 1 1 0 0 0 0 671173123 4096 0 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + "55270 (memfaultd) S 1 55270 55270 0 -1 4194368 825 0 0 0 1100 550 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + "24071 (systemd) S 1 24071 24071 0 -1 4194560 1580 2275 0 0 100 30 0 1 20 0 1 0 1465472 19828736 2784 18446744073709551615 1 1 0 0 0 0 671173123 4096 0 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + true, + )] + fn test_process_stats_from_proc( + #[case] process_a_sample_1: &str, + #[case] process_b_sample_1: &str, + #[case] process_a_sample_2: &str, + #[case] process_b_sample_2: &str, + #[case] use_auto: bool, + ) { + let mut collector = if use_auto { + ProcessMetricsCollector::::new(ProcessMetricsConfig::Auto, 100.0, 4096.0) + } else { + // If auto is not used, the configuration should capture metrics from both processes + ProcessMetricsCollector::::new( + ProcessMetricsConfig::Processes(HashSet::from_iter([ + "memfaultd".to_string(), + "systemd".to_string(), + ])), + 100.0, + 4096.0, + ) + }; + + // Create a temporary directory. + let dir = tempdir().unwrap(); + + let temp_proc_dir = dir.path().join("temp_proc"); + create_dir(&temp_proc_dir).unwrap(); + + // Set up /proc//stat files for first sample for both + // memfaultd and systemd + let process_a_dir = temp_proc_dir.join("55270"); + create_dir(&process_a_dir).unwrap(); + + let process_a_path = process_a_dir.join("stat"); + let mut process_a_file = File::create(process_a_path.clone()).unwrap(); + + let process_b_dir = temp_proc_dir.join("24071"); + create_dir(&process_b_dir).unwrap(); + let process_b_path = process_b_dir.join("stat"); + let mut process_b_file = File::create(process_b_path.clone()).unwrap(); + + writeln!(process_a_file, "{}", process_a_sample_1).unwrap(); + writeln!(process_b_file, "{}", process_b_sample_1).unwrap(); + + // Read /proc//stat files for second sample for both + // memfaultd and systemd + let process_a_path = temp_proc_dir.join("55270").join("stat"); + let mut process_a_file = File::create(process_a_path).unwrap(); + + let process_b_path = temp_proc_dir.join("24071").join("stat"); + let mut process_b_file = File::create(process_b_path).unwrap(); + + writeln!(process_a_file, "{}", process_a_sample_2).unwrap(); + writeln!(process_b_file, "{}", process_b_sample_2).unwrap(); + + // Read /proc/ "[timestamp]", "[].value.**.value" => rounded_redaction(5)}); + + // Delete the temporary directory. + dir.close().unwrap(); + } +} diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__basic_delta_a_b_metrics.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__basic_delta_a_b_metrics.snap new file mode 100644 index 0000000..0cda315 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__basic_delta_a_b_metrics.snap @@ -0,0 +1,69 @@ +--- +source: memfaultd/src/metrics/system_metrics/cpu.rs +expression: result_b.unwrap() +--- +[ + { + "name": "cpu/cpu/percent/user", + "value": { + "Histogram": { + "value": 96.15385, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/nice", + "value": { + "Histogram": { + "value": 2.88462, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/system", + "value": { + "Histogram": { + "value": 0.76923, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/idle", + "value": { + "Histogram": { + "value": 0.19231, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/iowait", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/irq", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/softirq", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__basic_delta_b_c_metrics.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__basic_delta_b_c_metrics.snap new file mode 100644 index 0000000..9e65a0a --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__basic_delta_b_c_metrics.snap @@ -0,0 +1,69 @@ +--- +source: memfaultd/src/metrics/system_metrics/cpu.rs +expression: result_c.unwrap() +--- +[ + { + "name": "cpu/cpu/percent/user", + "value": { + "Histogram": { + "value": 18.58736, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/nice", + "value": { + "Histogram": { + "value": 66.9145, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/system", + "value": { + "Histogram": { + "value": 13.3829, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/idle", + "value": { + "Histogram": { + "value": 0.74349, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/iowait", + "value": { + "Histogram": { + "value": 0.37175, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/irq", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/softirq", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__different_cores_a_c_metrics.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__different_cores_a_c_metrics.snap new file mode 100644 index 0000000..13dd9ed --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__different_cores_a_c_metrics.snap @@ -0,0 +1,69 @@ +--- +source: memfaultd/src/metrics/system_metrics/cpu.rs +expression: result_c.unwrap() +--- +[ + { + "name": "cpu/cpu1/percent/user", + "value": { + "Histogram": { + "value": 70.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu1/percent/nice", + "value": { + "Histogram": { + "value": 10.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu1/percent/system", + "value": { + "Histogram": { + "value": 10.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu1/percent/idle", + "value": { + "Histogram": { + "value": 2.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu1/percent/iowait", + "value": { + "Histogram": { + "value": 5.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu1/percent/irq", + "value": { + "Histogram": { + "value": 3.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu1/percent/softirq", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__test_basic_line.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__test_basic_line.snap new file mode 100644 index 0000000..bf5cca9 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__test_basic_line.snap @@ -0,0 +1,16 @@ +--- +source: memfaultd/src/metrics/system_metrics/cpu.rs +expression: "CpuMetricCollector::parse_proc_stat_line_cpu(proc_stat_line).unwrap()" +--- +[ + "cpu", + [ + 1000.0, + 5.0, + 0.0, + 0.0, + 2.0, + 0.0, + 0.0 + ] +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__test_basic_line_with_extra.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__test_basic_line_with_extra.snap new file mode 100644 index 0000000..11e8b35 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__test_basic_line_with_extra.snap @@ -0,0 +1,16 @@ +--- +source: memfaultd/src/metrics/system_metrics/cpu.rs +expression: "CpuMetricCollector::parse_proc_stat_line_cpu(proc_stat_line).unwrap()" +--- +[ + "cpu0", + [ + 1000.0, + 5.0, + 0.0, + 0.0, + 2.0, + 0.0, + 0.0 + ] +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__disk_space__test__initialize_and_calc_disk_space_for_mounts-2.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__disk_space__test__initialize_and_calc_disk_space_for_mounts-2.snap new file mode 100644 index 0000000..c3dbaf0 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__disk_space__test__initialize_and_calc_disk_space_for_mounts-2.snap @@ -0,0 +1,42 @@ +--- +source: memfaultd/src/metrics/system_metrics/disk_space.rs +expression: metrics +--- +[ + { + "name": "disk_space/sda1/free_bytes", + "value": { + "Histogram": { + "value": 1171456.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "disk_space/sda1/used_bytes", + "value": { + "Histogram": { + "value": 3022848.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "disk_space/sda2/free_bytes", + "value": { + "Histogram": { + "value": 1171456.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "disk_space/sda2/used_bytes", + "value": { + "Histogram": { + "value": 3022848.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__disk_space__test__initialize_and_calc_disk_space_for_mounts.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__disk_space__test__initialize_and_calc_disk_space_for_mounts.snap new file mode 100644 index 0000000..720db1a --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__disk_space__test__initialize_and_calc_disk_space_for_mounts.snap @@ -0,0 +1,14 @@ +--- +source: memfaultd/src/metrics/system_metrics/disk_space.rs +expression: disk_space_collector.mounts +--- +[ + { + "device": "/dev/sda1", + "mount_point": "/" + }, + { + "device": "/dev/sda2", + "mount_point": "/media" + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__disk_space__test__unmonitored_disks_not_initialized.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__disk_space__test__unmonitored_disks_not_initialized.snap new file mode 100644 index 0000000..f41c0ed --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__disk_space__test__unmonitored_disks_not_initialized.snap @@ -0,0 +1,5 @@ +--- +source: memfaultd/src/metrics/system_metrics/disk_space.rs +expression: disk_space_collector.mounts +--- +[] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__memory__test__get_memory_metrics.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__memory__test__get_memory_metrics.snap new file mode 100644 index 0000000..9964a31 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__memory__test__get_memory_metrics.snap @@ -0,0 +1,24 @@ +--- +source: memfaultd/src/metrics/system_metrics/memory.rs +expression: "MemoryMetricsCollector::parse_meminfo_memory_stats(meminfo).unwrap()" +--- +[ + { + "name": "memory/memory/free", + "value": { + "Histogram": { + "value": 248090624.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "memory/memory/used", + "value": { + "Histogram": { + "value": 75599872.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__memory__test__get_memory_metrics_no_memavailable.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__memory__test__get_memory_metrics_no_memavailable.snap new file mode 100644 index 0000000..171db3e --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__memory__test__get_memory_metrics_no_memavailable.snap @@ -0,0 +1,24 @@ +--- +source: memfaultd/src/metrics/system_metrics/memory.rs +expression: "MemoryMetricsCollector::parse_meminfo_memory_stats(meminfo).unwrap()" +--- +[ + { + "name": "memory/memory/free", + "value": { + "Histogram": { + "value": 248090624.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "memory/memory/used", + "value": { + "Histogram": { + "value": 126607360.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__basic_delta_a_b_metrics.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__basic_delta_a_b_metrics.snap new file mode 100644 index 0000000..b075d2c --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__basic_delta_a_b_metrics.snap @@ -0,0 +1,78 @@ +--- +source: memfaultd/src/metrics/system_metrics/network_interfaces.rs +expression: result_b.unwrap() +--- +[ + { + "name": "interface/eth0/bytes_per_second/rx", + "value": { + "Histogram": { + "value": 150.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/packets_per_second/rx", + "value": { + "Histogram": { + "value": 5.5, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/errors_per_second/rx", + "value": { + "Histogram": { + "value": 1.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/dropped_per_second/rx", + "value": { + "Histogram": { + "value": 1.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/bytes_per_second/tx", + "value": { + "Histogram": { + "value": 100.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/packets_per_second/tx", + "value": { + "Histogram": { + "value": 2.5, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/errors_per_second/tx", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/dropped_per_second/tx", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__basic_delta_b_c_metrics.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__basic_delta_b_c_metrics.snap new file mode 100644 index 0000000..747c537 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__basic_delta_b_c_metrics.snap @@ -0,0 +1,78 @@ +--- +source: memfaultd/src/metrics/system_metrics/network_interfaces.rs +expression: result_c.unwrap() +--- +[ + { + "name": "interface/eth0/bytes_per_second/rx", + "value": { + "Histogram": { + "value": 83.33333, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/packets_per_second/rx", + "value": { + "Histogram": { + "value": 0.66667, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/errors_per_second/rx", + "value": { + "Histogram": { + "value": 0.16667, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/dropped_per_second/rx", + "value": { + "Histogram": { + "value": 0.16667, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/bytes_per_second/tx", + "value": { + "Histogram": { + "value": 66.66667, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/packets_per_second/tx", + "value": { + "Histogram": { + "value": 0.83333, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/errors_per_second/tx", + "value": { + "Histogram": { + "value": 0.66667, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/dropped_per_second/tx", + "value": { + "Histogram": { + "value": 0.66667, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__different_interfaces_a_c_metrics.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__different_interfaces_a_c_metrics.snap new file mode 100644 index 0000000..dcab1f9 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__different_interfaces_a_c_metrics.snap @@ -0,0 +1,78 @@ +--- +source: memfaultd/src/metrics/system_metrics/network_interfaces.rs +expression: result_c.unwrap() +--- +[ + { + "name": "interface/eth0/bytes_per_second/rx", + "value": { + "Histogram": { + "value": 100.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/packets_per_second/rx", + "value": { + "Histogram": { + "value": 1.875, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/errors_per_second/rx", + "value": { + "Histogram": { + "value": 0.375, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/dropped_per_second/rx", + "value": { + "Histogram": { + "value": 0.375, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/bytes_per_second/tx", + "value": { + "Histogram": { + "value": 75.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/packets_per_second/tx", + "value": { + "Histogram": { + "value": 1.25, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/errors_per_second/tx", + "value": { + "Histogram": { + "value": 0.5, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/dropped_per_second/tx", + "value": { + "Histogram": { + "value": 0.5, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__eth0.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__eth0.snap new file mode 100644 index 0000000..8488d1b --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__eth0.snap @@ -0,0 +1,25 @@ +--- +source: memfaultd/src/metrics/system_metrics/network_interfaces.rs +expression: "NetworkInterfaceMetricCollector::::parse_proc_net_dev_line(proc_net_dev_line).unwrap()" +--- +[ + "eth0", + [ + 2707, + 25, + 0, + 0, + 0, + 0, + 0, + 0, + 2707, + 25, + 0, + 0, + 0, + 0, + 0, + 0 + ] +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__with_overflow_a_b_metrics.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__with_overflow_a_b_metrics.snap new file mode 100644 index 0000000..6aaac07 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__with_overflow_a_b_metrics.snap @@ -0,0 +1,78 @@ +--- +source: memfaultd/src/metrics/system_metrics/network_interfaces.rs +expression: result_b.unwrap() +--- +[ + { + "name": "interface/eth0/bytes_per_second/rx", + "value": { + "Histogram": { + "value": 250.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/packets_per_second/rx", + "value": { + "Histogram": { + "value": 5.5, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/errors_per_second/rx", + "value": { + "Histogram": { + "value": 1.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/dropped_per_second/rx", + "value": { + "Histogram": { + "value": 1.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/bytes_per_second/tx", + "value": { + "Histogram": { + "value": 100.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/packets_per_second/tx", + "value": { + "Histogram": { + "value": 2.5, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/errors_per_second/tx", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/dropped_per_second/tx", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__with_overflow_b_c_metrics.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__with_overflow_b_c_metrics.snap new file mode 100644 index 0000000..fb6382a --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__with_overflow_b_c_metrics.snap @@ -0,0 +1,78 @@ +--- +source: memfaultd/src/metrics/system_metrics/network_interfaces.rs +expression: result_c.unwrap() +--- +[ + { + "name": "interface/eth0/bytes_per_second/rx", + "value": { + "Histogram": { + "value": 83.4, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/packets_per_second/rx", + "value": { + "Histogram": { + "value": 0.66667, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/errors_per_second/rx", + "value": { + "Histogram": { + "value": 0.16667, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/dropped_per_second/rx", + "value": { + "Histogram": { + "value": 0.16667, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/bytes_per_second/tx", + "value": { + "Histogram": { + "value": 66.66667, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/packets_per_second/tx", + "value": { + "Histogram": { + "value": 0.83333, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/errors_per_second/tx", + "value": { + "Histogram": { + "value": 0.66667, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/dropped_per_second/tx", + "value": { + "Histogram": { + "value": 0.66667, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__wlan1.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__wlan1.snap new file mode 100644 index 0000000..0476b78 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__wlan1.snap @@ -0,0 +1,25 @@ +--- +source: memfaultd/src/metrics/system_metrics/network_interfaces.rs +expression: "NetworkInterfaceMetricCollector::::parse_proc_net_dev_line(proc_net_dev_line).unwrap()" +--- +[ + "wlan1", + [ + 2707, + 25, + 0, + 0, + 0, + 0, + 0, + 0, + 2707, + 25, + 0, + 0, + 0, + 0, + 0, + 0 + ] +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__processes__tests__process_metrics_auto_false.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__processes__tests__process_metrics_auto_false.snap new file mode 100644 index 0000000..3371196 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__processes__tests__process_metrics_auto_false.snap @@ -0,0 +1,132 @@ +--- +source: memfaultd/src/metrics/system_metrics/processes.rs +expression: process_metric_readings_2 +--- +[ + { + "name": "processes/memfaultd/cpu/percent/system", + "value": { + "Histogram": { + "value": 0.05, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/cpu/percent/user", + "value": { + "Histogram": { + "value": 0.1, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/num_threads", + "value": { + "Histogram": { + "value": 19.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/pagefaults/major", + "value": { + "Histogram": { + "value": 16.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/pagefaults/minor", + "value": { + "Histogram": { + "value": 20.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/rss_bytes", + "value": { + "Histogram": { + "value": 18010112.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/vm_bytes", + "value": { + "Histogram": { + "value": 1411293184.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/systemd/cpu/percent/system", + "value": { + "Histogram": { + "value": 0.0028, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/systemd/cpu/percent/user", + "value": { + "Histogram": { + "value": 0.0088, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/systemd/num_threads", + "value": { + "Histogram": { + "value": 1.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/systemd/pagefaults/major", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/systemd/pagefaults/minor", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/systemd/rss_bytes", + "value": { + "Histogram": { + "value": 11403264.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/systemd/vm_bytes", + "value": { + "Histogram": { + "value": 19828736.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__processes__tests__process_metrics_auto_true.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__processes__tests__process_metrics_auto_true.snap new file mode 100644 index 0000000..8380217 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__processes__tests__process_metrics_auto_true.snap @@ -0,0 +1,69 @@ +--- +source: memfaultd/src/metrics/system_metrics/processes.rs +expression: process_metric_readings_2 +--- +[ + { + "name": "processes/memfaultd/cpu/percent/system", + "value": { + "Histogram": { + "value": 0.05, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/cpu/percent/user", + "value": { + "Histogram": { + "value": 0.1, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/num_threads", + "value": { + "Histogram": { + "value": 19.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/pagefaults/major", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/pagefaults/minor", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/rss_bytes", + "value": { + "Histogram": { + "value": 18010112.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/vm_bytes", + "value": { + "Histogram": { + "value": 1411293184.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__processes__tests__simple_cpu_delta_metrics.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__processes__tests__simple_cpu_delta_metrics.snap new file mode 100644 index 0000000..3c733cc --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__processes__tests__simple_cpu_delta_metrics.snap @@ -0,0 +1,69 @@ +--- +source: memfaultd/src/metrics/system_metrics/processes.rs +expression: process_metric_readings.unwrap() +--- +[ + { + "name": "processes/memfaultd/rss_bytes", + "value": { + "Histogram": { + "value": 18010112.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/vm_bytes", + "value": { + "Histogram": { + "value": 1411293184.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/num_threads", + "value": { + "Histogram": { + "value": 19.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/cpu/percent/system", + "value": { + "Histogram": { + "value": 0.05, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/cpu/percent/user", + "value": { + "Histogram": { + "value": 0.1, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/pagefaults/minor", + "value": { + "Histogram": { + "value": 50.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/pagefaults/major", + "value": { + "Histogram": { + "value": 10.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/thermal.rs b/memfaultd/src/metrics/system_metrics/thermal.rs new file mode 100644 index 0000000..d251f32 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/thermal.rs @@ -0,0 +1,138 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Collect temperature readings from /sys/class/thermal +//! +//! This module parses thermal readings from /sys/class/thermal and constructs +//! KeyedMetricReadings based on those statistics. +//! +//! Example /sys/class/thermal contents: +//! +//! /sys/class/thermal/ +//! ├── cooling_device0 -> ../../devices/virtual/thermal/cooling_device0 +//! ├── cooling_device1 -> ../../devices/virtual/thermal/cooling_device1 +//! ├── thermal_zone0 -> ../../devices/virtual/thermal/thermal_zone0 +//! └── thermal_zone1 -> ../../devices/virtual/thermal/thermal_zone1 +//! +//! Example /sys/class/thermal/thermal_zone[0-*] contents: +//! +//! /sys/class/thermal/thermal_zone0 +//! ├── ... +//! └── temp // this is the property we're interested in +//! +//! See additional Linux kernel documentation on /sys/class/thermal here: +//! https://www.kernel.org/doc/Documentation/thermal/sysfs-api.txt + +use std::str::FromStr; + +use crate::metrics::{ + system_metrics::SystemMetricFamilyCollector, KeyedMetricReading, MetricStringKey, +}; +use eyre::{eyre, Result}; + +const SYS_CLASS_THERMAL_PATH: &str = "/sys/class/thermal"; +pub const THERMAL_METRIC_NAMESPACE: &str = "thermal"; + +pub struct ThermalMetricsCollector; + +impl ThermalMetricsCollector { + pub fn new() -> Self { + ThermalMetricsCollector {} + } + + fn read_thermal_zone_temp(zone_name: &str, root_dir: &str) -> Result { + let temp_file = &format!("{}/{}/temp", root_dir, zone_name); + // The readings are in millidegrees Celsius, so we divide by 1000 to get + // the temperature in degrees Celsius. + let temp_in_celsius = std::fs::read_to_string(temp_file)?.trim().parse::()? / 1000.0; + + Ok(KeyedMetricReading::new_gauge( + MetricStringKey::from_str(zone_name).map_err(|e| { + eyre!( + "Failed to construct MetricStringKey for thermal zone: {}", + e + ) + })?, + temp_in_celsius, + )) + } + + // To facilitate unit testing, make the thermal directory path an arg + fn read_thermal_metrics_from_dir(dir: &str) -> Result> { + // The /sys/class/thermal/ directory will contain symlinks to + // pseudo-files named "thermal_zone0" etc, depending on the number of + // thermal zones in the system. The file we read for the temperature + // reading is for example /sys/class/thermal/thermal_zone0/temp, + // containing an integer value in millidegrees Celsius, ex: "53000" + let metrics: Vec<_> = std::fs::read_dir(dir)? + .filter_map(|entry| entry.ok()) + .map(|entry| entry.path()) + .filter_map(|path| Some(path.file_name()?.to_str()?.to_string())) + .filter(|name| name.starts_with("thermal_zone")) + .filter_map(|name| Self::read_thermal_zone_temp(&name, dir).ok()) + .collect(); + + Ok(metrics) + } + + pub fn get_thermal_metrics() -> Result> { + Self::read_thermal_metrics_from_dir(SYS_CLASS_THERMAL_PATH) + } +} + +impl SystemMetricFamilyCollector for ThermalMetricsCollector { + fn family_name(&self) -> &'static str { + THERMAL_METRIC_NAMESPACE + } + + fn collect_metrics(&mut self) -> Result> { + Self::get_thermal_metrics() + } +} + +#[cfg(test)] +// The floating point literal pattern is allowed in this test module because +// the input and output values are known. +mod tests { + use super::*; + use crate::metrics::MetricReading; + use std::fs::File; + use std::io::Write; + use tempfile::tempdir; + + #[test] + fn test_read_thermal_zone_temp() { + // Create a temporary directory. + let dir = tempdir().unwrap(); + + // Create a "thermal_zone0" directory inside the temporary directory. + let thermal_zone_dir = dir.path().join("thermal_zone0"); + std::fs::create_dir(&thermal_zone_dir).unwrap(); + + // Create a "temp" file inside the "thermal_zone0" directory. + let temp_file_path = thermal_zone_dir.join("temp"); + let mut temp_file = File::create(temp_file_path).unwrap(); + + // Write the temperature (in millidegrees Celsius) to the "temp" file. + writeln!(temp_file, "50000").unwrap(); + + // Call the function and check the result. + let result = ThermalMetricsCollector::read_thermal_zone_temp( + "thermal_zone0", + dir.path().to_str().unwrap(), + ) + .unwrap(); + // The temperature should be 50.0 degrees Celsius. + assert!(matches!( + result.value, + MetricReading::Gauge { + #[allow(illegal_floating_point_literal_pattern)] + value: 50.0, + .. + } + )); + + // Delete the temporary directory. + dir.close().unwrap(); + } +} diff --git a/memfaultd/src/metrics/timeseries/mod.rs b/memfaultd/src/metrics/timeseries/mod.rs new file mode 100644 index 0000000..836feae --- /dev/null +++ b/memfaultd/src/metrics/timeseries/mod.rs @@ -0,0 +1,447 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use chrono::{DateTime, Utc}; +use eyre::{eyre, Result}; +use std::cmp; + +use super::{MetricReading, MetricValue}; + +const FINITENESS_ERROR: &str = "Metric values must be finite."; + +/// A trait for the storage of multiple metric events aggregated together. +/// This (roughly) maps to a time series in OpenTelemetry data model: +/// https://opentelemetry.io/docs/specs/otel/metrics/data-model/#timeseries-model +pub trait TimeSeries { + fn aggregate(&mut self, newer: &MetricReading) -> Result<()>; + fn value(&self) -> MetricValue; +} + +pub struct Histogram { + sum: f64, + count: u64, + start: DateTime, + end: DateTime, + min: f64, + max: f64, +} + +impl Histogram { + pub fn new(reading: &MetricReading) -> Result { + match *reading { + MetricReading::Histogram { value, timestamp } => { + if !value.is_finite() { + return Err(eyre!(FINITENESS_ERROR)); + } + Ok(Self { + sum: value, + count: 1, + start: timestamp, + end: timestamp, + min: value, + max: value, + }) + } + _ => Err(eyre!("Cannot create a histogram from a non-gauge metric")), + } + } +} + +impl TimeSeries for Histogram { + fn aggregate(&mut self, newer: &MetricReading) -> Result<()> { + match newer { + MetricReading::Histogram { value, timestamp } => { + if !value.is_finite() { + return Err(eyre!(FINITENESS_ERROR)); + } + self.sum += value; + self.count += 1; + self.start = cmp::min(self.start, *timestamp); + self.end = cmp::max(self.end, *timestamp); + self.min = f64::min(self.min, *value); + self.max = f64::max(self.max, *value); + Ok(()) + } + _ => Err(eyre!( + "Cannot aggregate a histogram with a non-gauge metric" + )), + } + } + + fn value(&self) -> MetricValue { + if self.count > 0 { + MetricValue::Number(self.sum / self.count as f64) + } else { + MetricValue::Number(f64::NAN) + } + } +} + +/// An aggregation that calculates the sum of all values received. This assumes that all readings will be positive numbers. +/// Monotonic counter in OpenTelemetry data model. +pub struct Counter { + sum: f64, + end: DateTime, +} + +impl Counter { + pub fn new(reading: &MetricReading) -> Result { + match *reading { + MetricReading::Counter { value, timestamp } => { + if !value.is_finite() { + return Err(eyre!(FINITENESS_ERROR)); + } + Ok(Self { + sum: value, + end: timestamp, + }) + } + _ => Err(eyre!("Cannot create a sum from a non-counter metric")), + } + } +} + +impl TimeSeries for Counter { + fn aggregate(&mut self, newer: &MetricReading) -> Result<()> { + match newer { + MetricReading::Counter { value, timestamp } => { + if !value.is_finite() { + return Err(eyre!(FINITENESS_ERROR)); + } + self.sum += value; + self.end = *timestamp; + Ok(()) + } + _ => Err(eyre!("Cannot aggregate a sum with a non-counter metric")), + } + } + + fn value(&self) -> MetricValue { + MetricValue::Number(self.sum) + } +} + +/// A time-weighted sum of all values received. This is useful to maintain an accurate average measurement when the interval between readings is not constant. +pub struct TimeWeightedAverage { + weighted_sum: f64, + duration: u64, + end: DateTime, +} + +impl TimeWeightedAverage { + pub fn new(reading: &MetricReading) -> Result { + match *reading { + MetricReading::TimeWeightedAverage { + value, + timestamp, + interval, + } => { + if !value.is_finite() { + return Err(eyre!(FINITENESS_ERROR)); + } + Ok(Self { + weighted_sum: value * interval.num_milliseconds() as f64, + duration: interval.num_milliseconds() as u64, + end: timestamp, + }) + } + _ => Err(eyre!( + "Mismatch between Time Weighted Average aggregation and {:?} reading", + reading + )), + } + } +} + +impl TimeSeries for TimeWeightedAverage { + fn aggregate(&mut self, newer: &MetricReading) -> Result<()> { + match newer { + MetricReading::TimeWeightedAverage { + value, timestamp, .. + } => { + if !value.is_finite() { + return Err(eyre!(FINITENESS_ERROR)); + } + if timestamp < &self.end { + return Err(eyre!( + "Cannot aggregate a time-weighted average with an older timestamp" + )); + } + let duration = (*timestamp - self.end).num_milliseconds() as u64; + self.weighted_sum += value * duration as f64; + self.duration += duration; + self.end = *timestamp; + Ok(()) + } + _ => Err(eyre!( + "Cannot aggregate a time-weighted average with a non-gauge metric" + )), + } + } + + fn value(&self) -> MetricValue { + if self.duration > 0 { + MetricValue::Number(self.weighted_sum / self.duration as f64) + } else { + MetricValue::Number(f64::NAN) + } + } +} + +pub struct Gauge { + value: f64, + end: DateTime, +} + +impl Gauge { + pub fn new(reading: &MetricReading) -> Result { + match *reading { + MetricReading::Gauge { value, timestamp } => { + if !value.is_finite() { + return Err(eyre!(FINITENESS_ERROR)); + } + Ok(Self { + value, + end: timestamp, + }) + } + _ => Err(eyre!( + "Cannot create a gauge aggregation from a non-gauge metric" + )), + } + } +} + +impl TimeSeries for Gauge { + fn aggregate(&mut self, newer: &MetricReading) -> Result<()> { + match newer { + MetricReading::Gauge { value, timestamp } => { + if !value.is_finite() { + return Err(eyre!(FINITENESS_ERROR)); + } + if *timestamp > self.end { + self.value = *value; + self.end = *timestamp; + } + Ok(()) + } + _ => Err(eyre!( + "Cannot aggregate a histogram with a non-gauge metric" + )), + } + } + + fn value(&self) -> MetricValue { + MetricValue::Number(self.value) + } +} + +/// An aggregation stores the most recently received String +/// associated with a key as a tag on the report +pub struct ReportTag { + value: String, + end: DateTime, +} + +impl ReportTag { + pub fn new(reading: &MetricReading) -> Result { + match reading { + MetricReading::ReportTag { value, timestamp } => Ok(Self { + value: value.clone(), + end: *timestamp, + }), + _ => Err(eyre!( + "Cannot create a report tag from a non-report tag reading" + )), + } + } +} + +impl TimeSeries for ReportTag { + fn aggregate(&mut self, newer: &MetricReading) -> Result<()> { + match newer { + MetricReading::ReportTag { value, timestamp } => { + if *timestamp > self.end { + self.value = value.clone(); + self.end = *timestamp; + } + Ok(()) + } + _ => Err(eyre!( + "Cannot aggregate a report tag with a non-report-tag reading" + )), + } + } + + fn value(&self) -> MetricValue { + MetricValue::String(self.value.clone()) + } +} +#[cfg(test)] +mod tests { + use chrono::Duration; + use rstest::rstest; + + use crate::metrics::{MetricReading, MetricTimestamp, MetricValue}; + use std::{f64::INFINITY, f64::NAN, f64::NEG_INFINITY, str::FromStr}; + + use super::TimeSeries; + use super::{Counter, Gauge, Histogram, TimeWeightedAverage}; + + #[rstest] + #[case(1.0, 1000, 2.0, 1.5, 1000)] + #[case(10.0, 10_000, 10.0, 10.0, 10_000)] + #[case(1.0, 9_000, 0.0, 0.5, 9_000)] + #[case(1.0, 0, 2.0, 1.5, 0)] + #[case(1.0, 1000, 2.0, 1.5, 1000)] + fn test_histogram_aggregation( + #[case] a: f64, + #[case] duration_between_ms: i64, + #[case] b: f64, + #[case] expected: f64, + #[case] expected_ms: i64, + ) { + let t0 = MetricTimestamp::from_str("2021-01-01T00:00:00Z").unwrap(); + + let a = MetricReading::Histogram { + value: a, + timestamp: t0, + }; + let b = MetricReading::Histogram { + value: b, + timestamp: t0 + Duration::milliseconds(duration_between_ms), + }; + + let mut h = Histogram::new(&a).unwrap(); + h.aggregate(&b).unwrap(); + + assert_eq!(h.start, t0); + assert_eq!(h.end, t0 + Duration::milliseconds(duration_between_ms)); + assert_eq!((h.end - h.start).num_milliseconds(), expected_ms); + assert_eq!(h.value(), MetricValue::Number(expected)); + } + + #[rstest] + #[case(1.0, 1000, 2.0, 1000, 1.5, 2000)] + #[case(10.0, 10000, 10.0, 1000, 10.0, 11000)] + #[case(1.0, 9_000, 0.0, 1_000, 0.9, 10_000)] + #[case(1.0, 0, 2.0, 1, 2.0, 1)] + #[case(1.0, 1000, 2.0, 0, 1.0, 1000)] + fn test_time_weighted_aggregation( + #[case] a: f64, + #[case] a_ms: i64, + #[case] b: f64, + #[case] b_ms: i64, + #[case] expected: f64, + #[case] expected_ms: u64, + ) { + let t0 = MetricTimestamp::from_str("2021-01-01T00:00:00Z").unwrap(); + + let a = MetricReading::TimeWeightedAverage { + value: a, + timestamp: t0 + Duration::milliseconds(a_ms), + interval: Duration::milliseconds(a_ms), + }; + let b = MetricReading::TimeWeightedAverage { + value: b, + timestamp: t0 + Duration::milliseconds(a_ms + b_ms), + interval: Duration::milliseconds(b_ms), + }; + + let mut h = TimeWeightedAverage::new(&a).unwrap(); + h.aggregate(&b).unwrap(); + + assert_eq!(h.end, t0 + Duration::milliseconds(a_ms + b_ms)); + assert_eq!(h.duration, expected_ms); + assert_eq!(h.value(), MetricValue::Number(expected)); + } + + #[rstest] + fn test_incompatible_metric_type_on_histogram() { + let timestamp = MetricTimestamp::from_str("2021-01-01T00:00:00Z").unwrap(); + + let a = MetricReading::Histogram { + value: 1.0, + timestamp, + }; + let b = MetricReading::Counter { + value: 2.0, + timestamp, + }; + + assert!(Histogram::new(&a).unwrap().aggregate(&b).is_err()); + } + + #[rstest] + #[case(INFINITY)] + #[case(NEG_INFINITY)] + #[case(NAN)] + fn test_edge_values_new(#[case] edge_value: f64) { + let timestamp = MetricTimestamp::from_str("2021-01-01T00:00:00Z").unwrap(); + let a = MetricReading::Histogram { + value: edge_value, + timestamp, + }; + assert!(Histogram::new(&a).is_err()); + } + + #[rstest] + #[case(INFINITY)] + #[case(NEG_INFINITY)] + #[case(NAN)] + fn test_edge_values_aggregate(#[case] edge_value: f64) { + let timestamp = MetricTimestamp::from_str("2021-01-01T00:00:00Z").unwrap(); + let a = MetricReading::Histogram { + value: 0.0, + timestamp, + }; + let b = MetricReading::Histogram { + value: edge_value, + timestamp, + }; + assert!(Histogram::new(&a).unwrap().aggregate(&b).is_err()); + } + + #[rstest] + #[case(1.0, 2.0, 3.0)] + fn test_counter_aggregation(#[case] a: f64, #[case] b: f64, #[case] expected: f64) { + let timestamp = MetricTimestamp::from_str("2021-01-01T00:00:00Z").unwrap(); + let timestamp2 = MetricTimestamp::from_str("2021-01-01T00:00:43Z").unwrap(); + + let a = MetricReading::Counter { + value: a, + timestamp, + }; + let b = MetricReading::Counter { + value: b, + timestamp: timestamp2, + }; + + let mut sum = Counter::new(&a).unwrap(); + sum.aggregate(&b).unwrap(); + assert_eq!(sum.end, timestamp2); + assert_eq!(sum.sum, expected); + } + + #[rstest] + #[case(1.0, 2.0, 2.0)] + #[case(10.0, 1.0, 1.0)] + fn test_gauge_aggregation(#[case] a: f64, #[case] b: f64, #[case] expected: f64) { + let timestamp = MetricTimestamp::from_str("2021-01-01T00:00:00Z").unwrap(); + let timestamp2 = MetricTimestamp::from_str("2021-01-01T00:00:43Z").unwrap(); + + let a = MetricReading::Gauge { + value: a, + timestamp, + }; + let b = MetricReading::Gauge { + value: b, + timestamp: timestamp2, + }; + + let mut gauge = Gauge::new(&a).unwrap(); + gauge.aggregate(&b).unwrap(); + assert_eq!(gauge.end, timestamp2); + assert_eq!(gauge.value, expected); + } +} diff --git a/memfaultd/src/network/client.rs b/memfaultd/src/network/client.rs new file mode 100644 index 0000000..e726e33 --- /dev/null +++ b/memfaultd/src/network/client.rs @@ -0,0 +1,238 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::fs::File; +use std::io::Read; +use std::str; + +use eyre::eyre; +use eyre::Context; +use eyre::Result; +use log::{debug, trace}; +use reqwest::blocking; +use reqwest::blocking::Body; +use reqwest::header; + +use crate::retriable_error::RetriableError; +use crate::util::io::StreamLen; +use crate::util::string::Ellipsis; + +use super::requests::DeviceConfigResponse; +use super::requests::UploadPrepareRequest; +use super::requests::UploadPrepareResponse; +use super::requests::{DeviceConfigRequest, MarUploadMetadata}; +use super::NetworkClient; +use super::NetworkConfig; + +/// Memfault Network client +pub struct NetworkClientImpl { + client: blocking::Client, + /// A separate client for upload to file storage + file_upload_client: blocking::Client, + config: NetworkConfig, +} + +#[allow(clippy::upper_case_acronyms)] +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Method { + POST, +} + +impl NetworkClientImpl { + pub fn new(config: NetworkConfig) -> Result { + let headers = [ + ( + header::ACCEPT, + header::HeaderValue::from_static("application/json"), + ), + ( + header::CONTENT_TYPE, + header::HeaderValue::from_static("application/json"), + ), + ( + header::HeaderName::from_static("memfault-project-key"), + header::HeaderValue::from_str(&config.project_key)?, + ), + ( + header::CONTENT_ENCODING, + header::HeaderValue::from_static("utf-8"), + ), + ] + .into_iter() + .collect(); + + let client = blocking::ClientBuilder::new() + .default_headers(headers) + .build()?; + + Ok(NetworkClientImpl { + client, + file_upload_client: blocking::Client::new(), + config, + }) + } + + fn good_response_or_error(response: blocking::Response) -> Result { + // Map status code to an error + let status = response.status(); + match status.as_u16() { + 200..=299 => Ok(response), + // Server errors are expected to be temporary and will be retried later + 500..=599 => Err(RetriableError::ServerError { + status_code: status.as_u16(), + } + .into()), + // Any other error (404, etc) will be considered fatal and will not be retried. + // HTTP client errors (4xx) are not expected to happen in normal operation, but can + // occur due to misconfiguration/integration issues. Log the first 1KB of the response + // body to help with debugging: + _ => { + let mut response_text = response.text().unwrap_or_else(|_| "???".into()); + // Limit the size of the response text to avoid filling up the log: + response_text.truncate_with_ellipsis(1024); + Err(eyre!( + "Unexpected server response: {} {}", + status.as_u16(), + response_text + )) + } + } + } + + /// Send a request to Memfault backend + fn fetch(&self, method: Method, endpoint: &str, payload: &str) -> Result { + let url = format!("{}{}", self.config.base_url, endpoint); + debug!( + "{:?} {} - Payload {} bytes\n{:?}", + method, + url, + payload.len(), + payload + ); + let response = self + .client + .request( + match method { + Method::POST => reqwest::Method::POST, + }, + url, + ) + .body(payload.to_owned()) + .send() + // "send(): This method fails if there was an error while sending request, redirect loop was detected or redirect limit was exhausted." + // All kinds of errors here are considered "recoverable" and will be retried. + .map_err(|e| RetriableError::NetworkError { source: e })?; + debug!( + " Response status {} - Size {:?}", + response.status(), + response.content_length(), + ); + Self::good_response_or_error(response) + } + + /// Upload a file to S3 and return the file token + fn prepare_and_upload( + &self, + file: BodyAdapter, + gzipped: bool, + ) -> Result { + let prepare_request = + UploadPrepareRequest::prepare(&self.config, file.size as usize, gzipped); + + let prepare_response = self + .fetch( + Method::POST, + "/api/v0/upload", + &serde_json::to_string(&prepare_request)?, + )? + .json::() + .wrap_err("Prepare upload error")?; + + trace!("Upload prepare response: {:?}", prepare_response); + + self.put_file( + &prepare_response.data.upload_url, + file, + if gzipped { Some("gzip") } else { None }, + ) + .wrap_err("Storage upload error")?; + debug!("Successfully transmitted file"); + + Ok(prepare_response.data.token) + } + + fn put_file( + &self, + url: &str, + file: BodyAdapter, + content_encoding: Option<&str>, + ) -> Result<()> { + let mut req = self.file_upload_client.put(url); + + if let Some(content_encoding) = content_encoding { + trace!("Adding content-encoding header"); + req = req.header(header::CONTENT_ENCODING, content_encoding); + } + + trace!("Uploading file to {}", url); + let body: Body = file.into(); + let r = req.body(body).send()?; + Self::good_response_or_error(r).and(Ok(())) + } +} + +impl NetworkClient for NetworkClientImpl { + fn upload_mar_file(&self, file: F) -> Result<()> { + let token = self.prepare_and_upload(file.into(), false)?; + + let mar_upload = MarUploadMetadata::prepare(&self.config, &token); + self.fetch( + Method::POST, + "/api/v0/upload/mar", + &serde_json::to_string(&mar_upload)?, + ) + .wrap_err("MAR Upload Error") + .and(Ok(())) + } + + fn fetch_device_config(&self) -> Result { + let request = DeviceConfigRequest::from(&self.config); + self.fetch( + Method::POST, + "/api/v0/device-config", + &serde_json::to_string(&request)?, + )? + .json::() + .wrap_err("Fetch device-config error") + } +} + +/// Small helper to adapt a Read/File into a Body. +/// Note it's not possible to directly write: impl From for Body { ... } +/// because of orphan rules. See https://doc.rust-lang.org/error_codes/E0210.html +struct BodyAdapter { + reader: R, + size: u64, +} + +impl From for BodyAdapter { + fn from(reader: R) -> Self { + let size = reader.stream_len(); + Self { reader, size } + } +} + +impl TryFrom for BodyAdapter { + type Error = std::io::Error; + + fn try_from(file: File) -> Result { + let size = file.metadata()?.len(); + Ok(Self { reader: file, size }) + } +} + +impl From> for Body { + fn from(wrapper: BodyAdapter) -> Self { + Body::sized(wrapper.reader, wrapper.size) + } +} diff --git a/memfaultd/src/network/mod.rs b/memfaultd/src/network/mod.rs new file mode 100644 index 0000000..f8a41f0 --- /dev/null +++ b/memfaultd/src/network/mod.rs @@ -0,0 +1,64 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::Read; + +use eyre::Result; +#[cfg(test)] +use mockall::automock; + +use crate::config::Config; +use crate::util::io::StreamLen; + +mod client; +pub use client::NetworkClientImpl; + +mod requests; +pub use requests::*; + +#[cfg_attr(test, automock)] +pub trait NetworkClient { + /// Upload a MAR file to Memfault + fn upload_mar_file(&self, file: F) -> Result<()>; + + /// Fetch DeviceConfig from Memfault. + fn fetch_device_config(&self) -> Result; +} + +/// Internal representation of what is needed to talk to the backend. +#[derive(Clone)] +pub struct NetworkConfig { + pub project_key: String, + pub base_url: String, + pub device_id: String, + pub hardware_version: String, + pub software_version: String, + pub software_type: String, +} + +impl From<&Config> for NetworkConfig { + fn from(config: &Config) -> Self { + NetworkConfig { + project_key: config.config_file.project_key.clone(), + device_id: config.device_info.device_id.clone(), + base_url: config.config_file.base_url.clone(), + hardware_version: config.device_info.hardware_version.clone(), + software_type: config.software_type().to_string(), + software_version: config.software_version().to_string(), + } + } +} + +#[cfg(test)] +impl NetworkConfig { + pub fn test_fixture() -> Self { + NetworkConfig { + project_key: "abcd".to_owned(), + base_url: "https://devices.memfault.com/".to_owned(), + device_id: "001".to_owned(), + hardware_version: "DVT".to_owned(), + software_version: "1.0.0".to_owned(), + software_type: "test".to_owned(), + } + } +} diff --git a/memfaultd/src/network/requests.rs b/memfaultd/src/network/requests.rs new file mode 100644 index 0000000..444c0ba --- /dev/null +++ b/memfaultd/src/network/requests.rs @@ -0,0 +1,158 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use super::NetworkConfig; +use serde::{Deserialize, Serialize}; + +/// Device metadata required to prepare and commit uploads. +#[derive(Serialize, Deserialize, Debug)] +pub struct UploadDeviceMetadata<'a> { + device_serial: &'a str, + hardware_version: &'a str, + software_version: &'a str, + software_type: &'a str, +} + +impl<'a> From<&'a NetworkConfig> for UploadDeviceMetadata<'a> { + fn from(config: &'a NetworkConfig) -> Self { + UploadDeviceMetadata { + device_serial: config.device_id.as_str(), + hardware_version: config.hardware_version.as_str(), + software_type: config.software_type.as_str(), + software_version: config.software_version.as_str(), + } + } +} + +/// Request body to prepare an upload +#[derive(Serialize, Debug)] +pub struct UploadPrepareRequest<'a> { + content_encoding: Option<&'static str>, + size: usize, + device: UploadDeviceMetadata<'a>, +} + +impl<'a> UploadPrepareRequest<'a> { + pub fn prepare( + config: &'a NetworkConfig, + filesize: usize, + gzipped: bool, + ) -> UploadPrepareRequest<'a> { + UploadPrepareRequest { + content_encoding: if gzipped { Some("gzip") } else { None }, + size: filesize, + device: UploadDeviceMetadata::from(config), + } + } +} + +/// Response for prepare-upload request +#[derive(Serialize, Deserialize, Debug)] +pub struct UploadPrepareResponse { + pub data: UploadPrepareResponseData, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct UploadPrepareResponseData { + pub upload_url: String, + pub token: String, +} + +#[derive(Serialize, Debug)] +pub struct PreparedFile<'a> { + token: &'a str, +} + +#[derive(Serialize, Debug)] +pub struct MarUploadMetadata<'a> { + device_serial: &'a str, + file: PreparedFile<'a>, + hardware_version: &'a str, + software_type: &'a str, + software_version: &'a str, +} + +impl<'a> MarUploadMetadata<'a> { + pub fn prepare(config: &'a NetworkConfig, token: &'a str) -> Self { + Self { + device_serial: &config.device_id, + hardware_version: &config.hardware_version, + software_type: &config.software_type, + software_version: &config.software_version, + file: PreparedFile { token }, + } + } +} + +#[derive(Serialize, Deserialize, Debug)] +struct DeviceConfigDeviceInfo<'a> { + device_serial: &'a str, + hardware_version: &'a str, + software_version: &'a str, + software_type: &'a str, +} + +/// Device metadata required to prepare and commit uploads. +#[derive(Serialize, Deserialize, Debug)] +pub struct DeviceConfigRequest<'a> { + #[serde(borrow)] + device: DeviceConfigDeviceInfo<'a>, +} + +impl<'a> From<&'a NetworkConfig> for DeviceConfigRequest<'a> { + fn from(config: &'a NetworkConfig) -> Self { + DeviceConfigRequest { + device: DeviceConfigDeviceInfo { + device_serial: config.device_id.as_str(), + hardware_version: config.hardware_version.as_str(), + software_type: config.software_type.as_str(), + software_version: config.software_version.as_str(), + }, + } + } +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct DeviceConfigResponse { + pub data: DeviceConfigResponseData, +} + +pub type DeviceConfigRevision = u32; + +#[derive(Serialize, Deserialize, Debug)] +pub struct DeviceConfigResponseData { + pub config: DeviceConfigResponseConfig, + pub revision: DeviceConfigRevision, + pub completed: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct DeviceConfigResponseConfig { + pub memfault: DeviceConfigResponseMemfault, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct DeviceConfigResponseMemfault { + pub sampling: DeviceConfigResponseSampling, +} +#[derive(Serialize, Deserialize, Debug)] +pub struct DeviceConfigResponseSampling { + #[serde(rename = "debugging.resolution")] + pub debugging_resolution: DeviceConfigResponseResolution, + #[serde(rename = "logging.resolution")] + pub logging_resolution: DeviceConfigResponseResolution, + #[serde(rename = "monitoring.resolution")] + pub monitoring_resolution: DeviceConfigResponseResolution, +} + +#[derive(Serialize, Deserialize, Debug)] +pub enum DeviceConfigResponseResolution { + #[serde(rename = "off")] + Off, + #[serde(rename = "low")] + Low, + #[serde(rename = "normal")] + Normal, + #[serde(rename = "high")] + High, +} diff --git a/memfaultd/src/reboot/mod.rs b/memfaultd/src/reboot/mod.rs new file mode 100644 index 0000000..c0743fb --- /dev/null +++ b/memfaultd/src/reboot/mod.rs @@ -0,0 +1,465 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +mod reason; +pub use reason::RebootReason; +mod reason_codes; +pub use reason_codes::RebootReasonCode; + +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::{fs::write, fs::File, process::Command}; + +use eyre::{eyre, Context, Result}; +use log::{debug, error, info, warn}; +use uuid::Uuid; + +use crate::util::system::read_system_boot_id; +use crate::{config::Config, service_manager::ServiceManagerStatus}; +use crate::{mar::MarEntryBuilder, network::NetworkConfig}; +use crate::{mar::Metadata, service_manager::MemfaultdServiceManager}; + +const PSTORE_DIR: &str = "/sys/fs/pstore"; + +/// Manages reboot reasons and writes them to the MAR file if untracked. +/// +/// This tracker is responsible for tracking the boot_id and reboot reason +/// across reboots. It will write the reboot reason to the MAR file if it +/// has not been tracked yet. +pub struct RebootReasonTracker<'a> { + config: &'a Config, + sources: Vec, + service_manager: &'a dyn MemfaultdServiceManager, +} + +impl<'a> RebootReasonTracker<'a> { + pub fn new(config: &'a Config, service_manager: &'a dyn MemfaultdServiceManager) -> Self { + let sources = vec![ + RebootReasonSource { + name: "pstore", + func: read_reboot_reason_and_clear_file_pstore, + }, + RebootReasonSource { + name: "custom", + func: read_reboot_reason_and_clear_file_customer, + }, + RebootReasonSource { + name: "internal", + func: read_reboot_reason_and_clear_file_internal, + }, + ]; + + Self { + config, + sources, + service_manager, + } + } + + pub fn track_reboot(&self) -> Result<()> { + let boot_id = read_system_boot_id()?; + + if !self.config.config_file.enable_data_collection { + // Clear boot id since we haven't enabled data collection yet. + self.check_boot_id_is_tracked(&boot_id); + + process_pstore_files(PSTORE_DIR); + + return Ok(()); + } + + if !self.check_boot_id_is_tracked(&boot_id) { + let reboot_reason = self.resolve_reboot_reason(&boot_id)?; + + let mar_builder = MarEntryBuilder::new(&self.config.mar_staging_path())? + .set_metadata(Metadata::new_reboot(reboot_reason)); + + let network_config = NetworkConfig::from(self.config); + mar_builder.save(&network_config)?; + } + + Ok(()) + } + + fn check_boot_id_is_tracked(&self, boot_id: &Uuid) -> bool { + let tmp_filename = self + .config + .config_file + .generate_tmp_filename("last_tracked_boot_id"); + + let last_boot_id = std::fs::read_to_string(&tmp_filename) + .ok() + .and_then(|boot_id| Uuid::from_str(boot_id.trim()).ok()); + if last_boot_id.is_none() { + warn!("No last tracked boot_id found"); + } + + if let Err(e) = std::fs::write(tmp_filename, boot_id.to_string()) { + error!("Failed to write last tracked boot_id: {}", e); + } + + match last_boot_id { + Some(last_boot_id) => { + let is_tracked = &last_boot_id == boot_id; + if is_tracked { + info!("boot_id already tracked {}!", boot_id); + } + + is_tracked + } + None => false, + } + } + + fn resolve_reboot_reason(&self, boot_id: &Uuid) -> Result { + let mut reboot_reason = None; + for reason_source in &self.sources { + if let Some(new_reboot_reason) = (reason_source.func)(self.config) { + if reboot_reason.is_some() { + info!( + "Discarded reboot reason {:?} from {} source for boot_id {}", + new_reboot_reason, reason_source.name, boot_id + ); + } else { + info!( + "Using reboot reason {:?} from {} source for boot_id {}", + new_reboot_reason, reason_source.name, boot_id + ); + reboot_reason = Some(new_reboot_reason); + } + } + } + + Ok(reboot_reason.unwrap_or_else(|| RebootReason::from(RebootReasonCode::Unknown))) + } +} + +impl<'a> Drop for RebootReasonTracker<'a> { + fn drop(&mut self) { + let status = self.service_manager.service_manager_status(); + + match status { + Ok(ServiceManagerStatus::Stopping) => { + // Only write the user reset reason if the service manager is stopping + let reboot_file = self + .config + .config_file + .generate_persist_filename("lastrebootreason"); + + // Helper function to combine errors and handle them easier + fn inner_write(mut file: File) -> Result<()> { + // Ensure file is written as the process could exit before it is done. + let reason_int = RebootReasonCode::UserReset as u32; + + file.write_all(reason_int.to_string().as_bytes())?; + file.sync_all()?; + + Ok(()) + } + + info!("Writing reboot reason to {:?}", reboot_file); + match File::create(reboot_file) { + Ok(file) => { + if let Err(e) = inner_write(file) { + error!("Failed to write reboot reason: {}", e); + } + } + Err(e) => { + error!("Failed to create reboot reason file: {}", e); + } + } + } + Ok(status) => { + debug!( + "Service manager in state {:?} while closing. Not writing reboot reason", + status + ); + } + Err(e) => error!("Failed to get service manager status: {}", e), + } + } +} + +struct RebootReasonSource { + name: &'static str, + func: fn(&Config) -> Option, +} + +const PSTORE_DMESG_FILE: &str = "/sys/fs/pstore/dmesg-ramoops-0"; + +fn read_reboot_reason_and_clear_file_pstore(_config: &Config) -> Option { + if Path::new(PSTORE_DMESG_FILE).exists() { + process_pstore_files(PSTORE_DIR); + + Some(RebootReason::from(RebootReasonCode::KernelPanic)) + } else { + None + } +} + +fn read_reboot_reason_and_clear_file_internal(config: &Config) -> Option { + let reboot_file = config + .config_file + .generate_persist_filename("lastrebootreason"); + + read_reboot_reason_and_clear_file(&reboot_file) +} + +fn read_reboot_reason_and_clear_file_customer(config: &Config) -> Option { + let file_name = &config.config_file.reboot.last_reboot_reason_file; + + read_reboot_reason_and_clear_file(file_name) +} + +fn read_reboot_reason_and_clear_file(file_name: &PathBuf) -> Option { + let reboot_reason_string = match std::fs::read_to_string(file_name) { + Ok(reboot_reason) => reboot_reason, + Err(e) => { + debug!("Failed to open {:?}: {}", file_name, e.kind()); + return None; + } + }; + + let reboot_reason = RebootReason::from_str(reboot_reason_string.trim()).unwrap_or_else(|_| { + warn!("Couldn't parse reboot reason: {}", reboot_reason_string); + RebootReason::Code(RebootReasonCode::Unknown) + }); + + if let Err(e) = std::fs::remove_file(file_name) { + error!("Failed to remove {:?}: {}", file_name, e.kind()); + } + + Some(reboot_reason) +} + +fn process_pstore_files(pstore_dir: &str) { + // TODO: MFLT-7805 Process last kmsg/console logs + debug!("Cleaning up pstore..."); + + fn inner_process_pstore(pstore_dir: &str) -> Result<()> { + for entry in std::fs::read_dir(pstore_dir)? { + let path = entry?.path(); + + if path.is_file() || path.is_symlink() { + debug!("Cleaning pstore - Removing {}...", path.display()); + std::fs::remove_file(path)?; + } + } + + Ok(()) + } + + if let Err(e) = inner_process_pstore(pstore_dir) { + error!("Failed to process pstore files: {}", e); + } +} + +pub fn write_reboot_reason_and_reboot( + last_reboot_reason_file: &Path, + reason: RebootReason, +) -> Result<()> { + write(last_reboot_reason_file, format!("{}", reason)).wrap_err_with(|| { + format!( + "Unable to write reboot reason (path: {}).", + last_reboot_reason_file.display() + ) + })?; + if !Command::new("reboot").status()?.success() { + return Err(eyre!("Failed to reboot")); + } + Ok(()) +} + +#[cfg(test)] +mod test { + use super::*; + + use crate::mar::manifest::{Manifest, Metadata}; + use crate::service_manager::{MockMemfaultdServiceManager, ServiceManagerStatus}; + use crate::test_utils::setup_logger; + use crate::util::path::AbsolutePath; + + use rstest::rstest; + use tempfile::tempdir; + + impl<'a> RebootReasonTracker<'a> { + fn new_with_sources( + config: &'a Config, + sources: Vec, + service_manager: &'a dyn MemfaultdServiceManager, + ) -> Self { + Self { + config, + sources, + service_manager, + } + } + } + + const TEST_BOOT_ID: &str = "32c45579-8881-4a43-b7d1-f1df8invalid"; + + #[rstest] + fn test_reboot_reason_source_ordering(_setup_logger: ()) { + let mut config = Config::test_fixture(); + config.config_file.enable_data_collection = true; + + let persist_dir = tempdir().unwrap(); + config.config_file.persist_dir = + AbsolutePath::try_from(persist_dir.path().to_path_buf()).unwrap(); + + let main_source = RebootReasonSource { + name: "main", + func: |_: &Config| Some(RebootReason::from(RebootReasonCode::HardFault)), + }; + let secondary_source = RebootReasonSource { + name: "secondary", + func: |_: &Config| Some(RebootReason::from(RebootReasonCode::UserReset)), + }; + + let mut service_manager = MockMemfaultdServiceManager::new(); + service_manager + .expect_service_manager_status() + .once() + .returning(|| Ok(ServiceManagerStatus::Stopping)); + + let mar_staging_path = config.mar_staging_path(); + std::fs::create_dir_all(&mar_staging_path).expect("Failed to create mar staging dir"); + + let tracker = RebootReasonTracker::new_with_sources( + &config, + vec![main_source, secondary_source], + &service_manager, + ); + tracker + .track_reboot() + .expect("Failed to init reboot tracker"); + + // Verify that the first reboot reason source is used + verify_mar_reboot_reason( + RebootReason::from(RebootReasonCode::HardFault), + &mar_staging_path, + ); + } + + #[rstest] + fn test_reboot_reason_parsing(_setup_logger: ()) { + let mut config = Config::test_fixture(); + config.config_file.enable_data_collection = true; + + let persist_dir = tempdir().unwrap(); + config.config_file.persist_dir = + AbsolutePath::try_from(persist_dir.path().to_path_buf()).unwrap(); + + let reboot_reason = RebootReason::from(RebootReasonCode::HardFault); + + // Write values to last boot id and last reboot reason files + let last_reboot_file = persist_dir.path().join("lastrebootreason"); + std::fs::write(&last_reboot_file, reboot_reason.to_string()) + .expect("Failed to write last reboot file"); + let last_boot_id_file = persist_dir.path().join("last_tracked_boot_id"); + std::fs::write(last_boot_id_file, TEST_BOOT_ID).expect("Failed to write last boot id file"); + + let source = RebootReasonSource { + name: "test", + func: read_reboot_reason_and_clear_file_internal, + }; + + let mut service_manager = MockMemfaultdServiceManager::new(); + service_manager + .expect_service_manager_status() + .once() + .returning(|| Ok(ServiceManagerStatus::Stopping)); + + let mar_staging_path = config.mar_staging_path(); + + // Create mar staging dir + std::fs::create_dir_all(&mar_staging_path).expect("Failed to create mar staging dir"); + + let tracker = + RebootReasonTracker::new_with_sources(&config, vec![source], &service_manager); + tracker + .track_reboot() + .expect("Failed to init reboot tracker"); + + verify_mar_reboot_reason(reboot_reason, &mar_staging_path); + + // Drop tracker and ensure new reboot reason is written to file + drop(tracker); + let reboot_reason = std::fs::read_to_string(&last_reboot_file) + .expect("Failed to read last reboot file") + .parse::() + .expect("Failed to parse reboot reason"); + let reboot_reason = RebootReason::from( + RebootReasonCode::from_repr(reboot_reason).expect("Invalid reboot reason"), + ); + + assert_eq!( + reboot_reason, + RebootReason::from(RebootReasonCode::UserReset) + ); + } + + #[rstest] + fn test_custom_reboot_reason_parsing(_setup_logger: ()) { + let mut config = Config::test_fixture(); + config.config_file.enable_data_collection = true; + + let persist_dir = tempdir().unwrap(); + config.config_file.persist_dir = + AbsolutePath::try_from(persist_dir.path().to_path_buf()).unwrap(); + + let reboot_reason = RebootReason::from_str("CustomRebootReason").unwrap(); + + // Write values to last boot id and last reboot reason files + let last_reboot_file = persist_dir.path().join("lastrebootreason"); + std::fs::write(last_reboot_file, "CustomRebootReason") + .expect("Failed to write last reboot file"); + let last_boot_id_file = persist_dir.path().join("last_tracked_boot_id"); + std::fs::write(last_boot_id_file, TEST_BOOT_ID).expect("Failed to write last boot id file"); + + let source = RebootReasonSource { + name: "test", + func: read_reboot_reason_and_clear_file_internal, + }; + + let mut service_manager = MockMemfaultdServiceManager::new(); + service_manager + .expect_service_manager_status() + .once() + .returning(|| Ok(ServiceManagerStatus::Stopping)); + + let mar_staging_path = config.mar_staging_path(); + + // Create mar staging dir + std::fs::create_dir_all(&mar_staging_path).expect("Failed to create mar staging dir"); + + let tracker = + RebootReasonTracker::new_with_sources(&config, vec![source], &service_manager); + tracker + .track_reboot() + .expect("Failed to init reboot tracker"); + + verify_mar_reboot_reason(reboot_reason, &mar_staging_path); + } + + fn verify_mar_reboot_reason(reboot_reason: RebootReason, mar_staging_path: &Path) { + let mar_dir = std::fs::read_dir(mar_staging_path) + .expect("Failed to read temp dir") + .filter_map(|entry| entry.ok()) + .collect::>(); + + // There should only be an entry for the reboot reason + assert_eq!(mar_dir.len(), 1); + + let mar_manifest = mar_dir[0].path().join("manifest.json"); + let manifest_string = std::fs::read_to_string(mar_manifest).unwrap(); + let manifest: Manifest = serde_json::from_str(&manifest_string).unwrap(); + + if let Metadata::LinuxReboot { reason, .. } = manifest.metadata { + assert_eq!(reboot_reason, reason); + } else { + panic!("Unexpected metadata type"); + } + } +} diff --git a/memfaultd/src/reboot/reason.rs b/memfaultd/src/reboot/reason.rs new file mode 100644 index 0000000..3e904c4 --- /dev/null +++ b/memfaultd/src/reboot/reason.rs @@ -0,0 +1,110 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::fmt::{Debug, Display, Formatter, Result as FmtResult}; +use std::str::FromStr; + +use eyre::{eyre, ErrReport, Result}; +use serde::{Deserialize, Serialize}; + +use crate::{ + reboot::reason_codes::RebootReasonCode, util::patterns::alphanum_slug_dots_colon_is_valid, +}; + +#[derive(Clone, PartialEq, Eq, Deserialize, Serialize)] +#[serde(untagged)] +pub enum RebootReason { + Code(RebootReasonCode), + Custom(RebootReasonString), +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +pub struct RebootReasonString { + unexpected: bool, + name: String, +} + +impl From for RebootReason { + fn from(code: RebootReasonCode) -> RebootReason { + RebootReason::Code(code) + } +} + +impl FromStr for RebootReasonString { + type Err = ErrReport; + fn from_str(s: &str) -> Result { + // Leading '!' indicates an unexpected reboot reason + if let Some(stripped) = s.strip_prefix('!') { + if stripped.is_empty() { + Err(eyre!("\"!\" on its own is not a valid reboot reaosn!")) + } else { + match alphanum_slug_dots_colon_is_valid(stripped, 64) { + Ok(()) => Ok(RebootReasonString { + unexpected: true, + name: stripped.to_string(), + }), + Err(e) => Err(e), + } + } + } else { + match alphanum_slug_dots_colon_is_valid(s, 64) { + Ok(()) => Ok(RebootReasonString { + unexpected: false, + name: s.to_string(), + }), + Err(e) => Err(e), + } + } + } +} + +impl FromStr for RebootReason { + type Err = ErrReport; + + fn from_str(s: &str) -> Result { + match u32::from_str(s) { + Ok(code) => match RebootReasonCode::from_repr(code) { + Some(reset_code) => Ok(RebootReason::Code(reset_code)), + None => Ok(RebootReason::Code(RebootReasonCode::Unknown)), + }, + // If the reboot reason isn't parse-able to a u32, it's custom + Err(_) => match RebootReasonString::from_str(s) { + Ok(reason) => Ok(RebootReason::Custom(reason)), + Err(e) => Err(eyre!("Failed to parse custom reboot reason: {}", e)), + }, + } + } +} + +impl Display for RebootReason { + fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { + match self { + RebootReason::Code(c) => write!(f, "{}", (*c as u32)), + RebootReason::Custom(RebootReasonString { unexpected, name }) => { + if *unexpected { + write!(f, "!{}", name) + } else { + write!(f, "{}", name) + } + } + } + } +} + +impl Debug for RebootReason { + fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { + match self { + RebootReason::Code(c) => write!(f, "{} ({})", (*c as u32), c), + RebootReason::Custom(RebootReasonString { unexpected, name }) => write!( + f, + "{} ({})", + name, + if *unexpected { + "unexpected reboot" + } else { + "expected reboot" + } + ), + } + } +} diff --git a/memfaultd/src/reboot/reason_codes.rs b/memfaultd/src/reboot/reason_codes.rs new file mode 100644 index 0000000..c5c14f6 --- /dev/null +++ b/memfaultd/src/reboot/reason_codes.rs @@ -0,0 +1,85 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use serde_repr::{Deserialize_repr, Serialize_repr}; +use strum_macros::{Display, EnumString, FromRepr}; + +/// Definitions for reboot reasons +/// +/// See the [Memfault docs](https://docs.memfault.com/docs/platform/reference-reboot-reason-ids/) +/// for more details. +#[derive( + Debug, + Clone, + Copy, + EnumString, + Display, + Serialize_repr, + Deserialize_repr, + PartialEq, + Eq, + FromRepr, +)] +#[repr(u32)] +pub enum RebootReasonCode { + Unknown = 0x0000, + + // + // Normal Resets + // + UserShutdown = 0x0001, + UserReset = 0x0002, + FirmwareUpdate = 0x0003, + LowPower = 0x0004, + DebuggerHalted = 0x0005, + ButtonReset = 0x0006, + PowerOnReset = 0x0007, + SoftwareReset = 0x0008, + + /// MCU went through a full reboot due to exit from lowest power state + DeepSleep = 0x0009, + /// MCU reset pin was toggled + PinReset = 0x000A, + + // + // Error Resets + // + /// Can be used to flag an unexpected reset path. i.e NVIC_SystemReset() being called without any + /// reboot logic getting invoked. + UnknownError = 0x8000, + Assert = 0x8001, + + /// Deprecated in favor of HardwareWatchdog & SoftwareWatchdog. + /// + /// This way, the amount of watchdogs not caught by software can be easily tracked. + WatchdogDeprecated = 0x8002, + + BrownOutReset = 0x8003, + Nmi = 0x8004, // Non-Maskable Interrupt + + // More details about nomenclature in https://mflt.io/root-cause-watchdogs + HardwareWatchdog = 0x8005, + SoftwareWatchdog = 0x8006, + + /// A reset triggered due to the MCU losing a stable clock. + /// + /// This can happen, for example, if power to the clock is cut or the lock for the PLL is lost. + ClockFailure = 0x8007, + + /// A software reset triggered when the OS or RTOS end-user code is running on top of identifies + /// a fatal error condition. + KernelPanic = 0x8008, + + /// A reset triggered when an attempt to upgrade to a new OTA image has failed and a rollback + /// to a previous version was initiated + FirmwareUpdateError = 0x8009, + + // Resets from Arm Faults + BusFault = 0x9100, + MemFault = 0x9200, + UsageFault = 0x9300, + HardFault = 0x9400, + /// A reset which is triggered when the processor faults while already + /// executing from a fault handler. + Lockup = 0x9401, +} diff --git a/memfaultd/src/retriable_error.rs b/memfaultd/src/retriable_error.rs new file mode 100644 index 0000000..547698f --- /dev/null +++ b/memfaultd/src/retriable_error.rs @@ -0,0 +1,65 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::{ErrReport, Report, Result}; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum RetriableError { + #[error("Temporary server error: {status_code}")] + ServerError { status_code: u16 }, + #[error("Network error ({source})")] + NetworkError { source: reqwest::Error }, +} + +pub trait IgnoreNonRetriableError { + /// Ignore non-retriable errors, turning them into `Ok(None)`. + /// If the Err holds a RetriableError, it will be returned as-is. + fn ignore_non_retriable_errors_with(self, x: R) -> Result<(), ErrReport>; +} + +impl IgnoreNonRetriableError for Result { + fn ignore_non_retriable_errors_with( + self, + mut on_error: R, + ) -> Result<(), ErrReport> { + match self { + Ok(_) => Ok(()), + Err(e) => { + if e.downcast_ref::().is_some() { + Err(e) + } else { + on_error(&e); + Ok(()) + } + } + } + } +} + +#[cfg(test)] +mod tests { + use eyre::eyre; + use rstest::*; + + use super::*; + + #[rstest] + #[case(Ok(()), false, true)] + #[case(Err(eyre!("Some error")), true, true)] + #[case(Err(eyre!(RetriableError::ServerError { status_code: 503 })), false, false)] + fn test_ignore_non_retriable_errors_with( + #[case] result: Result<(), Report>, + #[case] expected_called: bool, + #[case] expected_ok: bool, + ) { + let mut called = false; + assert_eq!( + result + .ignore_non_retriable_errors_with(|_| called = true) + .is_ok(), + expected_ok + ); + assert_eq!(called, expected_called); + } +} diff --git a/memfaultd/src/service_manager/default.rs b/memfaultd/src/service_manager/default.rs new file mode 100644 index 0000000..3dec942 --- /dev/null +++ b/memfaultd/src/service_manager/default.rs @@ -0,0 +1,20 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use nix::sys::signal::Signal::SIGHUP; + +use crate::util::pid_file::send_signal_to_pid; + +use super::MemfaultdServiceManager; + +pub struct DefaultServiceManager; + +impl MemfaultdServiceManager for DefaultServiceManager { + fn restart_memfaultd_if_running(&self) -> eyre::Result<()> { + send_signal_to_pid(SIGHUP) + } + + fn service_manager_status(&self) -> eyre::Result { + Ok(super::ServiceManagerStatus::Unknown) + } +} diff --git a/memfaultd/src/service_manager/mod.rs b/memfaultd/src/service_manager/mod.rs new file mode 100644 index 0000000..216cdbe --- /dev/null +++ b/memfaultd/src/service_manager/mod.rs @@ -0,0 +1,65 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Memfaultd service management +//! +//! This module contains the trait for managing memfaultd services, as well as +//! the implementation for systemd. +//! + +mod default; +#[cfg(feature = "systemd")] +mod systemd; + +/// Return the system manager that was configured at build time. +pub fn get_service_manager() -> impl MemfaultdServiceManager { + #[cfg(feature = "systemd")] + { + use systemd::SystemdServiceManager; + SystemdServiceManager + } + #[cfg(not(feature = "systemd"))] + { + use default::DefaultServiceManager; + DefaultServiceManager + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ServiceManagerStatus { + Starting, + Running, + Stopping, + Stopped, + Unknown, +} + +/// Trait for managing memfaultd services +/// +/// This trait is implemented for different service managers, such as systemd. +#[cfg_attr(test, mockall::automock)] +pub trait MemfaultdServiceManager { + fn restart_memfaultd_if_running(&self) -> eyre::Result<()>; + fn service_manager_status(&self) -> eyre::Result; +} + +impl TryFrom<&str> for ServiceManagerStatus { + type Error = eyre::Error; + + fn try_from(status: &str) -> Result { + let status = match status { + "starting" => ServiceManagerStatus::Starting, + "running" => ServiceManagerStatus::Running, + "stopping" => ServiceManagerStatus::Stopping, + "stopped" => ServiceManagerStatus::Stopped, + _ => { + return Err(eyre::eyre!( + "Unknown systemd service manager status: {}", + status + )) + } + }; + + Ok(status) + } +} diff --git a/memfaultd/src/service_manager/systemd.rs b/memfaultd/src/service_manager/systemd.rs new file mode 100644 index 0000000..9f640e7 --- /dev/null +++ b/memfaultd/src/service_manager/systemd.rs @@ -0,0 +1,42 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::ffi::{CStr, CString}; + +use crate::service_manager::{MemfaultdServiceManager, ServiceManagerStatus}; +use memfaultc_sys::systemd::{ + memfaultd_get_systemd_bus_state, memfaultd_restart_systemd_service_if_running, +}; + +/// Systemd service manager +/// +/// This service manager uses the systemd D-Bus API to manage services. +pub struct SystemdServiceManager; + +impl MemfaultdServiceManager for SystemdServiceManager { + fn restart_memfaultd_if_running(&self) -> eyre::Result<()> { + let service_cstring = CString::new("memfaultd.service")?; + let restart_result = + unsafe { memfaultd_restart_systemd_service_if_running(service_cstring.as_ptr()) }; + + if restart_result { + Ok(()) + } else { + Err(eyre::eyre!("Failed to restart memfaultd service")) + } + } + + fn service_manager_status(&self) -> eyre::Result { + let status_ptr = unsafe { memfaultd_get_systemd_bus_state() }; + if status_ptr.is_null() { + return Err(eyre::eyre!("Failed to get systemd service bus state")); + } + + let status_str = unsafe { CStr::from_ptr(status_ptr).to_str()? }; + let status = ServiceManagerStatus::try_from(status_str)?; + + unsafe { libc::free(status_ptr as *mut libc::c_void) }; + + Ok(status) + } +} diff --git a/memfaultd/src/swupdate/config.rs b/memfaultd/src/swupdate/config.rs new file mode 100644 index 0000000..36b95cb --- /dev/null +++ b/memfaultd/src/swupdate/config.rs @@ -0,0 +1,51 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::{eyre, Result}; +use std::{ffi::CString, os::unix::prelude::OsStrExt}; + +use memfaultc_sys::swupdate::{memfault_swupdate_generate_config, MemfaultSwupdateCtx}; + +use crate::config::Config; + +pub fn generate_swupdate_config(config: &Config) -> Result<()> { + let base_url_cstring = CString::new(config.config_file.base_url.clone())?; + let software_version_cstring = CString::new(config.software_version())?; + let software_type_cstring = CString::new(config.software_type())?; + let hardware_version_cstring = CString::new(config.device_info.hardware_version.clone())?; + let device_id_cstring = CString::new(config.device_info.device_id.clone())?; + let project_key_cstring = CString::new(config.config_file.project_key.clone())?; + + let input_file_cstring = CString::new( + config + .config_file + .swupdate + .input_file + .as_os_str() + .as_bytes(), + )?; + let output_file_cstring = CString::new( + config + .config_file + .swupdate + .output_file + .as_os_str() + .as_bytes(), + )?; + + let ctx = MemfaultSwupdateCtx { + base_url: base_url_cstring.as_ptr(), + software_version: software_version_cstring.as_ptr(), + software_type: software_type_cstring.as_ptr(), + hardware_version: hardware_version_cstring.as_ptr(), + device_id: device_id_cstring.as_ptr(), + project_key: project_key_cstring.as_ptr(), + + input_file: input_file_cstring.as_ptr(), + output_file: output_file_cstring.as_ptr(), + }; + match unsafe { memfault_swupdate_generate_config(&ctx) } { + true => Ok(()), + false => Err(eyre!("Unable to prepare swupdate config.")), + } +} diff --git a/memfaultd/src/swupdate/mod.rs b/memfaultd/src/swupdate/mod.rs new file mode 100644 index 0000000..7c26353 --- /dev/null +++ b/memfaultd/src/swupdate/mod.rs @@ -0,0 +1,6 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +mod config; + +pub use config::generate_swupdate_config; diff --git a/memfaultd/src/test_utils.rs b/memfaultd/src/test_utils.rs new file mode 100644 index 0000000..13fbcf0 --- /dev/null +++ b/memfaultd/src/test_utils.rs @@ -0,0 +1,131 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Memfault Test Utils +//! +//! A collection of useful structs and functions for unit and integration testing. +//! + +use std::cmp::min; +use std::path::Path; +use std::str::FromStr; +use std::{ + fs::File, + io::{ErrorKind, Seek, Write}, +}; + +use rstest::fixture; + +mod test_instant; +pub use test_instant::*; + +mod test_connection_checker; +pub use test_connection_checker::*; + +use crate::metrics::{KeyedMetricReading, MetricReading, MetricStringKey, MetricTimestamp}; + +/// A file that will trigger write errors when it reaches a certain size. +/// Note that we currently enforce the limit on the total number of bytes +/// written, regardless of where they were written. We do implement Seek but do +/// not try to keep track of the file size. +pub struct SizeLimitedFile { + file: File, + limit: usize, + written: usize, +} + +impl SizeLimitedFile { + /// Create a new SizeLimitedFile which will write to file until limit is + /// reached. + #[allow(dead_code)] + pub fn new(file: File, limit: usize) -> Self { + Self { + file, + limit, + written: 0, + } + } +} + +impl Write for SizeLimitedFile { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let bytes_to_write = buf.len().min(self.limit - self.written); + + if bytes_to_write == 0 { + Err(std::io::Error::new( + ErrorKind::WriteZero, + "File size limit reached", + )) + } else { + self.written += bytes_to_write; + self.file.write(&buf[..bytes_to_write]) + } + } + fn flush(&mut self) -> std::io::Result<()> { + self.file.flush() + } +} + +impl Seek for SizeLimitedFile { + fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { + self.file.seek(pos) + } +} + +pub fn create_file_with_size(path: &Path, size: u64) -> std::io::Result<()> { + let mut file = File::create(path)?; + let buffer = vec![0; min(4096, size as usize)]; + let mut remaining = size; + while remaining > 0 { + let bytes_to_write = min(remaining, buffer.len() as u64); + file.write_all(&buffer[..bytes_to_write as usize])?; + remaining -= bytes_to_write; + } + Ok(()) +} + +pub fn create_file_with_contents(path: &Path, contents: &[u8]) -> std::io::Result<()> { + let mut file = File::create(path)?; + file.write_all(contents)?; + Ok(()) +} + +#[fixture] +/// Simple fixture to add to a test when you want the logger to work. +pub fn setup_logger() { + let _ = stderrlog::new().module("memfaultd").verbosity(10).init(); +} + +/// Macro to set the snapshot suffix for a test. +/// +/// This is a workaround suggested by the insta docs to scope the snapshot suffix +/// to a test when using rstest cases. See docs +/// [here](https://insta.rs/docs/patterns/). +macro_rules! set_snapshot_suffix { + ($($expr:expr),*) => { + let mut settings = insta::Settings::clone_current(); + settings.set_snapshot_suffix(format!($($expr,)*)); + let _guard = settings.bind_to_scope(); + } +} + +#[cfg(test)] +/// Constructs an iterator of Gauge metric readings for tests +/// to easily generate mock data +pub fn in_histograms( + metrics: Vec<(&'static str, f64)>, +) -> impl Iterator { + metrics + .into_iter() + .enumerate() + .map(|(i, (name, value))| KeyedMetricReading { + name: MetricStringKey::from_str(name).unwrap(), + value: MetricReading::Histogram { + value, + timestamp: MetricTimestamp::from_str("2021-01-01T00:00:00Z").unwrap() + + chrono::Duration::seconds(i as i64), + }, + }) +} + +pub(crate) use set_snapshot_suffix; diff --git a/memfaultd/src/test_utils/test_connection_checker.rs b/memfaultd/src/test_utils/test_connection_checker.rs new file mode 100644 index 0000000..06ccdbe --- /dev/null +++ b/memfaultd/src/test_utils/test_connection_checker.rs @@ -0,0 +1,41 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{cell::RefCell, net::IpAddr, time::Duration}; + +use eyre::{eyre, Result}; + +use crate::util::can_connect::CanConnect; + +#[derive(Debug, Clone, Copy)] +pub struct TestConnectionChecker {} + +thread_local! { + static CONNECTED: RefCell = RefCell::new(true); +} + +impl TestConnectionChecker { + pub fn connect() { + CONNECTED.with(|c| *c.borrow_mut() = true); + } + + pub fn disconnect() { + CONNECTED.with(|c| *c.borrow_mut() = false); + } +} + +impl CanConnect for TestConnectionChecker { + fn new(_timeout: Duration) -> Self { + Self {} + } + + fn can_connect(&self, _ip: &IpAddr, _port: u16) -> Result<()> { + let mut connected_status = true; + CONNECTED.with(|c| connected_status = *c.borrow()); + if connected_status { + Ok(()) + } else { + Err(eyre!("MockPinger is set to disconnected")) + } + } +} diff --git a/memfaultd/src/test_utils/test_instant.rs b/memfaultd/src/test_utils/test_instant.rs new file mode 100644 index 0000000..a5ac154 --- /dev/null +++ b/memfaultd/src/test_utils/test_instant.rs @@ -0,0 +1,82 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{cell::RefCell, ops::Add, ops::Sub, time::Duration}; + +use crate::util::time_measure::TimeMeasure; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct TestInstant { + t0: Duration, +} + +thread_local! { + static TIME: RefCell = RefCell::new(Duration::from_secs(0)); +} + +impl TestInstant { + pub fn now() -> Self { + let mut now = Duration::from_secs(0); + TIME.with(|t| now = *t.borrow()); + TestInstant { t0: now } + } + + pub fn from(d: Duration) -> Self { + TestInstant { t0: d } + } + + pub fn sleep(d: Duration) { + TIME.with(|t| { + let new_time = t.borrow().saturating_add(d); + *t.borrow_mut() = new_time; + }) + } +} + +impl TimeMeasure for TestInstant { + fn now() -> Self { + Self::now() + } + fn elapsed(&self) -> Duration { + Self::now().t0 - self.t0 + } + + fn since(&self, other: &Self) -> Duration { + self.t0.sub(other.t0) + } +} + +impl Add for TestInstant { + type Output = TestInstant; + + fn add(self, rhs: Duration) -> Self::Output { + TestInstant { + t0: self.t0.add(rhs), + } + } +} +impl Sub for TestInstant { + type Output = TestInstant; + + fn sub(self, rhs: Duration) -> Self::Output { + TestInstant { + t0: self.t0.sub(rhs), + } + } +} + +impl Add for TestInstant { + type Output = Duration; + + fn add(self, rhs: TestInstant) -> Duration { + self.t0.add(rhs.t0) + } +} + +impl Sub for TestInstant { + type Output = Duration; + + fn sub(self, rhs: TestInstant) -> Duration { + self.t0.sub(rhs.t0) + } +} diff --git a/memfaultd/src/util/can_connect.rs b/memfaultd/src/util/can_connect.rs new file mode 100644 index 0000000..321a5c1 --- /dev/null +++ b/memfaultd/src/util/can_connect.rs @@ -0,0 +1,78 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + net::{IpAddr, Shutdown, SocketAddr, TcpStream}, + time::Duration, +}; + +use eyre::Result; + +pub trait CanConnect { + fn new(timeout: Duration) -> Self; + fn can_connect(&self, ip_addr: &IpAddr, port: u16) -> Result<()>; +} + +pub struct TcpConnectionChecker { + timeout: Duration, +} + +impl CanConnect for TcpConnectionChecker { + fn new(timeout: Duration) -> Self { + Self { timeout } + } + + fn can_connect(&self, ip_addr: &IpAddr, port: u16) -> Result<()> { + let socket = SocketAddr::new(*ip_addr, port); + let stream = TcpStream::connect_timeout(&socket, self.timeout)?; + stream.shutdown(Shutdown::Both)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::{ + net::{IpAddr, TcpListener}, + str::FromStr, + time::Duration, + }; + + use rstest::rstest; + + use super::{CanConnect, TcpConnectionChecker}; + + #[rstest] + fn test_localhost_reachable() { + let ip_str = "127.0.0.1"; + let port = 19443; + + // Need a TCP listener to be running on the port + // TcpConnectionChecker will try to create a connection with + let listener = TcpListener::bind(format!("{}:{}", ip_str, port)) + .expect("Could not create TcpListener for testing!"); + + let connection_checker = TcpConnectionChecker::new(Duration::from_secs(10)); + assert!(connection_checker + .can_connect( + &IpAddr::from_str(ip_str) + .unwrap_or_else(|_| panic!("{} should be parseable to an IP Address", ip_str)), + port + ) + .is_ok()); + drop(listener); + } + + #[rstest] + fn test_unreachable_ip_errors() { + let ip_str = "127.0.0.1"; + let connection_checker = TcpConnectionChecker::new(Duration::from_secs(10)); + assert!(connection_checker + .can_connect( + &IpAddr::from_str(ip_str) + .unwrap_or_else(|_| panic!("{} should be parseable to an IP Address", ip_str)), + 443 + ) + .is_err()); + } +} diff --git a/memfaultd/src/util/circular_queue.rs b/memfaultd/src/util/circular_queue.rs new file mode 100644 index 0000000..c351aab --- /dev/null +++ b/memfaultd/src/util/circular_queue.rs @@ -0,0 +1,121 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! A simple circular queue implementation +//! +//! This module provides a simple circular queue implementation. The buffer has a +//! fixed capacity and will overwrite the oldest item when it is full. + +use std::collections::VecDeque; + +#[derive(Debug, Default, Clone)] +pub struct CircularQueue { + buffer: VecDeque, + capacity: usize, +} + +impl CircularQueue { + /// Create a new circular queue with the given capacity + pub fn new(capacity: usize) -> Self { + CircularQueue { + buffer: VecDeque::with_capacity(capacity + 1), + capacity, + } + } + + /// Push an item onto the buffer. + /// + /// If the buffer is full, the oldest item will be removed. + pub fn push(&mut self, item: T) { + self.buffer.push_back(item); + + // If the buffer is full, remove the oldest item + if self.buffer.len() > self.capacity { + self.buffer.pop_front(); + } + } + + /// Pop an item from the buffer. + pub fn pop(&mut self) -> Option { + self.buffer.pop_front() + } + + /// Return true if the buffer is empty + pub fn is_empty(&self) -> bool { + self.buffer.is_empty() + } + + /// Return true if the buffer is full + pub fn is_full(&self) -> bool { + self.buffer.len() == self.capacity + } + + /// Return the number of items in the buffer + pub fn len(&self) -> usize { + self.buffer.len() + } + + /// Return the buffer capacity + pub fn capacity(&self) -> usize { + self.capacity + } + + /// Return a reference to the back item in the buffer + pub fn back(&self) -> Option<&T> { + self.buffer.back() + } + + pub fn iter(&self) -> impl Iterator { + self.buffer.iter() + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_ring_buffer() { + let mut rb = CircularQueue::new(3); + assert_eq!(rb.len(), 0); + assert_eq!(rb.capacity(), 3); + assert!(rb.is_empty()); + + rb.push(1); + assert_eq!(rb.len(), 1); + assert_eq!(rb.pop(), Some(1)); + assert!(rb.is_empty()); + + rb.push(2); + rb.push(3); + rb.push(4); + assert_eq!(rb.len(), 3); + assert_eq!(rb.pop(), Some(2)); + assert_eq!(rb.pop(), Some(3)); + assert_eq!(rb.pop(), Some(4)); + assert!(rb.is_empty()); + } + + #[test] + fn test_wrap_around() { + let mut rb = CircularQueue::new(3); + rb.push(1); + rb.push(2); + rb.push(3); + assert_eq!(rb.len(), 3); + + rb.push(4); + rb.push(5); + assert_eq!(rb.len(), 3); + assert_eq!(rb.pop(), Some(3)); + assert_eq!(rb.pop(), Some(4)); + assert_eq!(rb.pop(), Some(5)); + assert!(rb.is_empty()); + } + + #[test] + fn empty_pop() { + let mut rb: CircularQueue = CircularQueue::new(3); + assert_eq!(rb.pop(), None); + } +} diff --git a/memfaultd/src/util/die.rs b/memfaultd/src/util/die.rs new file mode 100644 index 0000000..bbb3941 --- /dev/null +++ b/memfaultd/src/util/die.rs @@ -0,0 +1,26 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! In loving memory of the practical extraction and report language. +use std::error::Error; + +use log::error; + +/// Prints the error message to the error log, and then panics. +pub fn die(e: E) -> ! { + error!("Irrecoverable error: {:#}", e); + panic!("Irrecoverable error: {:#}", e) +} + +pub trait UnwrapOrDie { + fn unwrap_or_die(self) -> T; +} + +impl UnwrapOrDie for Result { + fn unwrap_or_die(self) -> T { + match self { + Ok(v) => v, + Err(e) => die(e), + } + } +} diff --git a/memfaultd/src/util/disk_backed.rs b/memfaultd/src/util/disk_backed.rs new file mode 100644 index 0000000..fd39eb1 --- /dev/null +++ b/memfaultd/src/util/disk_backed.rs @@ -0,0 +1,221 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::Result; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use std::{ + fs::File, + io::BufReader, + path::{Path, PathBuf}, +}; + +/// A serializable struct that is backed by a JSON file on disk. +/// +/// The disk version will be loaded on object creation. If it does not exist, +/// or is invalid, the default value will be returned. +pub struct DiskBacked> { + path: PathBuf, + cache: Option, + default: T, +} + +#[derive(Debug, Eq, PartialEq)] +pub enum UpdateStatus { + Unchanged, + Updated, +} + +impl DiskBacked { + /// New instance from a path on disk + pub fn from_path(path: &Path) -> Self { + Self::from_path_with_default(path, T::default()) + } +} + +impl DiskBacked { + pub fn from_path_with_default(path: &Path, default: T) -> Self { + let cache = match File::open(path) { + Ok(file) => { + let reader = BufReader::new(file); + serde_json::from_reader(reader).ok() + } + Err(_) => None, + }; + Self { + path: path.to_owned(), + cache, + default, + } + } + + /// Return current value (disk value or default). + pub fn get(&self) -> &T { + match self.cache.as_ref() { + Some(v) => v, + None => &self.default, + } + } + + /// Updates self and writes the provided value to disk. + pub fn set(&mut self, new_value: T) -> Result { + let has_changed_from_previous_effective_value = match self.cache.as_ref() { + Some(v) => new_value != *v, + None => new_value != self.default, + }; + let needs_writing = match &self.cache { + Some(v) => v != &new_value, + None => true, + }; + + if needs_writing { + let file = File::create(&self.path)?; + serde_json::to_writer_pretty(file, &new_value)?; + self.cache = Some(new_value); + } + + Ok(match has_changed_from_previous_effective_value { + true => UpdateStatus::Updated, + false => UpdateStatus::Unchanged, + }) + } +} + +#[cfg(test)] +mod tests { + use std::{borrow::Cow, io::BufReader, path::PathBuf}; + + use rstest::{fixture, rstest}; + + use crate::test_utils::create_file_with_contents; + + use super::*; + + #[rstest] + fn test_defaults_to_default(fixture: Fixture) { + assert_eq!( + *DiskBacked::::from_path(&fixture.path).get(), + TestJson::default() + ); + } + + #[rstest] + fn test_load_from_disk(#[with(Some(TEST1))] fixture: Fixture) { + let config = DiskBacked::::from_path(&fixture.path); + assert_eq!(*config.get(), TEST1); + } + + #[rstest] + fn test_write_with_no_existing_file_and_new_equals_default(#[with(None)] fixture: Fixture) { + let mut config = DiskBacked::::from_path(&fixture.path); + + let result = config.set(TestJson::default()); + + assert!(matches!(result, Ok(UpdateStatus::Unchanged))); + + // We should have created a file on disk and stored the config + assert_eq!(fixture.read_config(), Some(TestJson::default())); + } + + #[rstest] + fn test_write_with_no_existing_file_and_new_is_not_default(#[with(None)] fixture: Fixture) { + let mut config = DiskBacked::::from_path(&fixture.path); + + let result = config.set(TEST1); + + assert!(matches!(result, Ok(UpdateStatus::Updated))); + + // We should have created a file on disk and stored the config + assert_eq!(fixture.read_config(), Some(TEST1)); + } + + #[rstest] + fn test_write_with_corrupted_local_file(#[with(None)] fixture: Fixture) { + create_file_with_contents(&fixture.path, "DIS*IS*NOT*JSON".as_bytes()).unwrap(); + let mut config = DiskBacked::::from_path(&fixture.path); + + let result = config.set(TestJson::default()); + + // Unchanged because we use default when file is corrupted + assert!(matches!(result, Ok(UpdateStatus::Unchanged))); + + // We should have created a file on disk and stored the config + assert_eq!(fixture.read_config(), Some(TestJson::default())); + } + + #[rstest] + fn test_write_without_change(#[with(Some(TEST1))] fixture: Fixture) { + let mut config = DiskBacked::::from_path(&fixture.path); + + // Delete the config file so we can see if it has been recreated + std::fs::remove_file(&fixture.path).unwrap(); + + let result = config.set(TEST1); + + assert!(matches!(result, Ok(UpdateStatus::Unchanged))); + + // We should NOT have re-created a file on disk and stored the config + assert_eq!(fixture.read_config(), None); + } + + #[rstest] + fn test_write_with_change(#[with(Some(TEST1))] fixture: Fixture) { + let mut config = DiskBacked::::from_path(&fixture.path); + + // Delete the config file so we can see if it has been recreated + std::fs::remove_file(&fixture.path).unwrap(); + + let result = config.set(TestJson::default()); + + assert!(matches!(result, Ok(UpdateStatus::Updated))); + + assert_eq!(fixture.read_config(), Some(TestJson::default())); + } + + #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Default, Debug)] + struct TestJson<'a> { + pub message: Cow<'a, str>, + } + + const TEST1: TestJson = TestJson { + message: Cow::Borrowed("test1"), + }; + + #[fixture] + fn fixture(#[default(None)] config: Option) -> Fixture { + Fixture::new(config) + } + + struct Fixture { + _temp_dir: tempfile::TempDir, + path: PathBuf, + } + + impl Fixture { + fn new(preexisting: Option) -> Self { + let temp_dir = tempfile::tempdir().unwrap(); + let path = temp_dir.path().join("data.json"); + + if let Some(value) = preexisting { + let file = File::create(&path).unwrap(); + serde_json::to_writer_pretty(file, &value).unwrap(); + } + + Self { + _temp_dir: temp_dir, + path, + } + } + + fn read_config(&self) -> Option { + let file = match File::open(&self.path) { + Ok(file) => file, + Err(_) => return None, + }; + let reader = BufReader::new(file); + match serde_json::from_reader(reader) { + Ok(config) => Some(config), + Err(_) => None, + } + } + } +} diff --git a/memfaultd/src/util/disk_size.rs b/memfaultd/src/util/disk_size.rs new file mode 100644 index 0000000..c36a27a --- /dev/null +++ b/memfaultd/src/util/disk_size.rs @@ -0,0 +1,176 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + ffi::CString, + fs::{read_dir, Metadata}, + mem, + ops::{Add, AddAssign}, + os::unix::prelude::OsStrExt, + path::Path, +}; + +use eyre::{eyre, Result}; + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Disk space information in bytes and inodes. +pub struct DiskSize { + /// Bytes on disk + pub bytes: u64, + /// Number of inodes + pub inodes: u64, +} + +impl DiskSize { + pub fn new_capacity(bytes: u64) -> Self { + Self { + bytes, + inodes: u64::MAX, + } + } + + pub const ZERO: Self = Self { + bytes: 0, + inodes: 0, + }; + + pub fn min(a: Self, b: Self) -> Self { + Self { + bytes: a.bytes.min(b.bytes), + inodes: a.inodes.min(b.inodes), + } + } + + pub fn max(a: Self, b: Self) -> Self { + Self { + bytes: a.bytes.max(b.bytes), + inodes: a.inodes.max(b.inodes), + } + } + + pub fn exceeds(&self, other: &Self) -> bool { + (self.bytes != other.bytes || self.inodes != other.inodes) + && self.bytes >= other.bytes + && self.inodes >= other.inodes + } + + pub fn saturating_sub(self, other: Self) -> Self { + Self { + bytes: self.bytes.saturating_sub(other.bytes), + inodes: self.inodes.saturating_sub(other.inodes), + } + } +} + +impl Add for DiskSize { + type Output = Self; + + fn add(self, other: Self) -> Self { + Self { + bytes: self.bytes + other.bytes, + inodes: self.inodes + other.inodes, + } + } +} + +impl AddAssign for DiskSize { + fn add_assign(&mut self, rhs: Self) { + self.bytes += rhs.bytes; + self.inodes += rhs.inodes; + } +} + +impl From for DiskSize { + fn from(metadata: Metadata) -> Self { + Self { + bytes: metadata.len(), + inodes: 1, + } + } +} + +// We need to cast to u64 here on some platforms. +#[allow(clippy::unnecessary_cast)] +pub fn get_disk_space(path: &Path) -> Result { + let mut stat: libc::statvfs = unsafe { mem::zeroed() }; + let cpath = CString::new(path.as_os_str().as_bytes()).map_err(|_| eyre!("Invalid path"))?; + // danburkert/fs2-rs#1: cast is necessary for platforms where c_char != u8. + if unsafe { libc::statvfs(cpath.as_ptr() as *const _, &mut stat) } != 0 { + Err(eyre!("Unable to call statvfs")) + } else { + Ok(DiskSize { + // Note that we use f_bavail/f_favail instead of f_bfree/f_bavail. + // f_bfree is the number of free blocks available to the + // superuser, but we want to stop before getting to that + // point. [bf]avail is what is available to normal users. + bytes: stat.f_frsize as u64 * stat.f_bavail as u64, + inodes: stat.f_favail as u64, + }) + } +} + +/// fs_extra::get_size but also returning the number of inodes +pub fn get_size

(path: P) -> Result +where + P: AsRef, +{ + // Using `fs::symlink_metadata` since we don't want to follow symlinks, + // as we're calculating the exact size of the requested path itself. + let path_metadata = path.as_ref().symlink_metadata()?; + + let mut size = DiskSize::ZERO; + + if path_metadata.is_dir() { + for entry in read_dir(&path)? { + let entry = entry?; + // `DirEntry::metadata` does not follow symlinks (unlike `fs::metadata`), so in the + // case of symlinks, this is the size of the symlink itself, not its target. + let entry_metadata = entry.metadata()?; + + if entry_metadata.is_dir() { + // The size of the directory entry itself will be counted inside the `get_size()` call, + // so we intentionally don't also add `entry_metadata.len()` to the total here. + size += get_size(entry.path())?; + } else { + size += entry_metadata.into(); + } + } + } else { + size = path_metadata.into(); + } + + Ok(size) +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + #[rstest] + #[case(0, 0, 0, 0, false)] + // When bytes and inodes are greater + #[case(1024, 1, 0, 0, true)] + // When bytes and inodes are lesser + #[case(1024, 10, 2048, 20, false)] + // When bytes or inodes are greater + #[case(1024, 100, 2048, 20, false)] + #[case(4096, 10, 2048, 20, false)] + // When bytes are equal and inodes are greater + #[case(1024, 100, 1024, 10, true)] + fn test_size_cmp( + #[case] bytes: u64, + #[case] inodes: u64, + #[case] free_bytes: u64, + #[case] free_inodes: u64, + + #[case] exceeds: bool, + ) { + let size1 = DiskSize { bytes, inodes }; + let size2 = DiskSize { + bytes: free_bytes, + inodes: free_inodes, + }; + assert_eq!(size1.exceeds(&size2), exceeds); + } +} diff --git a/memfaultd/src/util/etc_os_release.rs b/memfaultd/src/util/etc_os_release.rs new file mode 100644 index 0000000..bd4e706 --- /dev/null +++ b/memfaultd/src/util/etc_os_release.rs @@ -0,0 +1,107 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Parser for /etc/os-release. +//! +//! This file is used by various Linux distributions to store information about the operating system. +//! See documentation for more details: https://www.freedesktop.org/software/systemd/man/latest/os-release.html +//! Currently only the `ID` and `VERSION_ID` fields are parsed. Below is an example of the file: +//! +//! ```text +//! PRETTY_NAME="Ubuntu 22.04.3 LTS" +//! NAME="Ubuntu" +//! VERSION_ID="22.04" +//! VERSION="22.04.3 LTS (Jammy Jellyfish)" +//! VERSION_CODENAME=jammy +//! ID=ubuntu +//! ID_LIKE=debian +//! HOME_URL="https://www.ubuntu.com/" +//! SUPPORT_URL="https://help.ubuntu.com/" +//! BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" +//! PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" +//! UBUNTU_CODENAME=jammy +//! ``` + +use eyre::Result; +use nom::{ + bytes::complete::{tag, take_until}, + character::complete::not_line_ending, + sequence::separated_pair, + IResult, +}; + +use std::collections::HashMap; + +const OS_RELEASE_PATH: &str = "/etc/os-release"; +const OS_RELEASE_ID: &str = "ID"; +const OS_RELEASE_VERSION_ID: &str = "VERSION_ID"; + +pub struct EtcOsRelease { + id: Option, + version_id: Option, +} + +impl EtcOsRelease { + pub fn load() -> Result { + let etc_os_release_str = std::fs::read_to_string(OS_RELEASE_PATH)?; + Self::parse(&etc_os_release_str) + } + + pub fn id(&self) -> Option { + self.id.clone() + } + + pub fn version_id(&self) -> Option { + self.version_id.clone() + } + + fn parse(etc_os_release_str: &str) -> Result { + let parsed_map = etc_os_release_str + .trim() + .lines() + .filter_map(|line| { + let parse_result: IResult<&str, (&str, &str)> = + separated_pair(take_until("="), tag("="), not_line_ending)(line); + + // Silently ignore lines that don't parse correctly. + parse_result.map(|(_, (key, value))| (key, value)).ok() + }) + .collect::>(); + + let id = parsed_map + .get(OS_RELEASE_ID) + .map(|s| s.trim_matches('\"').to_string()); + let version_id = parsed_map + .get(OS_RELEASE_VERSION_ID) + .map(|s| s.trim_matches('\"').to_string()); + + Ok(Self { id, version_id }) + } +} + +#[cfg(test)] +mod test { + use super::*; + + use rstest::rstest; + + #[rstest] + #[case::version_with_quotes("VERSION_ID=\"12.1\"\nNAME=\"Ubuntu\"", Some("12.1".to_string()))] + #[case::version_without_quotes("VERSION_ID=12.1\nNAME=\"Ubuntu\"", Some("12.1".to_string()))] + #[case::no_version("BAD_ID=\"12.1\"\nNAME=\"Ubuntu\"", None)] + fn test_version_id_parse(#[case] etc_os_release: &str, #[case] expected: Option) { + let os_release = EtcOsRelease::parse(etc_os_release).unwrap(); + + assert_eq!(os_release.version_id(), expected); + } + + #[rstest] + #[case::id_with_quotes("ID=\"ubuntu\"\nVERSION_ID=\"12.1\"", Some("ubuntu".to_string()))] + #[case::id_without_quotes("ID=ubuntu\nVERSION_ID=\"12.1\"", Some("ubuntu".to_string()))] + #[case::no_id("BAD_ID=ubuntu\nVERSION_ID=\"12.1\"", None)] + fn test_id_parse(#[case] etc_os_release: &str, #[case] expected: Option) { + let os_release = EtcOsRelease::parse(etc_os_release).unwrap(); + + assert_eq!(os_release.id(), expected); + } +} diff --git a/memfaultd/src/util/fs.rs b/memfaultd/src/util/fs.rs new file mode 100644 index 0000000..ff24a77 --- /dev/null +++ b/memfaultd/src/util/fs.rs @@ -0,0 +1,52 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + fs::{self}, + path::{Path, PathBuf}, +}; + +use eyre::{Context, Result}; +use log::warn; + +/// Takes a directory and returns a vector of all files in that directory, sorted +/// by creation date: +#[allow(dead_code)] // Required to build without warnings and --no-default-features +pub fn get_files_sorted_by_mtime(dir: &Path) -> Result> { + let read_dir = std::fs::read_dir(dir)?; + let mut entries = read_dir + .filter_map(|e| match e { + Ok(e) => Some(e), + Err(e) => { + warn!("Error reading directory entry: {:#}", e); + None + } + }) + .filter(|entry| entry.path().is_file()) + .collect::>(); + // Order by oldest first: + entries.sort_by_key(|entry| { + entry + .metadata() + .map(|m| m.modified().unwrap_or(std::time::SystemTime::UNIX_EPOCH)) + .unwrap_or(std::time::SystemTime::UNIX_EPOCH) + }); + Ok(entries.into_iter().map(|m| m.path()).collect()) +} + +/// Move a file. Try fs::rename first which is most efficient but only works if source +/// and destination are on the same filesystem. +/// Use Copy/Delete strategy if rename failed. +pub fn move_file(source: &PathBuf, target: &PathBuf) -> Result<()> { + if fs::rename(source, target).is_err() { + fs::copy(source, target).wrap_err_with(|| { + format!( + "Error moving file {} to {}", + source.display(), + target.display() + ) + })?; + fs::remove_file(source)?; + } + Ok(()) +} diff --git a/memfaultd/src/util/io.rs b/memfaultd/src/util/io.rs new file mode 100644 index 0000000..bec7c51 --- /dev/null +++ b/memfaultd/src/util/io.rs @@ -0,0 +1,194 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{ + self, copy, sink, BufReader, Chain, Cursor, ErrorKind, IoSlice, Read, Seek, SeekFrom, Write, +}; + +/// A trait for getting the length of a stream. +/// Note std::io::Seek also has a stream_len() method, but that method is fallible. +pub trait StreamLen { + /// Gets the length of the stream in bytes. + fn stream_len(&self) -> u64; +} + +impl StreamLen for BufReader { + fn stream_len(&self) -> u64 { + self.get_ref().stream_len() + } +} + +impl, B: StreamLen> StreamLen for Chain, B> { + fn stream_len(&self) -> u64 { + let (a, b) = self.get_ref(); + + a.get_ref().as_ref().len() as u64 + b.stream_len() + } +} + +pub trait StreamPosition { + /// Gets the current position in the stream. + fn stream_position(&self) -> usize; +} + +/// A passthrough `Write` implementation that keeps track of position. +/// +/// This is used to keep track of the current position in the output stream, since we can't use +/// `Seek` on all output streams. Additionally this allows us to keep track of the position +/// when using functions like `copy` that may call write several times and potentially fail. +pub struct StreamPositionTracker { + writer: T, + pos: usize, +} + +impl StreamPositionTracker { + pub fn new(writer: T) -> Self { + Self { writer, pos: 0 } + } +} + +impl Write for StreamPositionTracker { + /// Passthrough to the underlying writer, but also updates the position. + fn write(&mut self, buf: &[u8]) -> io::Result { + let written = self.writer.write(buf)?; + self.pos += written; + Ok(written) + } + + // Passthrough to the underlying writer. + fn flush(&mut self) -> io::Result<()> { + self.writer.flush() + } + + fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> std::io::Result { + let written = self.writer.write_vectored(bufs)?; + self.pos += written; + Ok(written) + } +} + +impl StreamPosition for StreamPositionTracker { + fn stream_position(&self) -> usize { + self.pos + } +} + +/// Stream wrapper that implements a subset of `Seek`. +/// +/// This is needed because we read from streams that do not implement `Seek`, +/// such as `Stdin`. Instead of implementing ad-hoc skipping we'll implement +/// `Seek` such that it only allows seeking forward. Any seek from the end of +/// the stream, or that would go backwards, will result in an error. +pub struct ForwardOnlySeeker { + reader: T, + pos: u64, +} + +impl ForwardOnlySeeker { + pub fn new(reader: T) -> Self { + Self { reader, pos: 0 } + } +} + +impl Read for ForwardOnlySeeker { + /// Passthrough to the underlying reader, but also updates the position. + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let bytes_read = self.reader.read(buf)?; + self.pos += bytes_read as u64; + Ok(bytes_read) + } +} + +impl Seek for ForwardOnlySeeker { + /// Seeks forward in the stream, returns an error if seeking backwards. + fn seek(&mut self, pos: SeekFrom) -> io::Result { + let seek_size = match pos { + SeekFrom::Current(offset) => u64::try_from(offset).ok(), + SeekFrom::Start(offset) => offset.checked_sub(self.pos), + SeekFrom::End(_) => { + return Err(io::Error::new( + ErrorKind::InvalidInput, + "Cannot seek from end", + )) + } + }; + + match seek_size { + Some(seek_size) => { + copy(&mut self.by_ref().take(seek_size), &mut sink())?; + Ok(self.pos) + } + None => Err(io::Error::new( + ErrorKind::InvalidInput, + "Only seeking forward allowed", + )), + } + } + + fn stream_position(&mut self) -> io::Result { + Ok(self.pos) + } +} + +#[cfg(test)] +mod test { + use rstest::rstest; + + use super::*; + + #[test] + fn test_write_cursor() { + let mut buf = Vec::new(); + let mut writer = StreamPositionTracker::new(&mut buf); + + writer.write_all(b"hello").unwrap(); + assert_eq!(writer.stream_position(), 5); + writer.write_all(b" world").unwrap(); + assert_eq!(writer.stream_position(), 11); + writer.write_all(b"!").unwrap(); + assert_eq!(writer.stream_position(), 12); + + assert_eq!(buf, b"hello world!"); + } + + #[test] + fn test_write_vectored_cursor() { + let mut buf = Vec::new(); + let mut writer = StreamPositionTracker::new(&mut buf); + + let write_vector = [ + IoSlice::new(b"hello"), + IoSlice::new(b" world"), + IoSlice::new(b"!"), + ]; + let bytes_written = writer.write_vectored(&write_vector).unwrap(); + + assert_eq!(bytes_written, 12); + assert_eq!(writer.stream_position(), 12); + assert_eq!(buf, b"hello world!"); + } + + #[test] + fn test_forward_seeker_stream() { + let mut input_stream = b"Hello world".as_ref(); + + let mut reader = ForwardOnlySeeker::new(&mut input_stream); + let mut out_buf = [0u8; 5]; + reader.read_exact(&mut out_buf).unwrap(); + assert_eq!(&out_buf, b"Hello"); + + reader.seek(SeekFrom::Current(1)).unwrap(); + reader.read_exact(&mut out_buf).unwrap(); + assert_eq!(&out_buf, b"world"); + } + + #[rstest] + #[case(SeekFrom::End(0))] + #[case(SeekFrom::Current(-1))] + #[case(SeekFrom::Start(0))] + fn test_forward_seeker_seek_fail(#[case] seek: SeekFrom) { + let mut reader = ForwardOnlySeeker::new(b"Hello world".as_ref()); + reader.seek(SeekFrom::Start(1)).unwrap(); + assert!(reader.seek(seek).is_err()); + } +} diff --git a/memfaultd/src/util/ipc.rs b/memfaultd/src/util/ipc.rs new file mode 100644 index 0000000..5ac6e2f --- /dev/null +++ b/memfaultd/src/util/ipc.rs @@ -0,0 +1,11 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::Result; +use nix::sys::signal::Signal::SIGUSR1; + +use super::pid_file::send_signal_to_pid; + +pub fn send_flush_signal() -> Result<()> { + send_signal_to_pid(SIGUSR1) +} diff --git a/memfaultd/src/util/math.rs b/memfaultd/src/util/math.rs new file mode 100644 index 0000000..58fd58c --- /dev/null +++ b/memfaultd/src/util/math.rs @@ -0,0 +1,12 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +/// Rounds the given value up to the nearest multiple of the given alignment. +/// +/// For values <= 1, the value is returned unchanged. +pub fn align_up(value: usize, alignment: usize) -> usize { + if alignment <= 1 { + return value; + } + ((value) + (alignment - 1)) & !(alignment - 1) +} diff --git a/memfaultd/src/util/mem.rs b/memfaultd/src/util/mem.rs new file mode 100644 index 0000000..4767097 --- /dev/null +++ b/memfaultd/src/util/mem.rs @@ -0,0 +1,28 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::mem::size_of; +use std::slice::{from_raw_parts, from_raw_parts_mut}; + +pub trait AsBytes { + /// Returns a slice of bytes representing the raw memory of the object. + /// # Safety + /// It is on the caller to ensure the interpretation of the bytes is correct. + unsafe fn as_bytes(&self) -> &[u8]; + + /// Returns a mutable slice of bytes representing the raw memory of the object. + /// # Safety + /// The type must not contain any references, pointers or types that require + /// validating invariants. + unsafe fn as_mut_bytes(&mut self) -> &mut [u8]; +} + +impl AsBytes for T { + unsafe fn as_bytes(&self) -> &[u8] { + from_raw_parts((self as *const T) as *const u8, size_of::()) + } + + unsafe fn as_mut_bytes(&mut self) -> &mut [u8] { + from_raw_parts_mut((self as *mut T) as *mut u8, size_of::()) + } +} diff --git a/memfaultd/src/util/mod.rs b/memfaultd/src/util/mod.rs new file mode 100644 index 0000000..906428b --- /dev/null +++ b/memfaultd/src/util/mod.rs @@ -0,0 +1,31 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +mod disk_backed; +pub use disk_backed::*; +mod die; +pub mod disk_size; +pub mod patterns; +pub use die::*; +pub mod can_connect; +pub mod circular_queue; +pub mod etc_os_release; +pub mod fs; +pub mod io; +pub mod ipc; +pub mod math; +pub mod mem; +pub mod output_arg; +pub mod path; +pub mod persistent_rate_limiter; +pub mod pid_file; +#[cfg(feature = "logging")] +pub mod rate_limiter; +pub mod serialization; +pub mod string; +pub mod system; +pub mod task; +#[cfg(feature = "logging")] +pub mod tcp_server; +pub mod time_measure; +pub mod zip; diff --git a/memfaultd/src/util/output_arg.rs b/memfaultd/src/util/output_arg.rs new file mode 100644 index 0000000..6221fe2 --- /dev/null +++ b/memfaultd/src/util/output_arg.rs @@ -0,0 +1,42 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + fs::File, + io::{stdout, Write}, + path::PathBuf, +}; + +use argh::FromArgValue; +use eyre::{Context, Result}; + +/// An Argh argument that can be either a `PathBuf` or a reference to `stdout` (`-`). +pub enum OutputArg { + Stdout, + File(PathBuf), +} + +impl FromArgValue for OutputArg { + fn from_arg_value(value: &str) -> Result { + if value == "-" { + Ok(OutputArg::Stdout) + } else { + Ok(OutputArg::File(value.into())) + } + } +} + +impl OutputArg { + /// Open the output stream designated by the user. + pub fn get_output_stream(&self) -> Result> { + let stream: Box = + match self { + OutputArg::Stdout => Box::new(stdout()), + OutputArg::File(path) => Box::new(File::create(path).wrap_err_with(|| { + format!("Error opening destination file {}", path.display()) + })?), + }; + + Ok(stream) + } +} diff --git a/memfaultd/src/util/path.rs b/memfaultd/src/util/path.rs new file mode 100644 index 0000000..5444429 --- /dev/null +++ b/memfaultd/src/util/path.rs @@ -0,0 +1,66 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::{eyre, Error, Result}; +use std::path::Path; +use std::{ffi::OsStr, path::PathBuf}; + +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(try_from = "PathBuf")] +/// A path that must be absolute. Use `AbsolutePath::try_from` to construct. +pub struct AbsolutePath(PathBuf); + +impl TryFrom for AbsolutePath { + type Error = Error; + + fn try_from(path: PathBuf) -> Result { + if path.is_absolute() { + Ok(Self(path)) + } else { + Err(eyre!("Path must be absolute: {:?}", path)) + } + } +} +impl From for PathBuf { + fn from(p: AbsolutePath) -> PathBuf { + p.0 + } +} +impl PartialEq for PathBuf { + fn eq(&self, other: &AbsolutePath) -> bool { + *self == *other.0 + } +} + +impl std::ops::Deref for AbsolutePath { + type Target = Path; + #[inline] + fn deref(&self) -> &Path { + self.0.as_path() + } +} + +/// Splits the filename at the first dot. +/// This is similar to the nighly-only std::path::Path::file_prefix. +#[allow(dead_code)] +pub fn file_prefix(path: &Path) -> Option<&OsStr> { + let file_name = path.file_name()?; + file_name.to_str()?.split('.').next().map(OsStr::new) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case("/test", "test")] + #[case("/test.log", "test")] + #[case("/test.log.zlib", "test")] + fn test_file_prefix(#[case] path: &str, #[case] expected: &str) { + assert_eq!(file_prefix(Path::new(path)), Some(OsStr::new(expected))); + } +} diff --git a/memfaultd/src/util/patterns.rs b/memfaultd/src/util/patterns.rs new file mode 100644 index 0000000..82c6c63 --- /dev/null +++ b/memfaultd/src/util/patterns.rs @@ -0,0 +1,84 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +pub fn alphanum_slug_is_valid(s: &str, max_len: usize) -> eyre::Result<()> { + match ( + (1..max_len).contains(&s.len()), + s.chars() + .all(|c| matches!(c, 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' )), + ) { + (true, true) => Ok(()), + (false, _) => Err(eyre::eyre!("Must be with 1 and 128 characters long")), + (_, false) => Err(eyre::eyre!( + "Must only contain alphanumeric characters and - or _" + )), + } +} + +pub fn alphanum_slug_is_valid_and_starts_alpha(s: &str, max_len: usize) -> eyre::Result<()> { + alphanum_slug_is_valid(s, max_len)?; + if s.starts_with(char::is_alphabetic) { + Ok(()) + } else { + Err(eyre::eyre!("Must start with an alphabetic character")) + } +} + +pub fn alphanum_slug_dots_colon_is_valid(s: &str, max_len: usize) -> eyre::Result<()> { + match ( + (1..max_len).contains(&s.len()), + s.chars() + .all(|c| matches!(c, 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' | '+' | '.' | ':')), + ) { + (true, true) => Ok(()), + (false, _) => Err(eyre::eyre!("Must be with 1 and 128 characters long")), + (_, false) => Err(eyre::eyre!( + "Must only contain alphanumeric characters, -,_,+,., and :" + )), + } +} + +pub fn alphanum_slug_dots_colon_spaces_parens_slash_is_valid( + s: &str, + max_len: usize, +) -> eyre::Result<()> { + match ((1..max_len).contains(&s.len()), s.chars().all(|c| matches!(c, 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' | '+' | '.' | ':' | ' ' | '[' | ']' | '(' | ')' | '\\'))) { + (true, true) => Ok(()), + (false, _) => Err(eyre::eyre!("Must be with 1 and 128 characters long")), + (_, false) => Err(eyre::eyre!("Must only contain alphanumeric characters, spaces, -,_,+,.,:,[,],(,), and \\")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + #[rstest] + #[case("1.0.0-rc2", true)] + #[case("qemuarm64", true)] + #[case("1.0.0-$", false)] + #[case("2.30^.1-rc4", false)] + #[case("2.30\\.1-rc4", false)] + #[case("spaces are invalid", false)] + fn test_alphanum_slug_dots_colon_is_valid(#[case] input: &str, #[case] result: bool) { + assert_eq!(alphanum_slug_dots_colon_is_valid(input, 64).is_ok(), result); + } + + #[rstest] + #[case("1.0.0-rc2", true)] + #[case("qemuarm64", true)] + #[case("1.0.0-$", false)] + #[case("2.30^.1-rc4", false)] + #[case("2.30\\.1-rc4", true)] + #[case("spaces are valid", true)] + fn test_alphanum_slug_dots_colon_spaces_parens_slash_is_valid( + #[case] input: &str, + #[case] result: bool, + ) { + assert_eq!( + alphanum_slug_dots_colon_spaces_parens_slash_is_valid(input, 64).is_ok(), + result + ); + } +} diff --git a/memfaultd/src/util/persistent_rate_limiter.rs b/memfaultd/src/util/persistent_rate_limiter.rs new file mode 100644 index 0000000..e007ae0 --- /dev/null +++ b/memfaultd/src/util/persistent_rate_limiter.rs @@ -0,0 +1,146 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use chrono::{DateTime, Duration, TimeZone, Utc}; +use eyre::Result; +use itertools::Itertools; +use std::{ + collections::VecDeque, + fs::File, + io::{BufRead, BufReader, Write}, + path::{Path, PathBuf}, +}; + +/// A `PersistentRateLimiter` that enforces action limits based on time +/// duration. +/// +/// - It allows actions up to the specified `count` within the `duration`. +/// - It stores a history of attempts in a file `path`. +pub struct PersistentRateLimiter { + path: PathBuf, + count: u32, + duration: Duration, + /// A list of all the times the rate limiter has been hit. We keep them + /// sorted from most recent to oldest and we cap this list to count. + history: VecDeque>, +} + +impl PersistentRateLimiter { + /// Load the rate limiter state from disk. + /// + /// Non-existent file is not considered an error. Garbage in the file will be skipped over. + pub fn load>(path: P, count: u32, duration: Duration) -> Result { + if count == 0 { + return Err(eyre::eyre!("count must be greater than 0")); + } + if duration.num_milliseconds() == 0 { + return Err(eyre::eyre!("duration must be greater than 0")); + } + + // Load previous invocations of the rate limiter, discarding anything that is not parseable. + let history = match File::open(&path) { + Ok(file) => BufReader::new(file) + .split(b' ') + .filter_map::, _>(|t| { + let ts: i64 = std::str::from_utf8(&t.ok()?).ok()?.parse().ok()?; + Utc.timestamp_opt(ts, 0).single() + }) + .sorted_by(|a, b| b.cmp(a)) + .collect::>(), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => VecDeque::new(), + Err(e) => return Err(e.into()), + }; + Ok(Self { + path: path.as_ref().to_owned(), + count, + duration, + history, + }) + } + + fn check_with_time(&mut self, now: DateTime) -> bool { + if self.history.len() >= self.count as usize { + if let Some(oldest) = self.history.back() { + if now.signed_duration_since(*oldest) < self.duration { + return false; + } + } + } + + self.history.push_front(now); + self.history.truncate(self.count as usize); + + true + } + + /// Check if the rate limiter will allow one call now. The state is updated + /// but not written to disk. Call `save()` to persist the rate limiter. + pub fn check(&mut self) -> bool { + self.check_with_time(Utc::now()) + } + + /// Writes the rate limiter state to disk. + pub fn save(self) -> Result<()> { + let mut file = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(self.path)?; + for time in self.history.iter() { + write!(file, "{} ", time.timestamp())?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use tempfile::tempdir; + + use super::*; + + #[rstest] + #[case::invalid_count(0, Duration::seconds(1))] + #[case::invalid_duration(1, Duration::seconds(0))] + fn invalid_init(#[case] count: u32, #[case] duration: Duration) { + let tmpdir = tempdir().unwrap(); + let path = tmpdir.path().join("test"); + + assert!(PersistentRateLimiter::load(path, count, duration).is_err()); + } + + #[rstest] + #[case(1, Duration::seconds(10), vec![0, 10, 20, 30, 35, 40], vec![true, true, true, true, false, true])] + #[case(3, Duration::seconds(10), vec![0, 0, 9, 10, 11, 12], vec![true, true, true, true, true, false ])] + #[case(3, Duration::seconds(10), vec![0, 0, 9, 9, 9, 9, 18, 19], vec![true, true, true, false, false, false, true, true ])] + #[case(3, Duration::seconds(10), vec![0, 100, 200, 300, 400, 500, 600], vec![true, true, true, true, true, true, true])] + fn test_rate_limiter( + #[case] count: u32, + #[case] duration: Duration, + #[case] timestamps: Vec, + #[case] expected: Vec, + ) { + assert_eq!( + timestamps.len(), + expected.len(), + "timestamps and expected results should be the same length" + ); + + let tmpdir = tempdir().unwrap(); + let path = tmpdir.path().join("test"); + + for (time, expected) in timestamps.into_iter().zip(expected.into_iter()) { + let mut limiter = + PersistentRateLimiter::load(&path, count, duration).expect("load error"); + assert_eq!( + limiter.check_with_time(Utc.timestamp_opt(time, 0).single().unwrap()), + expected, + "time: {} - history: {:?}", + time, + limiter.history + ); + limiter.save().expect("save error"); + } + } +} diff --git a/memfaultd/src/util/pid_file.rs b/memfaultd/src/util/pid_file.rs new file mode 100644 index 0000000..bb47a14 --- /dev/null +++ b/memfaultd/src/util/pid_file.rs @@ -0,0 +1,59 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::fs::{remove_file, OpenOptions}; +use std::io::{ErrorKind, Write}; +use std::path::Path; + +use eyre::{eyre, Report, Result, WrapErr}; +use nix::unistd::Pid; + +const PID_FILE: &str = "/var/run/memfaultd.pid"; + +pub fn write_pid_file() -> Result<()> { + let file = OpenOptions::new() + .write(true) + .create_new(true) + .open(Path::new(PID_FILE)); + + match file { + Ok(mut file) => writeln!(file, "{}", Pid::this()).wrap_err("Failed to write PID file"), + Err(e) => { + let msg = match e.kind() { + ErrorKind::AlreadyExists => "Daemon already running, aborting.", + _ => "Failed to open PID file, aborting.", + }; + Err(Report::new(e).wrap_err(msg)) + } + } +} + +/// Returns true if (and only if) our PID file exists, is readable and contains our current PID. +pub fn is_pid_file_about_me() -> bool { + match get_pid_from_file() { + Ok(pid) => pid == Pid::this(), + _ => false, + } +} + +pub fn get_pid_from_file() -> Result { + match std::fs::read_to_string(PID_FILE) { + Ok(pid_string) => { + let pid = pid_string + .trim() + .parse() + .wrap_err("Failed to parse PID file contents")?; + Ok(Pid::from_raw(pid)) + } + Err(_) => Err(eyre!("Couldn't read memfaultd PID file at {}.", PID_FILE)), + } +} + +pub fn remove_pid_file() -> Result<()> { + remove_file(PID_FILE).wrap_err("Failed to remove PID file") +} + +pub fn send_signal_to_pid(signal: nix::sys::signal::Signal) -> Result<()> { + let pid = get_pid_from_file()?; + nix::sys::signal::kill(pid, signal).wrap_err("Failed to send signal to memfaultd") +} diff --git a/memfaultd/src/util/rate_limiter.rs b/memfaultd/src/util/rate_limiter.rs new file mode 100644 index 0000000..16502a2 --- /dev/null +++ b/memfaultd/src/util/rate_limiter.rs @@ -0,0 +1,154 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Support for rate limiting some execution paths. +use std::num::NonZeroU32; + +use eyre::Result; +use governor::{ + clock, + middleware::NoOpMiddleware, + state::{InMemoryState, NotKeyed}, + Quota, RateLimiter as GovRateLimiter, +}; + +/// A rate limiter which keeps track of how many calls were rate limited. +/// You can also provide a `info: I` parameter with each call. The latest one +/// will be passed to the runner when not rate limited anymore. +pub struct RateLimiter { + rate_limiter: GovRateLimiter>, + limited_calls: Option>, +} + +pub struct RateLimitedCalls { + pub count: usize, + pub latest_call: I, +} + +impl RateLimiter { + /// Create a new rate limiter with given capacity, quantum and rate (see ratelimit::Ratelimiter). + pub fn new(capacity_per_minute: NonZeroU32) -> Self { + Self { + rate_limiter: GovRateLimiter::direct(Quota::per_minute(capacity_per_minute)), + limited_calls: None, + } + } +} + +impl RateLimiter { + /// Run the provided work function if the rate limitings limits have not been reached. + pub fn run_within_limits(&mut self, info: I, work: W) -> Result<()> + where + W: FnOnce(Option>) -> Result<()>, + { + if self.rate_limiter.check().is_ok() { + work(self.limited_calls.take()) + } else { + self.limited_calls = Some(match self.limited_calls.take() { + None => RateLimitedCalls { + count: 1, + latest_call: info, + }, + Some(l) => RateLimitedCalls { + count: l.count + 1, + latest_call: info, + }, + }); + Ok(()) + } + } +} + +#[cfg(test)] +mod tests { + use std::{num::NonZeroU32, time::Duration}; + + use governor::clock; + use governor::clock::FakeRelativeClock; + use rstest::fixture; + use rstest::rstest; + + use super::RateLimiter; + + #[rstest] + fn test_sustained_100_per_minute(mut rl: RLFixture) { + for _ in 0..20 { + rl.assert_wait_and_grab_tokens(15_000, 25); + } + } + + #[rstest] + fn test_bursty_start(mut rl: RLFixture) { + rl.assert_wait_and_grab_tokens(0, 100); + rl.assert_empty(); + rl.assert_wait_and_grab_tokens(15_000, 25); + rl.assert_empty(); + } + + #[rstest] + fn test_reject_burst(mut rl: RLFixture) { + rl.assert_wait_and_grab_tokens(200_000, 100); + rl.assert_empty(); + rl.assert_wait_and_grab_tokens(1000, 1); + } + + #[fixture(limit = 100)] + fn rl(limit: u32) -> RLFixture { + let clock = FakeRelativeClock::default(); + RLFixture { + rl: RateLimiter::new_with_clock(NonZeroU32::new(limit).unwrap(), &clock), + clock, + } + } + + struct RLFixture { + rl: RateLimiter<(), FakeRelativeClock>, + clock: FakeRelativeClock, + } + + impl RLFixture { + pub fn assert_wait_and_grab_tokens(&mut self, sleep_ms: u64, count_tokens: u64) { + let grabbed = self.wait_and_grab_tokens(sleep_ms, count_tokens); + assert!( + count_tokens == grabbed, + "Expected to grab {count_tokens} but only {grabbed} available." + ); + } + + pub fn assert_empty(&mut self) { + assert!(self.grab_tokens(1) == 0, "Rate limiter is not empty"); + } + + pub fn wait_and_grab_tokens(&mut self, sleep_ms: u64, count_tokens: u64) -> u64 { + self.clock.advance(Duration::from_millis(sleep_ms)); + self.grab_tokens(count_tokens) + } + + pub fn grab_tokens(&mut self, c: u64) -> u64 { + for i in 0..c { + let work_done = &mut false; + let _result = self.rl.run_within_limits((), |_info| { + *work_done = true; + Ok(()) + }); + if !*work_done { + return i; + } + } + c + } + } + + impl RateLimiter { + #[cfg(test)] + pub fn new_with_clock(capacity: NonZeroU32, clock: &C) -> Self { + use governor::Quota; + use governor::RateLimiter as GovRateLimiter; + + Self { + rate_limiter: GovRateLimiter::direct_with_clock(Quota::per_minute(capacity), clock), + limited_calls: None, + } + } + } +} diff --git a/memfaultd/src/util/serialization/datetime_to_rfc3339.rs b/memfaultd/src/util/serialization/datetime_to_rfc3339.rs new file mode 100644 index 0000000..1502e1c --- /dev/null +++ b/memfaultd/src/util/serialization/datetime_to_rfc3339.rs @@ -0,0 +1,24 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Deserializer, Serializer}; + +pub fn serialize(time: &DateTime, serializer: S) -> Result +where + S: Serializer, +{ + let datetime_str = time.to_rfc3339(); + serializer.serialize_str(&datetime_str) +} + +pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let datetime_str = String::deserialize(deserializer)?; + let datetime = DateTime::parse_from_rfc3339(&datetime_str) + .map_err(|e| serde::de::Error::custom(format!("invalid timestamp: {}", e)))?; + + Ok(datetime.with_timezone(&Utc)) +} diff --git a/memfaultd/src/util/serialization/float_to_datetime.rs b/memfaultd/src/util/serialization/float_to_datetime.rs new file mode 100644 index 0000000..b70e4ac --- /dev/null +++ b/memfaultd/src/util/serialization/float_to_datetime.rs @@ -0,0 +1,30 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use chrono::{DateTime, NaiveDateTime, Utc}; +use serde::{Deserialize, Deserializer, Serializer}; + +pub fn serialize(time: &DateTime, serializer: S) -> Result +where + S: Serializer, +{ + serializer.serialize_f64( + time.timestamp() as f64 + (time.timestamp_subsec_micros() as f64 / 1_000_000.0), + ) +} + +pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let secs = f64::deserialize(deserializer)?; + + // Collectd only sends milli-seconds. We round the float to the nearest ms + // to avoid precision error. + let ms = ((secs.rem_euclid(1.0)) * 1e3).round() as u32; + + match NaiveDateTime::from_timestamp_opt(secs.floor() as i64, ms * 1_000_000) { + Some(naive) => Ok(DateTime::::from_utc(naive, Utc)), + None => Err(serde::de::Error::custom("invalid timestamp")), + } +} diff --git a/memfaultd/src/util/serialization/float_to_duration.rs b/memfaultd/src/util/serialization/float_to_duration.rs new file mode 100644 index 0000000..53f3f22 --- /dev/null +++ b/memfaultd/src/util/serialization/float_to_duration.rs @@ -0,0 +1,23 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use chrono::Duration; +use serde::{Deserialize, Deserializer, Serializer}; + +pub fn serialize(duration: &Duration, serializer: S) -> Result +where + S: Serializer, +{ + serializer + .serialize_f64(duration.num_seconds() as f64 + duration.num_milliseconds() as f64 / 1000.0) +} + +pub fn deserialize<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let d = f64::deserialize(deserializer)?; + let seconds = d.trunc() as i64; + let ms = (d.rem_euclid(1.0) * 1000.0) as i64; + Ok(Duration::seconds(seconds) + Duration::milliseconds(ms)) +} diff --git a/memfaultd/src/util/serialization/kib_to_usize.rs b/memfaultd/src/util/serialization/kib_to_usize.rs new file mode 100644 index 0000000..438e2b3 --- /dev/null +++ b/memfaultd/src/util/serialization/kib_to_usize.rs @@ -0,0 +1,45 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +pub fn serialize(size: &usize, serializer: S) -> Result +where + S: Serializer, +{ + if size % 1024 != 0 { + return Err(serde::ser::Error::custom( + "Cannot serialize non-multiple of 1024 to kib.", + )); + } + (size / 1024).serialize(serializer) +} + +pub fn deserialize<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let size = usize::deserialize(deserializer)?; + Ok(size * 1024) +} + +#[cfg(test)] +mod tests { + + #[test] + fn serialize_error() { + let mut serializer = serde_json::Serializer::new(std::io::stdout()); + let r = super::serialize(&1025, &mut serializer); + assert!(r.is_err()); + } + + #[test] + fn serialize_multiple_of_1024() { + let mut buf = Vec::new(); + let mut serializer = serde_json::Serializer::new(&mut buf); + let r = super::serialize(&43008, &mut serializer); + assert!(r.is_ok()); + + assert_eq!(&buf, b"42"); + } +} diff --git a/memfaultd/src/util/serialization/milliseconds_to_duration.rs b/memfaultd/src/util/serialization/milliseconds_to_duration.rs new file mode 100644 index 0000000..210ce52 --- /dev/null +++ b/memfaultd/src/util/serialization/milliseconds_to_duration.rs @@ -0,0 +1,21 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use serde::{Deserialize, Deserializer, Serializer}; + +use std::time::Duration; + +pub fn serialize(duration: &Duration, serializer: S) -> Result +where + S: Serializer, +{ + serializer.serialize_u128(duration.as_millis()) +} + +pub fn deserialize<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let secs = u64::deserialize(deserializer)?; + Ok(Duration::from_millis(secs)) +} diff --git a/memfaultd/src/util/serialization/mod.rs b/memfaultd/src/util/serialization/mod.rs new file mode 100644 index 0000000..568902f --- /dev/null +++ b/memfaultd/src/util/serialization/mod.rs @@ -0,0 +1,12 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +pub mod datetime_to_rfc3339; +pub mod float_to_datetime; +pub mod float_to_duration; +pub mod kib_to_usize; +pub mod milliseconds_to_duration; +pub mod number_to_compression; +pub mod optional_milliseconds_to_duration; +pub mod seconds_to_duration; +pub mod sorted_map; diff --git a/memfaultd/src/util/serialization/number_to_compression.rs b/memfaultd/src/util/serialization/number_to_compression.rs new file mode 100644 index 0000000..f11bfe1 --- /dev/null +++ b/memfaultd/src/util/serialization/number_to_compression.rs @@ -0,0 +1,25 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use flate2::Compression; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +pub fn serialize(compression: &Compression, serializer: S) -> Result +where + S: Serializer, +{ + (compression.level()).serialize(serializer) +} + +pub fn deserialize<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let level = u32::deserialize(deserializer)?; + match level { + 0..=9 => Ok(Compression::new(level)), + _ => Err(serde::de::Error::custom( + "Compression level must be between 0 and 9.", + )), + } +} diff --git a/memfaultd/src/util/serialization/optional_milliseconds_to_duration.rs b/memfaultd/src/util/serialization/optional_milliseconds_to_duration.rs new file mode 100644 index 0000000..cb79173 --- /dev/null +++ b/memfaultd/src/util/serialization/optional_milliseconds_to_duration.rs @@ -0,0 +1,28 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use serde::{Deserialize, Deserializer, Serializer}; + +use std::time::Duration; + +pub fn serialize(duration: &Option, serializer: S) -> Result +where + S: Serializer, +{ + if let Some(duration) = duration { + serializer.serialize_u128(duration.as_millis()) + } else { + serializer.serialize_none() + } +} + +pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + if let Ok(secs) = u64::deserialize(deserializer) { + Ok(Some(Duration::from_millis(secs))) + } else { + Ok(None) + } +} diff --git a/memfaultd/src/util/serialization/seconds_to_duration.rs b/memfaultd/src/util/serialization/seconds_to_duration.rs new file mode 100644 index 0000000..d62a5db --- /dev/null +++ b/memfaultd/src/util/serialization/seconds_to_duration.rs @@ -0,0 +1,21 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use serde::{Deserialize, Deserializer, Serializer}; + +use std::time::Duration; + +pub fn serialize(duration: &Duration, serializer: S) -> Result +where + S: Serializer, +{ + serializer.serialize_u64(duration.as_secs()) +} + +pub fn deserialize<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let secs = u64::deserialize(deserializer)?; + Ok(Duration::from_secs(secs)) +} diff --git a/memfaultd/src/util/serialization/sorted_map.rs b/memfaultd/src/util/serialization/sorted_map.rs new file mode 100644 index 0000000..0d30c4d --- /dev/null +++ b/memfaultd/src/util/serialization/sorted_map.rs @@ -0,0 +1,15 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::collections::{BTreeMap, HashMap}; + +use serde::{Serialize, Serializer}; + +pub fn sorted_map( + value: &HashMap, + serializer: S, +) -> Result { + let mut items: Vec<(_, _)> = value.iter().collect(); + items.sort_by(|a, b| a.0.cmp(b.0)); + BTreeMap::from_iter(items).serialize(serializer) +} diff --git a/memfaultd/src/util/string.rs b/memfaultd/src/util/string.rs new file mode 100644 index 0000000..e4ef612 --- /dev/null +++ b/memfaultd/src/util/string.rs @@ -0,0 +1,94 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +/// Simple implementation of remove-C-style-comments +/// Saves code space by not relying on an external library. +pub fn remove_comments(config_string: &str) -> String { + let mut data = String::from(config_string); + while let Some(index) = data.find("/*") { + if let Some(index_end) = data.find("*/") { + data = String::from(&data[..index]) + &data[index_end + 2..]; + } else { + // No matching close. Keep everything + break; + } + } + data +} + +pub trait Ellipsis { + fn truncate_with_ellipsis(&mut self, len_bytes: usize); +} + +impl Ellipsis for String { + fn truncate_with_ellipsis(&mut self, len_bytes: usize) { + const ELLIPSIS: &str = "…"; // Note: 3 bytes in UTF-8 + let max_len_bytes = len_bytes - ELLIPSIS.len(); + if self.len() <= max_len_bytes { + return; + } + let idx = (0..=max_len_bytes) + .rev() + .find(|idx| self.is_char_boundary(*idx)) + .unwrap_or(0); + self.truncate(idx); + self.push_str(ELLIPSIS); + } +} + +/// Capitalize the first letter of a string +pub fn capitalize(s: &str) -> String { + let mut c = s.chars(); + match c.next() { + None => String::new(), + Some(f) => f.to_uppercase().collect::() + c.as_str(), + } +} + +#[cfg(test)] +mod test { + use super::*; + use rstest::rstest; + + #[test] + fn test_remove_comments() { + assert_eq!(remove_comments(""), ""); + assert_eq!(remove_comments("hello world"), "hello world"); + assert_eq!(remove_comments("hello /* comment */ world"), "hello world"); + assert_eq!( + remove_comments("hello /* comment world"), + "hello /* comment world" + ); + assert_eq!( + remove_comments("hello /* comment */world/* comment */"), + "hello world" + ); + } + + #[rstest] + // No truncation: + #[case("foobar", 10, "foobar")] + // Truncation basic: + #[case("foobar", 6, "foo…")] + // Truncation inside a multi-byte character (smiling pile of poo is 4 bytes): + #[case("f💩bar", 6, "f…")] + // Panic: len_bytes too short to fit ellipsis: + #[should_panic] + #[case("foobar", 0, "…")] + fn truncate_with_ellipsis( + #[case] input: &str, + #[case] len_bytes: usize, + #[case] expected: &str, + ) { + let mut s = String::from(input); + s.truncate_with_ellipsis(len_bytes); + assert_eq!(&s, expected); + } + + #[rstest] + #[case("capital")] + #[case("Capital")] + fn test_capitalize(#[case] input: &str) { + assert_eq!(capitalize(input), "Capital"); + } +} diff --git a/memfaultd/src/util/system.rs b/memfaultd/src/util/system.rs new file mode 100644 index 0000000..b6b0553 --- /dev/null +++ b/memfaultd/src/util/system.rs @@ -0,0 +1,94 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::{eyre, Result}; +use libc::{clockid_t, timespec, CLOCK_MONOTONIC}; + +#[cfg(target_os = "linux")] +use libc::{sysconf, _SC_CLK_TCK, _SC_PAGE_SIZE}; + +use uuid::Uuid; + +#[cfg(target_os = "linux")] +pub fn read_system_boot_id() -> Result { + use eyre::Context; + use std::{fs::read_to_string, str::FromStr}; + + const BOOT_ID_PATH: &str = "/proc/sys/kernel/random/boot_id"; + let boot_id = read_to_string(BOOT_ID_PATH); + + match boot_id { + Ok(boot_id_str) => Uuid::from_str(boot_id_str.trim()).wrap_err("Invalid boot id"), + Err(_) => Err(eyre!("Unable to read boot id from system.")), + } +} + +#[cfg(target_os = "linux")] +pub fn clock_ticks_per_second() -> u64 { + unsafe { sysconf(_SC_CLK_TCK) as u64 } +} + +#[cfg(target_os = "linux")] +pub fn bytes_per_page() -> u64 { + unsafe { sysconf(_SC_PAGE_SIZE) as u64 } +} + +/// Calls clock_gettime +/// Most interesting to us are: +/// CLOCK_MONOTONIC: "clock that increments monotonically, tracking the time +/// since an arbitrary point, and will continue to increment while the system is +/// asleep." +/// CLOCK_BOOTTIME A nonsettable system-wide clock that is identical to +/// CLOCK_MONOTONIC, except that it also includes any time that the system is +/// suspended. This allows applications to get a suspend-aware monotonic clock +/// without having to deal with the complications of CLOCK_REALTIME, which may +/// have discontinuities if the time is changed using settimeofday(2) or +/// similar. +pub enum Clock { + Monotonic, + Boottime, +} +pub fn get_system_clock(clock: Clock) -> Result { + // Linux only so we define it here. + const CLOCK_BOOTTIME: clockid_t = 7; + + let mut t = timespec { + tv_sec: 0, + tv_nsec: 0, + }; + if unsafe { + libc::clock_gettime( + match clock { + Clock::Monotonic => CLOCK_MONOTONIC, + Clock::Boottime if cfg!(target_os = "linux") => CLOCK_BOOTTIME, + // Falls back to monotonic if not linux + Clock::Boottime => CLOCK_MONOTONIC, + }, + &mut t, + ) + } != 0 + { + Err(eyre!("Error getting system clock.")) + } else { + Ok(std::time::Duration::new(t.tv_sec as u64, t.tv_nsec as u32)) + } +} + +/// Provide some mock implementations for non-Linux systems. Designed for development. Not actual use. + +#[cfg(not(target_os = "linux"))] +pub fn read_system_boot_id() -> Result { + use once_cell::sync::Lazy; + static MOCK_BOOT_ID: Lazy = Lazy::new(Uuid::new_v4); + Ok(*MOCK_BOOT_ID) +} + +#[cfg(not(target_os = "linux"))] +pub fn clock_ticks_per_second() -> u64 { + 10_000 +} + +#[cfg(not(target_os = "linux"))] +pub fn bytes_per_page() -> u64 { + 4096 +} diff --git a/memfaultd/src/util/task.rs b/memfaultd/src/util/task.rs new file mode 100644 index 0000000..b3ce5af --- /dev/null +++ b/memfaultd/src/util/task.rs @@ -0,0 +1,263 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::Result; +use std::time::{Duration, Instant}; + +use log::{trace, warn}; + +use super::time_measure::TimeMeasure; + +/// Run `work` every `repeat_interval` while `condition` returns true. +/// +/// On error, wait `error_retry` and multiply `error_retry` by 2 every +/// time the error is repeated but never exceeding repeat_interval. +/// +/// This is useful if you need (for example) to make a network request at a +/// fixed interval (1 hour) and want to retry sooner (1 minute) if the connection fails. +/// If the connection keeps on failing, the retry time will be increased (2min, 4min, etc). +/// +/// When the process receives a signal we will immediately check the condition +/// and run the work if the condition is still true. +/// (You have to catch the signal somewhere - otherwise the process will be terminated.) +pub fn loop_with_exponential_error_backoff< + W: FnMut() -> Result<()>, + T: FnMut() -> LoopContinuation, +>( + work: W, + condition: T, + period: Duration, + error_retry: Duration, +) { + loop_with_exponential_error_backoff_internal::<_, _, Instant>( + work, + condition, + period, + error_retry, + interruptiple_sleep, + ) +} + +// std::thread::sleep automatically continues sleeping on SIGINT but we want to be interrupted so we use shuteye::sleep. +fn interruptiple_sleep(d: Duration) { + shuteye::sleep(d); +} + +/// Specify how to continue execution +#[derive(PartialEq, Eq)] +pub enum LoopContinuation { + /// Continue running the loop normally + KeepRunning, + /// Immediately re-process the loop + RerunImmediately, + /// Stop running the loop + Stop, +} + +fn loop_with_exponential_error_backoff_internal< + W: FnMut() -> Result<()>, + T: FnMut() -> LoopContinuation, + Time: TimeMeasure, +>( + mut work: W, + mut condition: T, + period: Duration, + error_retry: Duration, + sleep: fn(Duration), +) { + const BACKOFF_MULTIPLIER: u32 = 2; + let mut count_errors_since_success = 0; + while condition() != LoopContinuation::Stop { + let start_work = Time::now(); + let next_run_in = match work() { + Ok(_) => { + count_errors_since_success = 0; + period + } + Err(e) => { + let next_run = Duration::min( + error_retry.saturating_mul( + BACKOFF_MULTIPLIER.saturating_pow(count_errors_since_success), + ), + period, + ); + + count_errors_since_success += 1; + warn!("Error in Memfaultd main loop: {}", e); + next_run + } + }; + + if condition() == LoopContinuation::KeepRunning { + let sleep_maybe = next_run_in.checked_sub(start_work.elapsed()); + if let Some(howlong) = sleep_maybe { + trace!("Sleep for {:?}", howlong); + sleep(howlong); + } + } + } +} + +#[cfg(test)] +mod tests { + use eyre::eyre; + use std::cell::{Cell, RefCell}; + + use crate::test_utils::TestInstant; + + use super::*; + use rstest::rstest; + + #[rstest] + #[case::everything_ok(vec![ + TestInvocation { + ..Default::default() + }, + TestInvocation { + expect_called_at: TestInstant::from(TEST_PERIOD), + run_time: Duration::from_millis(150), + ..Default::default() + }, + TestInvocation { + expect_called_at: TestInstant::from(TEST_PERIOD * 2), + ..Default::default() + } + ])] + #[case::errors_are_retried_sooner(vec![ + TestInvocation { + run_time: Duration::from_millis(10), + is_error: true, + ..Default::default() + }, + TestInvocation { + expect_called_at: TestInstant::from(TEST_ERROR_RETRY), + ..Default::default() + }, + TestInvocation { + expect_called_at: TestInstant::from(TEST_ERROR_RETRY + TEST_PERIOD), + ..Default::default() + } + ])] + #[case::long_runs_will_rerun_immediately(vec![ + TestInvocation { + expect_called_at: TestInstant::from(Duration::from_secs(0)), + run_time: TEST_PERIOD * 10, + ..Default::default() + }, + TestInvocation { + expect_called_at: TestInstant::from( TEST_PERIOD * 10), + ..Default::default() + } + ])] + #[case::errors_retry_backoff(vec![ + TestInvocation { + run_time: Duration::from_millis(10), + is_error: true, + ..Default::default() + }, + TestInvocation { + expect_called_at: TestInstant::from(TEST_ERROR_RETRY), + is_error: true, + ..Default::default() + }, + TestInvocation { + expect_called_at: TestInstant::from(TEST_ERROR_RETRY + TEST_ERROR_RETRY * 2), + is_error: true, + ..Default::default() + }, + TestInvocation { + expect_called_at: TestInstant::from(TEST_ERROR_RETRY + TEST_ERROR_RETRY * 2 + TEST_ERROR_RETRY * 4), + ..Default::default() + }, + TestInvocation { + expect_called_at: TestInstant::from(TEST_ERROR_RETRY + TEST_ERROR_RETRY * 2 + TEST_ERROR_RETRY * 4 + TEST_PERIOD), + is_error: true, + ..Default::default() + }, + // This one should have reset to normal error retry + TestInvocation { + expect_called_at: TestInstant::from(TEST_ERROR_RETRY + TEST_ERROR_RETRY * 2 + TEST_ERROR_RETRY * 4 + TEST_PERIOD + TEST_ERROR_RETRY), + is_error: true, + ..Default::default() + }, + ])] + #[case::can_rerun_immediately(vec![ + TestInvocation { + run_time: Duration::from_millis(10), + is_error: false, + ..Default::default() + }, + TestInvocation { + expect_called_at: TestInstant::from(Duration::from_millis(10)), + run_immediately: true, + ..Default::default() + }, + ])] + fn test_loop_with_exponential_backoff(#[case] calls: Vec) { + let step = Cell::new(0); + let call_times = RefCell::new(vec![]); + + let work = || { + let invocation = &calls[step.get()]; + + call_times.borrow_mut().push(TestInstant::now()); + step.set(step.get() + 1); + + TestInstant::sleep(invocation.run_time); + + match invocation.is_error { + true => Err(eyre!("invocation failed")), + false => Ok(()), + } + }; + // Run until we have executed all the provided steps. + let condition = || { + if step.get() < calls.len() { + // We do not need a +1 here because the step has already been incremented + // when condition is called after doing the work. + if step.get() < calls.len() && calls[step.get()].run_immediately { + LoopContinuation::RerunImmediately + } else { + LoopContinuation::KeepRunning + } + } else { + LoopContinuation::Stop + } + }; + + loop_with_exponential_error_backoff_internal::<_, _, TestInstant>( + work, + condition, + TEST_PERIOD, + TEST_ERROR_RETRY, + TestInstant::sleep, + ); + + let expected_call_times = calls + .into_iter() + .map(|c| c.expect_called_at) + .collect::>(); + assert_eq!(expected_call_times, *call_times.borrow()); + } + + #[derive(Clone)] + struct TestInvocation { + run_time: Duration, + is_error: bool, + run_immediately: bool, + expect_called_at: TestInstant, + } + impl Default for TestInvocation { + fn default() -> Self { + Self { + run_time: Duration::from_millis(30), + is_error: false, + run_immediately: false, + expect_called_at: TestInstant::from(Duration::from_millis(0)), + } + } + } + + const TEST_PERIOD: Duration = Duration::from_secs(3600); + const TEST_ERROR_RETRY: Duration = Duration::from_secs(60); +} diff --git a/memfaultd/src/util/tcp_server.rs b/memfaultd/src/util/tcp_server.rs new file mode 100644 index 0000000..2edfe41 --- /dev/null +++ b/memfaultd/src/util/tcp_server.rs @@ -0,0 +1,86 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::Result; +use log::{trace, warn}; +use std::io::Read; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::thread; +use threadpool::ThreadPool; + +/// A TCP server that spawns threads to handle incoming connections. +/// Incoming connections will be delegated to a `TcpConnectionHandler`. +pub struct ThreadedTcpServer {} + +pub trait TcpConnectionHandler: Send + Sync + Clone + 'static { + fn handle_connection(&self, stream: TcpStream) -> Result<()>; +} + +impl ThreadedTcpServer { + pub fn start( + bind_address: SocketAddr, + max_connections: usize, + handler: impl TcpConnectionHandler, + ) -> Result { + let listener = TcpListener::bind(bind_address)?; + thread::spawn(move || Self::run(listener, max_connections, handler)); + Ok(ThreadedTcpServer {}) + } + + fn run( + listener: TcpListener, + max_connections: usize, + handler: impl TcpConnectionHandler, + ) -> Result<()> { + let pool = ThreadPool::new(max_connections); + + for stream in listener.incoming() { + match stream { + Ok(stream) => { + trace!( + "Connection from {:?} - Threads {}/{}", + stream.peer_addr(), + pool.active_count(), + pool.max_count() + ); + let handler = handler.clone(); + pool.execute(move || { + if let Err(e) = handler.handle_connection(stream) { + warn!("Error while handling connection: {}", e) + } + }) + } + Err(e) => { + warn!("TCP server listener error {}", e); + break; + } + } + } + trace!("Done listening - waiting for pool to terminate"); + pool.join(); + trace!("Pool joined."); + + Ok(()) + } +} + +/// Handler that reads and drops all data. +#[derive(Clone)] +pub struct TcpNullConnectionHandler {} + +impl TcpConnectionHandler for TcpNullConnectionHandler { + fn handle_connection(&self, mut stream: TcpStream) -> Result<()> { + loop { + let mut buf = [0; 8 * 1024]; + match stream.read(&mut buf) { + Ok(0) => break, // EOF + Ok(_) => {} // drop the data + Err(e) => { + warn!("TCP read error: {:?}", e); + break; + } + } + } + Ok(()) + } +} diff --git a/memfaultd/src/util/time_measure.rs b/memfaultd/src/util/time_measure.rs new file mode 100644 index 0000000..8b15ff7 --- /dev/null +++ b/memfaultd/src/util/time_measure.rs @@ -0,0 +1,27 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::time::{Duration, Instant}; + +/// A trait for measuring time. +/// +/// This is mostly a way to mock std::time::Instant for testing. +pub trait TimeMeasure { + fn now() -> Self; + fn elapsed(&self) -> Duration; + fn since(&self, other: &Self) -> Duration; +} + +impl TimeMeasure for Instant { + fn now() -> Self { + Instant::now() + } + + fn elapsed(&self) -> Duration { + Self::now().since(self) + } + + fn since(&self, other: &Self) -> Duration { + Instant::duration_since(self, *other) + } +} diff --git a/memfaultd/src/util/zip.rs b/memfaultd/src/util/zip.rs new file mode 100644 index 0000000..96de947 --- /dev/null +++ b/memfaultd/src/util/zip.rs @@ -0,0 +1,487 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::fs::File; +use std::io::Cursor; +use std::io::Read; +use std::os::unix::ffi::OsStrExt; +use std::path::{Path, PathBuf}; +#[cfg(test)] +use std::str::from_utf8; + +use crate::util::io::StreamLen; +use eyre::Result; +use flate2::CrcReader; +use take_mut::take; + +/// Minimalistic zip encoder which generates a compression-less zip stream on the fly from a list of +/// files. The benefit of this is that it requires no temporary file storage. It implements the +/// std::io::Read trait, so it can be used with any std::io::Read consumer. It can also tell the +/// length of the stream beforehand, only looking at the list of files and their sizes on disk. +/// This is useful for example when needing to specify a Content-Length header for a HTTP request. +/// Note it is very minimalistic in its implementation: it only supports "store" (no compression). +/// It only supports the 32-bit zip format, so it is limited to max. 4GB file sizes and does not +/// allow for more than 65,535 entries. File timestamps are not implemented and neither are UTF-8 +/// filenames. +/// Note that read() calls can copy less than the size of the caller's buffer, due to an +/// implementation detail. Therefore it's recommended to use std::io::BufReader to wrap this stream. + +// Some implementation notes: +// - The zip format is described here: https://en.wikipedia.org/wiki/ZIP_(file_format) +// - For each file in the zip file, the contents of the file is prepended with a "local file header" +// and suffixed with a "data descriptor". The local file header contains some metadata of the file +// that follows and the data descriptor contains additional metadata that is conveniently gathered +// while reading/writing the contents, like the CRC32 checksum of the file's contents. +// - After all files have been written, a "central directory" is written, which contains all the +// metadata of the files again, but in a slightly different, more elaborate format. This is used +// by the decoder/unarchiver to quickly access the list of files in the zip file. + +pub struct ZipEncoder { + files: Vec, + state: ZipEncoderState, + bytes_written: usize, +} + +enum ZipEncoderState { + Init, + LocalFiles { + index: usize, + reader: LocalFileReader, + }, + CentralDirectory { + index: usize, + start_offset: usize, + reader: Cursor>, + }, + EndOfCentralDirectory { + reader: Cursor>, + }, + Done, +} + +impl ZipEncoder { + /// Creates a new ZipEncoder from a list of source files that should be included in the zip. + pub fn new(files: Vec) -> Self { + Self { + files, + state: ZipEncoderState::Init, + bytes_written: 0, + } + } + + fn new_local_files_state(&self, index: usize) -> std::io::Result { + Ok(ZipEncoderState::LocalFiles { + index, + reader: LocalFileReader::new(&self.files[index])?, + }) + } + + fn new_central_directory_state(&self, index: usize, start_offset: usize) -> ZipEncoderState { + ZipEncoderState::CentralDirectory { + index, + start_offset, + reader: Cursor::new(make_file_header( + &self.files[index], + FileHeaderKind::CentralDirectory, + )), + } + } + + fn new_end_of_central_directory_state(&self, start_offset: usize) -> ZipEncoderState { + let num_files = self.files.len(); + ZipEncoderState::EndOfCentralDirectory { + reader: Cursor::new(make_end_of_central_directory( + num_files as u16, + (self.bytes_written - start_offset) as u32, + start_offset as u32, + )), + } + } + + #[cfg(test)] + pub fn file_names(&self) -> Vec<&str> { + self.files + .iter() + .map(|f| from_utf8(f.name.as_slice()).unwrap()) + .collect() + } +} + +impl StreamLen for ZipEncoder { + /// Length of the zip stream in bytes. + fn stream_len(&self) -> u64 { + zip_stream_len(&self.files) as u64 + } +} + +impl Read for ZipEncoder { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + loop { + let reader: &mut dyn Read = match &mut self.state { + ZipEncoderState::Init => { + if self.files.is_empty() { + self.state = self.new_end_of_central_directory_state(0); + } else { + self.state = self.new_local_files_state(0)?; + } + continue; + } + ZipEncoderState::LocalFiles { reader, .. } => reader, + ZipEncoderState::CentralDirectory { reader, .. } => reader, + ZipEncoderState::EndOfCentralDirectory { reader } => reader, + ZipEncoderState::Done => { + return Ok(0); + } + }; + let n = reader.read(buf)?; + self.bytes_written += n; + if n > 0 || buf.is_empty() { + return Ok(n); + } + self.state = match &self.state { + ZipEncoderState::Init => unreachable!(), + ZipEncoderState::LocalFiles { index, reader } => { + if let Some(crc) = reader.crc() { + self.files[*index].crc = crc; + let next_index = *index + 1; + if next_index < self.files.len() { + self.files[next_index].offset = self.bytes_written as u32; + self.new_local_files_state(next_index)? + } else { + self.new_central_directory_state(0, self.bytes_written) + } + } else { + // Will never panic because the above read() call returned Ok(0), which + // means the LocalFileReader is in the LocalFileReaderState::Done state. + unreachable!() + } + } + ZipEncoderState::CentralDirectory { + index, + start_offset, + .. + } => { + let next_index = *index + 1; + if next_index < self.files.len() { + self.new_central_directory_state(next_index, *start_offset) + } else { + self.new_end_of_central_directory_state(*start_offset) + } + } + ZipEncoderState::EndOfCentralDirectory { .. } => ZipEncoderState::Done, + ZipEncoderState::Done => unreachable!(), + } + } + } +} + +#[derive(Clone)] +pub struct ZipEntryInfo { + path: PathBuf, + name: Vec, + size: u64, + /// Offset from the start of the file to the local file header. + /// Gets filled in by ZipEncoder before it reads the file. + offset: u32, + // Checksum of the (uncompressed) file data. + // Gets filled in by ZipEncoder after the file is read. + crc: u32, +} + +impl ZipEntryInfo { + /// Creates a new ZipEntryInfo from a path to a file. + /// The path must be relative to the base path. The base path is used to determine the name of + /// the file in the zip, by stripping the base path from the file path. + pub fn new(path: PathBuf, base: &Path) -> Result { + let name = path.strip_prefix(base)?.as_os_str().as_bytes().to_owned(); + let metadata = path.metadata()?; + Ok(Self { + path, + name, + size: metadata.len(), + crc: 0, + offset: 0, + }) + } +} + +#[derive(Clone, Copy)] +enum FileHeaderKind { + Local, + CentralDirectory, +} + +pub fn zip_stream_len_empty() -> usize { + END_OF_CENTRAL_DIRECTORY_SIZE +} + +/// Returns the size of the entire zip stream in bytes, given a slice of ZipEntryInfo. +pub fn zip_stream_len(files: &[ZipEntryInfo]) -> usize { + files.iter().map(zip_stream_len_for_file).sum::() + zip_stream_len_empty() +} + +/// Returns the size that a single ZipEntryInfo contributes to the size of the zip stream. +pub fn zip_stream_len_for_file(file_info: &ZipEntryInfo) -> usize { + header_size(file_info, FileHeaderKind::Local) + + file_info.size as usize + + DATA_DESCRIPTOR_SIZE + + header_size(file_info, FileHeaderKind::CentralDirectory) +} + +fn header_size(info: &ZipEntryInfo, kind: FileHeaderKind) -> usize { + const LOCAL_FILE_HEADER_SIZE: usize = 30; + const DIRECTORY_HEADER_SIZE: usize = 46; + let name_len = info.name.len(); + match kind { + FileHeaderKind::Local => LOCAL_FILE_HEADER_SIZE + name_len, + FileHeaderKind::CentralDirectory => DIRECTORY_HEADER_SIZE + name_len, + } +} + +fn make_file_header(info: &ZipEntryInfo, kind: FileHeaderKind) -> Vec { + let mut header = Vec::with_capacity(header_size(info, kind)); + header.extend_from_slice(match &kind { + FileHeaderKind::Local => { + // Signature + // Version needed to extract + // General purpose bit flag (data descriptor enabled) + // Compression mode (store / no compression) + // File last modified time (all zeroes) + // File last modified date (all zeroes) + b"PK\x03\x04\ + \x0A\x00\ + \x08\x00\ + \x00\x00\ + \x00\x00\ + \x00\x00\ + " + } + FileHeaderKind::CentralDirectory => { + // Signature + // Version made by + // Version needed to extract + // General purpose bit flag (data descriptor enabled) + // Compression mode (store / no compression) + // File last modified time (all zeroes) + // File last modified date (all zeroes) + b"PK\x01\x02\ + \x0A\x00\ + \x0A\x00\ + \x08\x00\ + \x00\x00\ + \x00\x00\ + \x00\x00\ + " + } + }); + // CRC-32 of uncompressed data: + header.extend_from_slice(&info.crc.to_le_bytes()); + let size_slice = &(info.size as u32).to_le_bytes(); + // Compressed size: + header.extend_from_slice(size_slice); + // Uncompressed size: + header.extend_from_slice(size_slice); + // File name length: + header.extend_from_slice(&(info.name.len() as u16).to_le_bytes()); + // Extra field length (0 bytes): + header.extend_from_slice(b"\x00\x00"); + + if let FileHeaderKind::CentralDirectory = &kind { + // File comment length (0 bytes) + // Disk number where file starts (0) + // Internal file attributes (0) + // External file attributes (0) + header.extend_from_slice( + b"\x00\x00\ + \x00\x00\ + \x00\x00\ + \x00\x00\x00\x00\ + ", + ); + + // Relative offset of local file header: + header.extend_from_slice(&info.offset.to_le_bytes()); + }; + + // File name: + header.extend_from_slice(&info.name); + header +} + +const DATA_DESCRIPTOR_SIZE: usize = 16; + +fn make_data_descriptor(crc: u32, size: u32) -> Vec { + let mut desc = Vec::with_capacity(DATA_DESCRIPTOR_SIZE); + // Signature: + desc.extend_from_slice(b"PK\x07\x08"); + // CRC-32 of uncompressed data: + desc.extend_from_slice(&crc.to_le_bytes()); + let size_slice = &size.to_le_bytes(); + // Compressed size: + desc.extend_from_slice(size_slice); + // Uncompressed size: + desc.extend_from_slice(size_slice); + desc +} + +const END_OF_CENTRAL_DIRECTORY_SIZE: usize = 22; + +fn make_end_of_central_directory(num_files: u16, size: u32, offset: u32) -> Vec { + let mut desc = Vec::with_capacity(END_OF_CENTRAL_DIRECTORY_SIZE); + desc.extend_from_slice( + // Signature + // Number of this disk + // Disk where central directory starts + b"PK\x05\x06\ + \x00\x00\ + \x00\x00\ + ", + ); + let num_files_slice = &num_files.to_le_bytes(); + // Number of central directory records on this disk: + desc.extend_from_slice(num_files_slice); + // Total number of central directory records + desc.extend_from_slice(num_files_slice); + // Size of central directory + desc.extend_from_slice(&size.to_le_bytes()); + // Offset of start of central directory + desc.extend_from_slice(&offset.to_le_bytes()); + // Comment length + desc.extend_from_slice(b"\x00\x00"); + desc +} + +struct LocalFileReader { + state: LocalFileReaderState, +} + +enum LocalFileReaderState { + Header { reader: Cursor>, file: File }, + Data { reader: CrcReader }, + DataDescriptor { reader: Cursor>, crc: u32 }, + Done { crc: u32 }, +} + +impl LocalFileReader { + pub fn new(info: &ZipEntryInfo) -> std::io::Result { + Ok(Self { + state: LocalFileReaderState::Header { + reader: Cursor::new(make_file_header(info, FileHeaderKind::Local)), + file: File::open(&info.path)?, + }, + }) + } + + pub fn crc(&self) -> Option { + match self.state { + LocalFileReaderState::Done { crc } => Some(crc), + _ => None, + } + } +} + +impl Read for LocalFileReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + loop { + let reader: &mut dyn Read = match &mut self.state { + LocalFileReaderState::Header { reader, .. } => reader, + LocalFileReaderState::Data { reader } => reader, + LocalFileReaderState::DataDescriptor { reader, .. } => reader, + LocalFileReaderState::Done { .. } => { + return Ok(0); + } + }; + let n = reader.read(buf)?; + if n > 0 || buf.is_empty() { + return Ok(n); + } + take(&mut self.state, |state| match state { + LocalFileReaderState::Header { file, .. } => LocalFileReaderState::Data { + reader: CrcReader::new(file), + }, + LocalFileReaderState::Data { reader: crc_reader } => { + let crc = crc_reader.crc().sum(); + LocalFileReaderState::DataDescriptor { + reader: Cursor::new(make_data_descriptor(crc, crc_reader.crc().amount())), + crc, + } + } + LocalFileReaderState::DataDescriptor { crc, .. } => { + LocalFileReaderState::Done { crc } + } + LocalFileReaderState::Done { .. } => unreachable!(), + }) + } + } +} + +#[cfg(test)] +mod tests { + use std::io::{copy, Cursor}; + use std::io::{Seek, SeekFrom}; + + use crate::test_utils::create_file_with_contents; + use tempfile::tempdir; + use zip::ZipArchive; + + use super::*; + + /// Makes an empty zip file, then reads it with a "known good" zip unarchiver ('zip' crate). + #[test] + fn test_empty() { + let (zip, zip_encoder) = zip_round_trip(vec![]); + assert!(zip.is_empty()); + + let stream_len = zip.into_inner().into_inner().len(); + assert_eq!(stream_len as u64, zip_encoder.stream_len()); + } + + /// Makes a zip with some files, then reads it with a "known good" zip unarchiver ('zip' crate). + #[test] + fn test_basic() { + let tmp = tempdir().unwrap(); + let tempdir_path = tmp.path(); + + let filenames_and_contents = [("hello.txt", "Hello World"), ("bye.txt", "Goodbye")]; + + for (filename, contents) in filenames_and_contents.iter() { + let file_path = tempdir_path.join(filename); + create_file_with_contents(&file_path, contents.as_bytes()).unwrap(); + } + + let file_infos = filenames_and_contents + .iter() + .map(|(filename, _)| { + let file_path = tempdir_path.join(filename); + ZipEntryInfo::new(file_path, tempdir_path).unwrap() + }) + .collect::>(); + + let (mut zip, zip_encoder) = zip_round_trip(file_infos); + assert_eq!(zip.len(), filenames_and_contents.len()); + + for (filename, contents) in filenames_and_contents.iter() { + let vec: Vec = Vec::with_capacity(1024); + let mut cursor = Cursor::new(vec); + copy(&mut zip.by_name(filename).unwrap(), &mut cursor).unwrap(); + assert_eq!(cursor.into_inner(), contents.as_bytes()); + } + + let stream_len = zip.into_inner().into_inner().len(); + assert_eq!(stream_len as u64, zip_encoder.stream_len()); + } + + fn zip_round_trip( + source_files: Vec, + ) -> (ZipArchive>>, ZipEncoder) { + let mut zip_encoder = ZipEncoder::new(source_files); + let vec: Vec = Vec::with_capacity(1024 * 8); + let mut cursor = Cursor::new(vec); + copy(&mut zip_encoder, &mut cursor).unwrap(); + + // Use flate2's ZipArchive to read the zip file we just created: + cursor.seek(SeekFrom::Start(0)).unwrap(); + let zip = ZipArchive::new(cursor).unwrap(); + (zip, zip_encoder) + } +}