diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..eb5a316 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +target diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..78af428 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2759 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "argh" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab257697eb9496bf75526f0217b5ed64636a9cfafa78b8365c71bd283fcef93e" +dependencies = [ + "argh_derive", + "argh_shared", +] + +[[package]] +name = "argh_derive" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b382dbd3288e053331f03399e1db106c9fb0d8562ad62cb04859ae926f324fa6" +dependencies = [ + "argh_shared", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "argh_shared" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cb94155d965e3d37ffbbe7cc5b82c3dd79dd33bd48e536f73d2cfb8d85506f" + +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cc" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32a725bc159af97c3e629873bb9f88fb8cf8a4867175f76dc987815ea07c83b" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-integer", + "num-traits", + "serde", + "time", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "chunked_transfer" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cca491388666e04d7248af3f60f0c40cfb0991c72205595d7c396e3510207d1a" + +[[package]] +name = "ciborium" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" + +[[package]] +name = "ciborium-ll" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "console" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "windows-sys 0.42.0", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "cpufeatures" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "cxx" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f61f1b6389c3fe1c316bf8a4dccc90a38208354b330925bce1f74a6c4756eb93" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cee708e8962df2aeb38f594aae5d827c022b6460ac71a7a3e2c3c2aae5a07b" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn 2.0.52", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7944172ae7e4068c533afbb984114a56c46e9ccddda550499caa222902c7f7bb" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "digest" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" + +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "eyre" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c2b6b5a29c02cdc822728b7d7b8ae1bab3e3b05d44522770ddd49722eeac7eb" +dependencies = [ + "indenter", + "once_cell", +] + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "flate2" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +dependencies = [ + "crc32fast", + "libz-sys", + "miniz_oxide", +] + +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fragile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-executor" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-timer" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "goblin" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27c1b4369c2cd341b5de549380158b105a04c331be5db9110eef7b6d2742134" +dependencies = [ + "log", + "plain", + "scroll", +] + +[[package]] +name = "governor" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c390a940a5d157878dd057c78680a33ce3415bcd05b4799509ea44210914b4d5" +dependencies = [ + "cfg-if", + "futures", + "futures-timer", + "no-std-compat", + "nonzero_ext", + "parking_lot", + "smallvec", +] + +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http", + "hyper", + "rustls", + "tokio", + "tokio-rustls", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indenter" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" + +[[package]] +name = "indexmap" +version = "2.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "insta" +version = "1.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a28d25139df397cbca21408bb742cf6837e04cdbebf1b07b760caf971d6a972" +dependencies = [ + "console", + "lazy_static", + "linked-hash-map", + "pest", + "pest_derive", + "serde", + "similar", + "yaml-rust", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi 0.3.9", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" + +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "kernlog" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0dd79b6ea06a97c93bc5587fba4c2ed6d723b939d35d5e3fb3c6870d12e6d17" +dependencies = [ + "libc", + "log", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.153" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "libz-sys" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "link-cplusplus" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" +dependencies = [ + "cc", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "linux-raw-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" + +[[package]] +name = "linux-raw-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" + +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" + +[[package]] +name = "memchr" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" + +[[package]] +name = "memfaultc-sys" +version = "0.1.0" +dependencies = [ + "cc", + "libc", + "pkg-config", + "walkdir", +] + +[[package]] +name = "memfaultd" +version = "0.1.0" +dependencies = [ + "argh", + "cfg-if", + "chrono", + "ciborium", + "crc", + "crc-catalog", + "eyre", + "flate2", + "fs_extra", + "goblin", + "governor", + "hex", + "insta", + "itertools", + "kernlog", + "libc", + "log", + "memfaultc-sys", + "mockall", + "nix", + "nom", + "once_cell", + "prctl", + "procfs", + "psm", + "rand", + "regex", + "reqwest", + "rmp-serde", + "rmpv", + "rstest", + "scroll", + "sealed_test", + "serde", + "serde_bytes", + "serde_json", + "serde_repr", + "shuteye", + "signal-hook", + "stderrlog", + "strum", + "strum_macros", + "take_mut", + "tempfile", + "thiserror", + "threadpool", + "tiny_http", + "urlencoding", + "uuid", + "zip", +] + +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.48.0", +] + +[[package]] +name = "mockall" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c84490118f2ee2d74570d114f3d0493cbf02790df303d2707606c3e14e07c96" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nix" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", + "memoffset", + "pin-utils", + "static_assertions", +] + +[[package]] +name = "no-std-compat" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nonzero_ext" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +dependencies = [ + "hermit-abi 0.2.6", + "libc", +] + +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "openssl" +version = "0.10.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +dependencies = [ + "bitflags 2.4.2", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-src" +version = "300.2.3+3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cff92b6f71555b61bb9315f7c64da3ca43d87531622120fea0195fc761b4843" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff" +dependencies = [ + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys 0.45.0", +] + +[[package]] +name = "paste" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pest" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e68e84bfb01f0507134eac1e9b410a12ba379d064eab48c50ba4ce329a527b70" +dependencies = [ + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b79d4c71c865a25a4322296122e3924d30bc8ee0834c8bfc8b95f7f054afbfb" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c435bf1076437b851ebc8edc3a18442796b30f1728ffea6262d59bbe28b077e" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.52", +] + +[[package]] +name = "pest_meta" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "745a452f8eb71e39ffd8ee32b3c5f51d03845f99786fa9b68db6ff509c505411" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "prctl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "059a34f111a9dee2ce1ac2826a68b24601c4298cfeb1a587c3cb493d5ab46f52" +dependencies = [ + "libc", + "nix", +] + +[[package]] +name = "predicates" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" +dependencies = [ + "difflib", + "float-cmp", + "itertools", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" + +[[package]] +name = "predicates-tree" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "proc-macro2" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "procfs" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "943ca7f9f29bab5844ecd8fdb3992c5969b6622bb9609b9502fef9b4310e3f1f" +dependencies = [ + "bitflags 1.3.2", + "byteorder", + "chrono", + "flate2", + "hex", + "lazy_static", + "rustix 0.36.16", +] + +[[package]] +name = "psm" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" +dependencies = [ + "cc", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "regex" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + +[[package]] +name = "reqwest" +version = "0.11.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78bf93c4af7a8bb7d879d51cebe797356ff10ae8516ace542b5182d9dcac10b2" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-rustls", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-rustls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", + "winreg", +] + +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rmp" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44519172358fd6d58656c86ab8e7fbc9e1490c3e8f14d35ed78ca0dd07403c9f" +dependencies = [ + "byteorder", + "num-traits", + "paste", +] + +[[package]] +name = "rmp-serde" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5b13be192e0220b8afb7222aa5813cb62cc269ebb5cac346ca6487681d2913e" +dependencies = [ + "byteorder", + "rmp", + "serde", +] + +[[package]] +name = "rmpv" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de8813b3a2f95c5138fe5925bfb8784175d88d6bff059ba8ce090aa891319754" +dependencies = [ + "num-traits", + "rmp", +] + +[[package]] +name = "rstest" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b07f2d176c472198ec1e6551dc7da28f1c089652f66a7b722676c2238ebc0edf" +dependencies = [ + "futures", + "futures-timer", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7229b505ae0706e64f37ffc54a9c163e11022a6636d58fe1f3f52018257ff9f7" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "rustc_version", + "syn 1.0.109", + "unicode-ident", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.36.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6da3636faa25820d8648e0e31c5d519bbb01f72fdf57131f0f5f7da5fed36eab" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys 0.1.4", + "windows-sys 0.45.0", +] + +[[package]] +name = "rustix" +version = "0.38.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +dependencies = [ + "bitflags 2.4.2", + "errno", + "libc", + "linux-raw-sys 0.4.13", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.21.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" + +[[package]] +name = "rusty-forkfork" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce85af4dfa2fb0c0143121ab5e424c71ea693867357c9159b8777b59984c218" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "ryu" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "scratch" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" + +[[package]] +name = "scroll" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04c565b551bafbef4157586fa379538366e4385d42082f255bfd96e4fe8519da" +dependencies = [ + "scroll_derive", +] + +[[package]] +name = "scroll_derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "sealed_test" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a1867f8f005bd7fb73c367e2e45dd628417906a2ca27597fe59cbf04279a222" +dependencies = [ + "fs_extra", + "rusty-forkfork", + "sealed_test_derive", + "tempfile", +] + +[[package]] +name = "sealed_test_derive" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77253fb2d4451418d07025826028bcb96ee42d3e58859689a70ce62908009db6" +dependencies = [ + "quote", + "syn 2.0.52", +] + +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" + +[[package]] +name = "serde" +version = "1.0.197" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "416bda436f9aab92e02c8e10d49a15ddd339cea90b6e340fe51ed97abb548294" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.197" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + +[[package]] +name = "serde_json" +version = "1.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_repr" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcec881020c684085e55a25f7fd888954d56609ef363479dc5a1305eb0d40cab" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shuteye" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ca27229f5eb21ea854a551c24b268c824949a01c41652b6d423c141539252cf" +dependencies = [ + "libc", +] + +[[package]] +name = "signal-hook" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "732768f1176d21d09e076c23a93123d40bba92d50c4058da34d45c8de8e682b9" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "similar" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf" + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "socket2" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "stderrlog" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69a26bbf6de627d389164afa9783739b56746c6c72c4ed16539f4ff54170327b" +dependencies = [ + "atty", + "chrono", + "log", + "termcolor", + "thread_local", +] + +[[package]] +name = "strum" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 1.0.109", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" + +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if", + "fastrand", + "rustix 0.38.31", + "windows-sys 0.52.0", +] + +[[package]] +name = "termcolor" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "termtree" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" + +[[package]] +name = "thiserror" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + +[[package]] +name = "thread_local" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "time" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + +[[package]] +name = "tiny_http" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82" +dependencies = [ + "ascii", + "chunked_transfer", + "httpdate", + "log", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" + +[[package]] +name = "ucd-trie" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" + +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "urlencoding" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" + +[[package]] +name = "uuid" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dad5567ad0cf5b760e5665964bec1b47dfd077ba8a2544b513f3556d3d239a2" +dependencies = [ + "getrandom", + "serde", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.52", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + +[[package]] +name = "web-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.4", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "zip" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0445d0fbc924bb93539b4316c11afb121ea39296f99a3c4c9edad09e3658cdef" +dependencies = [ + "byteorder", + "crc32fast", + "crossbeam-utils", + "flate2", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..3e6570b --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,15 @@ +[workspace] + +members = [ + "memfaultd", + "memfaultc-sys" +] +resolver = "2" + +[profile.release] +# We do not handle FFI unwinding so we need to keep panic abort for now. +panic="abort" +opt-level = "z" + +[profile.dev] +panic="abort" diff --git a/Cross.toml b/Cross.toml new file mode 100644 index 0000000..5229193 --- /dev/null +++ b/Cross.toml @@ -0,0 +1,13 @@ +[target.aarch64-unknown-linux-gnu] +pre-build = [ + "dpkg --add-architecture $CROSS_DEB_ARCH", + "apt-get update", + "apt-get install --assume-yes libsystemd-dev:$CROSS_DEB_ARCH libconfig-dev:$CROSS_DEB_ARCH" +] + +[target.x86_64-unknown-linux-gnu] +pre-build = [ + "dpkg --add-architecture $CROSS_DEB_ARCH", + "apt-get update", + "apt-get install --assume-yes libsystemd-dev:$CROSS_DEB_ARCH libconfig-dev:$CROSS_DEB_ARCH" +] diff --git a/License.txt b/License.txt new file mode 100644 index 0000000..c863fb1 --- /dev/null +++ b/License.txt @@ -0,0 +1,31 @@ +Copyright (c) 2019 - Present, Memfault +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code or in binary form must reproduce +the above copyright notice, this list of conditions and the following +disclaimer in the documentation and/or other materials provided with the +distribution. + +2. Neither the name of Memfault nor the names of its contributors may be +used to endorse or promote products derived from this software without +specific prior written permission. + +3. This software, with or without modification, must only be used with +the Memfault services and integrated with the Memfault server. + +4. Any software provided in binary form under this license must not be +reverse engineered, decompiled, modified and/or disassembled. + +THIS SOFTWARE IS PROVIDED BY MEMFAULT "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, +NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT +SHALL MEMFAULT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..0399447 --- /dev/null +++ b/README.md @@ -0,0 +1,37 @@ +# `memfaultd` + +`memfaultd` is a daemon that runs on your device and collects crash reports and +metrics. It is the core of the +[Memfault Linux SDK](https://github.com/memfault/memfault-linux-sdk/blob/kirkstone/README.md). + +## Overview + +`memfaultd` supports several features to help you maintain and debug your fleet +of devices: + +- **Crash reporting**: When your device crashes, `memfaultd` collects a crash + report from + [Linux Coredumps](https://man7.org/linux/man-pages/man5/core.5.html). For more + information, see the + [Coredumps documentation](https://docs.memfault.com/docs/linux/coredumps). + +- **Metrics**: `memfaultd` collects metrics from your device and uploads them to + Memfault. For more information, see the + [Metrics documentation](https://docs.memfault.com/docs/linux/metrics). + +- **Reboot reason tracking**: `memfaultd` detects various reboot reasons from + the system and reports them to the Memfault Dashboard. Users can also provide + a specific reboot reason before restarting the device. For more information, + see the + [Reboot Reason Tracking documentation](https://docs.memfault.com/docs/linux/reboot-reason-tracking). + +- **OTA Updates**: `memfaultd` supports [SWUpdate](https://swupdate.org/) out of + the box and is able to configure it to talk to our hawkBit DDI-compatible + endpoint. For more information, see the + [Linux OTA Management documentation](https://docs.memfault.com/docs/linux/ota). + +- **Logging**: `memfaultd` collects logs from your system. For more information, + see the [Logging documentation](https://docs.memfault.com/docs/linux/logging). + +And much more! [Register](https://app.memfault.com/register) and get started +today! diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..295e6ae --- /dev/null +++ b/VERSION @@ -0,0 +1,3 @@ +BUILD ID: 2255289 +GIT COMMIT: add51b077b +VERSION: 1.13.0 diff --git a/libmemfaultc/.clang-format b/libmemfaultc/.clang-format new file mode 100644 index 0000000..0f374d7 --- /dev/null +++ b/libmemfaultc/.clang-format @@ -0,0 +1,10 @@ +--- +BasedOnStyle: Google +--- +Language: Cpp + +AlwaysBreakBeforeMultilineStrings: false +ColumnLimit: '100' +ContinuationIndentWidth: 2 +IndentPPDirectives: BeforeHash +... diff --git a/libmemfaultc/.gitignore b/libmemfaultc/.gitignore new file mode 100644 index 0000000..65af97a --- /dev/null +++ b/libmemfaultc/.gitignore @@ -0,0 +1,2 @@ +/build +/.cache diff --git a/libmemfaultc/src/crash.c b/libmemfaultc/src/crash.c new file mode 100644 index 0000000..1afeb00 --- /dev/null +++ b/libmemfaultc/src/crash.c @@ -0,0 +1,15 @@ +//! @file +//! +//! Copyright (c) Memfault, Inc. +//! See License.txt for details +//! +//! @brief +//! + +#include + +void memfault_trigger_fp_exception(void) { + int divisor = 0; + // Triggers an illegal instruction on x86 - Floating Point Error on ARM + printf("%i", 42 / divisor); +} diff --git a/libmemfaultc/src/swupdate.c b/libmemfaultc/src/swupdate.c new file mode 100644 index 0000000..da13cc4 --- /dev/null +++ b/libmemfaultc/src/swupdate.c @@ -0,0 +1,215 @@ +//! @file +//! +//! Copyright (c) Memfault, Inc. +//! See License.txt for details +//! +//! @brief +//! Memfault SWUpdate config file generation + +#include +#include +#include +#include +#include + +#define DEFAULT_SURICATTA_TENANT "default" + +#define HAWKBIT_PATH "/api/v0/hawkbit" + +typedef struct { + char *base_url; + char *software_version; + char *software_type; + char *hardware_version; + char *device_id; + char *project_key; + + char *input_file; + char *output_file; +} sMemfaultdSwupdateConfig; + +/** + * @brief Add 'global' section to config + * + * @param config config object to build into + * @return true Successfully added global options to config + * @return false Failed to add + */ +static bool prv_swupdate_add_globals(config_t *config) { + if (!config_lookup(config, "globals")) { + if (!config_setting_add(config_root_setting(config), "globals", CONFIG_TYPE_GROUP)) { + fprintf(stderr, "swupdate:: Failed to add globals setting group\n"); + return false; + } + } + return true; +} + +/** + * @brief Add 'suricatta' section to config + * + * @param handle swupdate config handle + * @param config config object to build into + * @return true Successfully added suricatta options to config + * @return false Failed to add + */ +static bool prv_swupdate_add_suricatta(sMemfaultdSwupdateConfig *handle, config_t *config) { + config_setting_t *suricatta = config_lookup(config, "suricatta"); + if (!suricatta) { + if (!(suricatta = + config_setting_add(config_root_setting(config), "suricatta", CONFIG_TYPE_GROUP))) { + fprintf(stderr, "swupdate:: Failed to add suricatta group\n"); + return false; + } + } + + char *url = malloc(strlen(HAWKBIT_PATH) + strlen(handle->base_url) + 1); + strcpy(url, handle->base_url); + strcat(url, HAWKBIT_PATH); + + config_setting_t *element; + config_setting_remove(suricatta, "url"); + if (!(element = config_setting_add(suricatta, "url", CONFIG_TYPE_STRING)) || + !config_setting_set_string(element, url)) { + fprintf(stderr, "swupdate:: Failed to add suricatta:url\n"); + free(url); + return false; + } + + free(url); + + config_setting_remove(suricatta, "id"); + if (!(element = config_setting_add(suricatta, "id", CONFIG_TYPE_STRING)) || + !config_setting_set_string(element, handle->device_id)) { + fprintf(stderr, "swupdate:: Failed to add suricatta:id\n"); + return false; + } + + config_setting_remove(suricatta, "tenant"); + if (!(element = config_setting_add(suricatta, "tenant", CONFIG_TYPE_STRING)) || + !config_setting_set_string(element, DEFAULT_SURICATTA_TENANT)) { + fprintf(stderr, "swupdate:: Failed to add suricatta:tenant\n"); + return false; + } + + config_setting_remove(suricatta, "gatewaytoken"); + if (!(element = config_setting_add(suricatta, "gatewaytoken", CONFIG_TYPE_STRING)) || + !config_setting_set_string(element, handle->project_key)) { + fprintf(stderr, "swupdate:: Failed to add suricatta:id\n"); + return false; + } + + return true; +} + +/** + * @brief Add 'identify' section to config + * + * @param handle swupdate config handle + * @param config config object to build into + * @return true Successfully added identify options to config + * @return false Failed to add + */ +static bool prv_swupdate_add_identify(sMemfaultdSwupdateConfig *handle, config_t *config) { + config_setting_t *identify; + + config_setting_remove(config_root_setting(config), "identify"); + if (!(identify = config_setting_add(config_root_setting(config), "identify", CONFIG_TYPE_LIST))) { + fprintf(stderr, "swupdate:: Failed to add identify list\n"); + return false; + } + + config_setting_t *setting; + config_setting_t *element; + if (!(setting = config_setting_add(identify, NULL, CONFIG_TYPE_GROUP))) { + fprintf(stderr, "swupdate:: Failed to add identify current_version\n"); + return false; + } + if (!(element = config_setting_add(setting, "name", CONFIG_TYPE_STRING)) || + !config_setting_set_string(element, "memfault__current_version")) { + fprintf(stderr, "swupdate:: Failed to add identify current_version\n"); + return false; + } + if (!(element = config_setting_add(setting, "value", CONFIG_TYPE_STRING)) || + !config_setting_set_string(element, handle->software_version)) { + fprintf(stderr, "swupdate:: Failed to add identify current_version\n"); + return false; + } + + if (!(setting = config_setting_add(identify, NULL, CONFIG_TYPE_GROUP))) { + fprintf(stderr, "swupdate:: Failed to add identify hardware_version\n"); + return false; + } + if (!(element = config_setting_add(setting, "name", CONFIG_TYPE_STRING)) || + !config_setting_set_string(element, "memfault__hardware_version")) { + fprintf(stderr, "swupdate:: Failed to add identify hardware_version\n"); + return false; + } + if (!(element = config_setting_add(setting, "value", CONFIG_TYPE_STRING)) || + !config_setting_set_string(element, handle->hardware_version)) { + fprintf(stderr, "swupdate:: Failed to add identify hardware_version\n"); + return false; + } + + if (!(setting = config_setting_add(identify, NULL, CONFIG_TYPE_GROUP))) { + fprintf(stderr, "swupdate:: Failed to add identify software_type\n"); + return false; + } + if (!(element = config_setting_add(setting, "name", CONFIG_TYPE_STRING)) || + !config_setting_set_string(element, "memfault__software_type")) { + fprintf(stderr, "swupdate:: Failed to add identify software_type\n"); + return false; + } + if (!(element = config_setting_add(setting, "value", CONFIG_TYPE_STRING)) || + !config_setting_set_string(element, handle->software_type)) { + fprintf(stderr, "swupdate:: Failed to add identify software_type\n"); + return false; + } + + return true; +} + +/** + * @brief Generate new swupdate.cfg file from config + * + * @param handle swupdate config handle + * @return true Successfully generated new config + * @return false Failed to generate + */ +bool memfault_swupdate_generate_config(sMemfaultdSwupdateConfig *handle) { + config_t config; + + config_init(&config); + if (!config_read_file(&config, handle->input_file)) { + fprintf(stderr, + "swupdate:: Failed to read '%s', proceeding " + "with defaults\n", + handle->input_file); + } + + if (!prv_swupdate_add_globals(&config)) { + fprintf(stderr, "swupdate:: Failed to add global options to config\n"); + config_destroy(&config); + return false; + } + if (!prv_swupdate_add_suricatta(handle, &config)) { + fprintf(stderr, "swupdate:: Failed to add suricatta options to config\n"); + config_destroy(&config); + return false; + } + if (!prv_swupdate_add_identify(handle, &config)) { + fprintf(stderr, "swupdate:: Failed to add identify options to config\n"); + config_destroy(&config); + return false; + } + + if (!config_write_file(&config, handle->output_file)) { + fprintf(stderr, "swupdate:: Failed to write config file to '%s'\n", handle->output_file); + config_destroy(&config); + return false; + } + + config_destroy(&config); + + return true; +} diff --git a/libmemfaultc/src/systemd.c b/libmemfaultc/src/systemd.c new file mode 100644 index 0000000..6b322f2 --- /dev/null +++ b/libmemfaultc/src/systemd.c @@ -0,0 +1,200 @@ +//! @file +//! +//! Copyright (c) Memfault, Inc. +//! See License.txt for details +//! +//! @brief +//! memfaultd systemd helper + +#include +#include +#include +#include + +static const char *const systemd_service = "org.freedesktop.systemd1"; + +/** + * @param state output variable for current state (to be freed by caller). + */ +static bool prv_systemd_get_service_state(sd_bus *bus, const char *service_name, char **state) { + static const char *const unit_interface = "org.freedesktop.systemd1.Unit"; + + char *unit_path = NULL; + bool result = true; + + if (sd_bus_path_encode("/org/freedesktop/systemd1/unit", service_name, &unit_path) < 0) { + fprintf(stderr, "memfaultd:: Failed to generate SystemD unit path\n"); + + return false; + } + + sd_bus_error error = SD_BUS_ERROR_NULL; + if (sd_bus_get_property_string(bus, systemd_service, unit_path, unit_interface, "ActiveState", + &error, state) < 0) { + fprintf(stderr, "memfaultd:: Failed to get state of %s: %s\n", service_name, error.name); + sd_bus_error_free(&error); + result = false; + } + + if (unit_path) { + free(unit_path); + } + return result; +} + +/* + * List of SystemD commands: https://www.freedesktop.org/wiki/Software/systemd/dbus/ + */ + +static bool prv_systemd_restart_service(sd_bus *bus, const char *service_name) { + const char *manager_path = "/org/freedesktop/systemd1"; + const char *manager_interface = "org.freedesktop.systemd1.Manager"; + + sd_bus_error error = SD_BUS_ERROR_NULL; + sd_bus_message *msg = NULL; + + if (sd_bus_call_method(bus, systemd_service, manager_path, manager_interface, "RestartUnit", + &error, &msg, "ss", service_name, "replace") < 0) { + fprintf(stderr, "memfaultd:: Failed to restart %s: %s\n", service_name, error.name); + sd_bus_error_free(&error); + return false; + } + + sd_bus_message_unref(msg); + + return true; +} + +static bool prv_systemd_kill_service(sd_bus *bus, const char *service_name, int signal) { + const char *manager_path = "/org/freedesktop/systemd1"; + const char *manager_interface = "org.freedesktop.systemd1.Manager"; + + sd_bus_error error = SD_BUS_ERROR_NULL; + sd_bus_message *msg = NULL; + + /* + Refer to SystemD Bus documentation for arguments to the call: + KillUnit(in s name, + in s who, // "all": is the default (like systemctl kill service) + in i signal); + */ + if (sd_bus_call_method(bus, systemd_service, manager_path, manager_interface, "KillUnit", &error, + &msg, "ssi", service_name, "all", signal) < 0) { + fprintf(stderr, "memfaultd:: Failed to kill %s: %s\n", service_name, error.name); + sd_bus_error_free(&error); + return false; + } + + sd_bus_message_unref(msg); + + return true; +} + +/** + * @brief Restart service using systemd dbus API if already running + * + * @param src_module Friendly name of caller module + * @param service_name Service name to restart, e.g. collectd.service + * @return true Successfully restarted requested service + * @return false Failed to restart service + */ +bool memfaultd_restart_systemd_service_if_running(const char *service_name) { + bool result = true; + + // Initialize connection to SystemD + sd_bus *bus; + if (sd_bus_default_system(&bus) < 0) { + fprintf(stderr, "memfaultd:: Failed to find systemd system bus\n"); + return false; + } + + // Check if service is active before restarting it + char *state = NULL; + if (!prv_systemd_get_service_state(bus, service_name, &state)) { + result = false; + goto cleanup; + } + if (strcmp("active", state) != 0 && strcmp("activating", state) != 0) { + fprintf(stderr, "memfaultd:: %s is not active (%s). Not starting.\n", service_name, state); + result = true; + goto cleanup; + } + + // Restart the service + if (!prv_systemd_restart_service(bus, service_name)) { + result = false; + goto cleanup; + } + +cleanup: + if (state != NULL) { + free(state); + } + sd_bus_unref(bus); + return result; +} + +/** + * @brief Send a signal to a service if it's running. + * + * @param src_module Friendly name of caller module + * @param service_name Service name to "kill", e.g. collectd.service + * @param signal Signal to send, e.g. SIGUSR1 + * @return true Successfully sent signal to requested service + * @return false Failed to restart service + */ +bool memfaultd_kill_systemd_service(const char *service_name, int signal) { + bool result = true; + + // Initialize connection to SystemD + sd_bus *bus; + if (sd_bus_default_system(&bus) < 0) { + fprintf(stderr, "memfaultd:: Failed to find systemd system bus\n"); + return false; + } + + // Send signal to service + if (!prv_systemd_kill_service(bus, service_name, signal)) { + result = false; + goto cleanup; + } + +cleanup: + sd_bus_unref(bus); + return result; +} + +/** + * @brief Checks if the current systemd state matches the requested state + * + * @return NULL Failed to get current systemd state + * + * @note Caller must free returned string + */ +char *memfaultd_get_systemd_bus_state() { + sd_bus *bus = NULL; + sd_bus_error error = SD_BUS_ERROR_NULL; + char *cur_state = NULL; + + const char *service = "org.freedesktop.systemd1"; + const char *path = "/org/freedesktop/systemd1"; + const char *interface = "org.freedesktop.systemd1.Manager"; + const char *system_state = "SystemState"; + + const int bus_result = sd_bus_default_system(&bus); + if (bus_result < 0) { + fprintf(stderr, "reboot:: Failed to find systemd system bus: %s\n", strerror(-bus_result)); + goto cleanup; + } + + if (sd_bus_get_property_string(bus, service, path, interface, system_state, &error, &cur_state) < + 0) { + fprintf(stderr, "reboot:: Failed to get system state: %s\n", error.name); + goto cleanup; + } + +cleanup: + sd_bus_error_free(&error); + sd_bus_unref(bus); + return cur_state; +} diff --git a/memfaultc-sys/Cargo.toml b/memfaultc-sys/Cargo.toml new file mode 100644 index 0000000..d89ea2a --- /dev/null +++ b/memfaultc-sys/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "memfaultc-sys" +version = "0.1.0" +edition = "2021" +autobins = false + +[dependencies] +libc = "0.2.138" + +[build-dependencies] +cc = "1.0.95" +pkg-config = "0.3" +walkdir = "2" + +[features] +default = [] +coredump = [] +systemd = [] +swupdate = [] diff --git a/memfaultc-sys/build.rs b/memfaultc-sys/build.rs new file mode 100644 index 0000000..ea9abcf --- /dev/null +++ b/memfaultc-sys/build.rs @@ -0,0 +1,78 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use cc::Build; +use std::env; +use walkdir::WalkDir; + +static LIBMEMFAULTC: &str = "../libmemfaultc/"; + +fn main() { + // Cross-compile C flags set by the CC Crate can conflict with the + // flags set by Yocto. Tell CC-rs to not set default flags. + env::set_var("CRATE_CC_NO_DEFAULTS", "true"); + + let mut cc = Build::new(); + cc.flag("-fPIC"); + cc.include(format!("{}/include", LIBMEMFAULTC)); + + cc.file(format!("{}/src/crash.c", LIBMEMFAULTC)); + + // Build a list of the library that we want to link into the final binary. + let mut libs = vec![]; + + if env::var("CARGO_FEATURE_SYSTEMD").is_ok() { + // Systemd is not available on macOS. We silently accept the feature so + // that the rust code can be checked but we don't actually build the C + // code. + if cfg!(not(target_os = "macos")) { + cc.file(format!("{}/src/systemd.c", LIBMEMFAULTC)); + libs.push("libsystemd"); + } + } + + if env::var("CARGO_FEATURE_SWUPDATE").is_ok() { + cc.file(format!("{}/src/swupdate.c", LIBMEMFAULTC)); + libs.push("libconfig"); + } + + // Linting needs to run `cargo` (and thus execute this file) to verify the + // project but the C dependencies are not installed on this machine. This + // environment variable will stop the script here, before doing any actual build. + // There is no standard Cargo way to tell if we are being called as part of `cargo lint`. + // See: https://github.com/rust-lang/cargo/issues/4001 + if env::var("MEMFAULTD_SKIP_CMAKE").is_ok() { + return; + } + + // Find required C libraries and tell Cargo how to link them + let pkg_config = pkg_config::Config::new(); + libs.iter().for_each(|lib| { + // This line will print the required Cargo config if the library is found. + match pkg_config.probe(lib) { + Ok(lib) => { + cc.includes(lib.include_paths); + } + Err(e) => println!("WARNING - Library {} was not found: {}", lib, e), + } + }); + + // Build the libmemfaultc library and link tell Cargo to link it in the project + cc.compile("memfaultc"); + println!("cargo:rustc-link-lib=static=memfaultc"); + + // Tell cargo to rebuild the project when any of the C project files changes + WalkDir::new("../libmemfaultc/src") + .into_iter() + .filter_map(Result::ok) + .filter_map( + |e| match e.path().extension().and_then(std::ffi::OsStr::to_str) { + Some("c") => Some(e), + Some("h") => Some(e), + _ => None, + }, + ) + .for_each(|e| println!("cargo:rerun-if-changed={}", e.path().display())); + + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/memfaultc-sys/src/coredump.rs b/memfaultc-sys/src/coredump.rs new file mode 100644 index 0000000..748322e --- /dev/null +++ b/memfaultc-sys/src/coredump.rs @@ -0,0 +1,6 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +extern "C" { + pub fn memfault_trigger_fp_exception(); +} diff --git a/memfaultc-sys/src/lib.rs b/memfaultc-sys/src/lib.rs new file mode 100644 index 0000000..3724f73 --- /dev/null +++ b/memfaultc-sys/src/lib.rs @@ -0,0 +1,18 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +// main functions + +#[cfg(feature = "coredump")] +pub mod coredump; + +#[cfg(all(feature = "systemd", not(target_os = "macos")))] +pub mod systemd; +#[cfg(all(feature = "systemd", target_os = "macos"))] +pub mod systemd_mock; + +#[cfg(all(feature = "systemd", target_os = "macos"))] +pub use systemd_mock as systemd; + +#[cfg(feature = "swupdate")] +pub mod swupdate; diff --git a/memfaultc-sys/src/swupdate.rs b/memfaultc-sys/src/swupdate.rs new file mode 100644 index 0000000..2b90807 --- /dev/null +++ b/memfaultc-sys/src/swupdate.rs @@ -0,0 +1,22 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use libc::c_char; + +#[repr(C)] +pub struct MemfaultSwupdateCtx { + pub base_url: *const c_char, + + pub software_version: *const c_char, + pub software_type: *const c_char, + pub hardware_version: *const c_char, + pub device_id: *const c_char, + pub project_key: *const c_char, + + pub input_file: *const c_char, + pub output_file: *const c_char, +} + +extern "C" { + pub fn memfault_swupdate_generate_config(ctx: *const MemfaultSwupdateCtx) -> bool; +} diff --git a/memfaultc-sys/src/systemd.rs b/memfaultc-sys/src/systemd.rs new file mode 100644 index 0000000..4e48d2b --- /dev/null +++ b/memfaultc-sys/src/systemd.rs @@ -0,0 +1,35 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::ffi::c_int; + +use libc::{c_char, size_t}; + +#[allow(non_camel_case_types)] +pub enum sd_journal {} + +extern "C" { + pub fn memfaultd_restart_systemd_service_if_running(service_name: *const c_char) -> bool; + pub fn memfaultd_get_systemd_bus_state() -> *const c_char; + pub fn sd_journal_open(ret: *mut *mut sd_journal, flags: c_int) -> c_int; + pub fn sd_journal_seek_tail(j: *mut sd_journal) -> c_int; + pub fn sd_journal_previous(j: *mut sd_journal) -> c_int; + pub fn sd_journal_next(j: *mut sd_journal) -> c_int; + pub fn sd_journal_get_data( + j: *mut sd_journal, + field: *const c_char, + data: *mut *mut u8, + l: *mut size_t, + ) -> c_int; + pub fn sd_journal_enumerate_data( + j: *mut sd_journal, + data: *mut *mut u8, + l: *mut size_t, + ) -> c_int; + pub fn sd_journal_get_fd(j: *mut sd_journal) -> c_int; + pub fn sd_journal_process(j: *mut sd_journal) -> c_int; + pub fn sd_journal_get_realtime_usec(j: *mut sd_journal, ret: *mut u64) -> c_int; + pub fn sd_journal_get_cursor(j: *mut sd_journal, cursor: *mut *const c_char) -> c_int; + pub fn sd_journal_seek_cursor(j: *mut sd_journal, cursor: *const c_char) -> c_int; + pub fn sd_journal_add_match(j: *mut sd_journal, data: *const c_char, size: size_t) -> c_int; +} diff --git a/memfaultc-sys/src/systemd_mock.rs b/memfaultc-sys/src/systemd_mock.rs new file mode 100644 index 0000000..67180a5 --- /dev/null +++ b/memfaultc-sys/src/systemd_mock.rs @@ -0,0 +1,119 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::ptr::null; + +use libc::{c_char, c_int, size_t}; + +#[allow(non_camel_case_types)] +pub enum sd_journal {} + +/// Get the status of the systemd service manager. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn memfaultd_restart_systemd_service_if_running(service_name: *const c_char) -> bool { + eprintln!("memfaultd_restart_systemd_service_if_running is not implemented for this target (restarting service {})", std::ffi::CStr::from_ptr(service_name).to_string_lossy()); + true +} + +/// Get the status of the systemd service manager. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn memfaultd_get_systemd_bus_state() -> *const c_char { + null() +} + +/// Open the systemd journal. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_open(_ret: *mut *mut sd_journal, _flags: c_int) -> c_int { + eprintln!("sd_journal_open is not implemented for this target"); + -1 +} + +/// Seek to the end of the systemd journal. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_seek_tail(_j: *mut sd_journal) -> c_int { + eprintln!("sd_journal_seek_tail is not implemented for this target"); + -1 +} + +/// Get the previous entry in the systemd journal. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_previous(_j: *mut sd_journal) -> c_int { + eprintln!("sd_journal_previous is not implemented for this target"); + -1 +} + +/// Get the next entry in the systemd journal. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_next(_j: *mut sd_journal) -> c_int { + eprintln!("sd_journal_next is not implemented for this target"); + -1 +} + +/// Get the data of the systemd journal entry field. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_get_data( + _j: *mut sd_journal, + _field: *const c_char, + _data: *mut *mut u8, + _l: *mut size_t, +) -> c_int { + eprintln!("sd_journal_get_data is not implemented for this target"); + -1 +} + +/// Get all the field data of the systemd journal entry. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_enumerate_data( + _j: *mut sd_journal, + _data: *mut *mut u8, + _l: *mut size_t, +) -> c_int { + eprintln!("sd_journal_enumerate_data is not implemented for this target"); + -1 +} + +/// Get the file descriptor of the systemd journal. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_get_fd(_j: *mut sd_journal) -> c_int { + eprintln!("sd_journal_get_fd is not implemented for this target"); + -1 +} + +/// Signal that we've processed the journal entry. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_process(_j: *mut sd_journal) -> c_int { + eprintln!("sd_journal_process is not implemented for this target"); + -1 +} + +/// Get timestamp for the systemd journal entry. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_get_realtime_usec(_j: *mut sd_journal, _ret: *mut u64) -> c_int { + eprintln!("sd_journal_get_realtime_usec is not implemented for this target"); + -1 +} + +/// Get the cursor for the systemd journal. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_get_cursor(_j: *mut sd_journal, _cursor: *mut *const c_char) -> c_int { + eprintln!("sd_journal_get_cursor is not implemented for this target"); + -1 +} + +/// Seek to the cursor in the systemd journal. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_seek_cursor(_j: *mut sd_journal, _cursor: *const c_char) -> c_int { + eprintln!("sd_journal_seek_cursor is not implemented for this target"); + -1 +} + +/// Seek to the cursor in the systemd journal. +#[allow(clippy::missing_safety_doc)] +pub unsafe fn sd_journal_add_match( + _j: *mut sd_journal, + _data: *const c_char, + _size: size_t, +) -> c_int { + eprintln!("sd_journal_add_match is not implemented for this target"); + -1 +} diff --git a/memfaultd.init b/memfaultd.init new file mode 100644 index 0000000..63e9ca8 --- /dev/null +++ b/memfaultd.init @@ -0,0 +1,150 @@ +#! /bin/sh +### BEGIN INIT INFO +# Provides: memfaultd +# Required-Start: $local_fs +# Should-Start: +# Required-Stop: $local_fs +# Should-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: manages the memfaultd daemon +### END INIT INFO + +# PATH should only include /usr/* if it runs after the mountnfs.sh script +PATH=/sbin:/usr/sbin:/bin:/usr/bin + +DESC="memfaultd" +NAME="memfaultd" +DAEMON=/usr/bin/memfaultd +PIDFILE=/var/run/$NAME.pid +ARGS="-Z -q" + +. /etc/init.d/functions || exit 1 + +# Exit if the package is not installed +[ -x "$DAEMON" ] || exit 0 + +# Read configuration variable file if it is present +[ -r /etc/default/$NAME ] && . /etc/default/$NAME + +# +# Function that starts the daemon/service +# +do_start() { + local status pid + + status=0 + pid=`pidofproc $NAME` || status=$? + case $status in + 0) + echo "$DESC already running ($pid)." + exit 1 + ;; + *) + echo "Starting $DESC ..." + cd /home/root + + exec $DAEMON $ARGS + exit 0 + ;; + esac +} + +# +# Function that stops the daemon/service +# +do_stop() { + local pid status + + status=0 + pid=`pidofproc $NAME` || status=$? + case $status in + 0) + # Exit when fail to stop, the kill would complain when fail + kill -s SIGTERM $pid >/dev/null && rm -f $PIDFILE && \ + echo "Stopped $DESC ($pid)." || exit $? + + # Wait up to 10 seconds for the process to exit + for i in `seq 10`; do + if ! pidofproc $NAME > /dev/null; then + break + fi + sleep 1 + done + ;; + *) + echo "$DESC is not running; none killed." >&2 + ;; + esac + + return 0 +} + +# +# Function that sends a SIGHUP to the daemon/service +# +do_reload() { + local pid status + + status=0 + pid=`pidofproc $NAME` || status=$? + case $status in + 0) + echo "Reloading $DESC ..." + kill -s SIGHUP $pid || exit $? + ;; + *) + echo "$DESC is not running; none reloaded." >&2 + ;; + esac + exit $status +} + + +# +# Function that shows the daemon/service status +# +status_of_proc () { + local pid status + + status=0 + # pidof output null when no program is running, so no "2>/dev/null". + pid=`pidofproc $NAME` || status=$? + case $status in + 0) + echo "$DESC is running ($pid)." + exit 0 + ;; + *) + echo "$DESC is not running." >&2 + exit $status + ;; + esac +} + +case "$1" in +start) + do_start + ;; +stop) + do_stop || exit $? + ;; +status) + status_of_proc + ;; +restart) + # Always start the service regardless the status of do_stop + do_stop + do_start + ;; +try-restart|force-reload) + do_stop && do_start + ;; +reload) + do_reload + ;; +*) + echo "Usage: $0 {start|stop|status|restart|try-restart|force-reload}" >&2 + exit 3 + ;; +esac diff --git a/memfaultd.service b/memfaultd.service new file mode 100644 index 0000000..7c13c32 --- /dev/null +++ b/memfaultd.service @@ -0,0 +1,15 @@ +[Unit] +Description=memfaultd daemon +After=local-fs.target network.target dbus.service +Before=swupdate.service collectd.service + +[Service] +Type=forking +PIDFile=/run/memfaultd.pid +ExecStart=/usr/bin/memfaultd --daemonize +# Wait for the PID file to be populated before returning +ExecStartPost=/bin/sh -c "while [ $(cat /run/memfaultd.pid 2>/dev/null | wc -c) -eq 0 ]; do sleep 0.1; done" +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/memfaultd/Cargo.toml b/memfaultd/Cargo.toml new file mode 100644 index 0000000..474de6e --- /dev/null +++ b/memfaultd/Cargo.toml @@ -0,0 +1,104 @@ +[package] +name = "memfaultd" +version = "0.1.0" +edition = "2021" +autobins = false +rust-version = "1.65" + +[[bin]] +name = "memfaultd" +path = "src/bin/memfaultd.rs" + +[[bin]] +name= "memfaultctl" +path= "src/bin/memfaultctl.rs" + +[[bin]] +name= "memfault-core-handler" +path= "src/bin/memfault-core-handler.rs" + +[[bin]] +name= "mfw" +path= "src/bin/mfw.rs" + +[dependencies] +memfaultc-sys = { path= "../memfaultc-sys" } +argh = "0.1.10" +cfg-if = "1.0.0" +chrono = { version = "0.4.23", features = ["serde"]} +ciborium = { version = "0.2.1", optional = true} +eyre = "0.6.8" +goblin = { version = "0.7", optional = true, default-features = false, features = ["elf32", "elf64", "std"] } +libc = "0.2.138" +log = "0.4.17" +prctl = { version = "1.0.0", optional = true} +psm = { version = "0.1.21", optional = true } +reqwest = { version = "0.11", default-features = false, features = ["blocking", "json" ] } +rmp-serde = { version = "1.1.1", optional = true } +rmpv = { version = "1.0.0", optional = true } +scroll = { version = "0.11", optional = true } +serde = { version = "1.0.150", features = ["derive"] } +serde_bytes = "0.11.8" +serde_json = "1.0.89" +serde_repr = "0.1" +shuteye = "0.3.3" +signal-hook = "0.3.14" +stderrlog = "0.5.4" +strum = { version = "0.24", features = ["derive"] } +strum_macros = "0.24" +tempfile = "3.3.0" +thiserror = "1.0.38" +threadpool = { version = "1.8.1"} +urlencoding = "2.1.2" +uuid = { version = "1.3.0", features = ["v4", "serde"] } +once_cell = "1.17.0" +fs_extra = "1.3.0" +flate2 = { version = "1.0.28", default-features = false, features = ["zlib"] } +take_mut = "0.2.2" +itertools = "0.10.5" +governor = { version = "0.5.1", default-features = false, features = ["std"], optional = true} +nix = { version = "0.26.2", default-features = false, features = ["process", "signal", "poll", "fs"]} +kernlog = { version = "0.3.1", optional = true } +tiny_http = {version = "0.12.0" } +crc = "3.0.1" +crc-catalog = "2.2.0" +regex= { version = "1.10.2", optional = true} +nom = "7.1.3" +sealed_test = "1.1.0" + +[target.'cfg(target_os = "linux")'.dependencies] +procfs = { version = "0.15.1", optional = true } + +[dev-dependencies] +tempfile = "3.3.0" +mockall = "0.11.3" +rstest = "0.16.0" +goblin = { version = "0.7", default-features = false, features = ["elf32", "elf64", "std", "endian_fd"] } +insta = {version= "1.26.0", features= ["json", "redactions"]} +zip = { version = "0.6.3", default-features = false, features = ["deflate"] } +hex = "0.4.3" +rand = "0.8.5" +nom = "7.1.3" + +[features] +default = ["coredump", "collectd", "logging", "log-to-metrics", "systemd", "rust-tls" ] +coredump = [ + "memfaultc-sys/coredump", + "dep:prctl", + "dep:procfs", + "dep:psm", + "dep:kernlog", + "dep:ciborium", + "dep:goblin", + "dep:scroll" +] +collectd = [] +swupdate = ["memfaultc-sys/swupdate"] +logging = ["dep:governor", "dep:rmp-serde", "dep:rmpv"] +systemd = ["memfaultc-sys/systemd"] +rust-tls = ["reqwest/rustls-tls"] +openssl-tls = ["reqwest/native-tls"] +openssl-vendored-tls = ["reqwest/native-tls-vendored"] +log-to-metrics = ["dep:regex"] +experimental = ["mfw"] +mfw = [] diff --git a/memfaultd/DEVELOPMENT.md b/memfaultd/DEVELOPMENT.md new file mode 100644 index 0000000..0019195 --- /dev/null +++ b/memfaultd/DEVELOPMENT.md @@ -0,0 +1,60 @@ +# Development + +`memfaultd` build is controlled by Cargo. The `Cargo.toml` and `build.rs` +control the rust build process and compile the few C files. + +## Building outside Yocto + +### Dependencies + +#### Debian/Ubuntu + +```sh +apt install libsystemd-dev libconfig-dev +``` + +#### macOS + +```sh +brew install libconfig +``` + +(note: `libsystemd` is not available on macOS and the build system will not try +to link it) + +### Building + +```sh +cargo build +``` + +## Building with Yocto + +Use the `docker/run.sh` script to run a docker container with all the required +dependencies. Use the alias `b` to build the image. + +## Running tests + +### Unit tests + +Do this after running a build, inside the (cmake) build directory: + +```sh +cargo test +``` + +### Updating snapshots + +Install `insta` if necessary, and run the command: + +```bash +cargo install cargo-insta +cargo insta review +``` + +## IDE integration + +### Using VSCode to work on memfaultd + +VSCode rust plugin will not find the `Cargo.toml` file unless you open the +`meta-memfault/recipes-memfault/memfaultd/files/memfaultd/` directly. diff --git a/memfaultd/build.rs b/memfaultd/build.rs new file mode 100644 index 0000000..9f6ef59 --- /dev/null +++ b/memfaultd/build.rs @@ -0,0 +1,66 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::collections::HashMap; +use std::env; +use std::fs::write; +use std::path::{Path, PathBuf}; + +/// Generates $OUT_DIR/build_info.rs, based on the values in the VERSION file (generated by sdk_release.py). +fn generate_build_info_rs() { + let mut version_file_path = PathBuf::from(&env::var("CARGO_MANIFEST_DIR").unwrap()); + version_file_path.push("../VERSION"); + println!("cargo:rerun-if-changed={}", version_file_path.display()); + + let version_file_content = + std::fs::read_to_string(version_file_path).unwrap_or_else(|_| String::new()); + + let key_values_from_version_file: HashMap<&str, &str> = version_file_content + .lines() + .filter_map(|l| l.split_once(':')) + .map(|(k, v)| (k.trim(), v.trim())) + .collect(); + + struct VersionVarInfo { + key: &'static str, + var: &'static str, + default: &'static str, + } + + let keys_and_vars_defaults = [ + VersionVarInfo { + key: "VERSION", + var: "VERSION", + default: "dev", + }, + VersionVarInfo { + key: "GIT COMMIT", + var: "GIT_COMMIT", + default: "unknown", + }, + VersionVarInfo { + key: "BUILD ID", + var: "BUILD_ID", + default: "unknown", + }, + ]; + + let build_info_rs_src = keys_and_vars_defaults + .iter() + .fold(String::new(), |mut acc, info| { + let value = key_values_from_version_file + .get(info.key) + .unwrap_or(&info.default); + acc.push_str(&format!("pub const {}: &str = \"{}\";\n", info.var, value)); + acc + }); + + let out_dir = env::var_os("OUT_DIR").unwrap(); + let dest_path = Path::new(&out_dir).join("build_info.rs"); + write(dest_path, build_info_rs_src).unwrap(); +} + +fn main() { + generate_build_info_rs(); + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/memfaultd/builtin.conf b/memfaultd/builtin.conf new file mode 100644 index 0000000..40917c9 --- /dev/null +++ b/memfaultd/builtin.conf @@ -0,0 +1,64 @@ +/* +See https://mflt.io/memfaultd.conf for information on the available +parameters. +*/ +{ + "persist_dir": "/media/memfault", + "tmp_dir": null, + "tmp_dir_min_headroom_kib": 10240, + "tmp_dir_min_inodes": 100, + "tmp_dir_max_usage_kib": 102400, + "upload_interval_seconds": 3600, + "heartbeat_interval_seconds": 3600, + "enable_data_collection": false, + "enable_dev_mode": false, + "project_key": "", + "base_url": "https://device.memfault.com", + "swupdate": { + "input_file": "/etc/swupdate.cfg", + "output_file": "/tmp/swupdate.cfg" + }, + "reboot": { + "last_reboot_reason_file": "/media/last_reboot_reason" + }, + "coredump": { + "coredump_max_size_kib": 96000, + "compression": "gzip", + "rate_limit_count": 5, + "rate_limit_duration_seconds": 3600, + "capture_strategy": { + "type": "threads", + "max_thread_size_kib": 32 + }, + "log_lines": 100 + }, + "http_server": { + "bind_address": "127.0.0.1:8787" + }, + "fluent-bit": { + "extra_fluentd_attributes": [], + "bind_address": "127.0.0.1:5170", + "max_buffered_lines": 1000, + "max_connections": 4 + }, + "logs": { + "compression_level": 1, + "max_lines_per_minute": 500, + "rotate_size_kib": 10240, + "rotate_after_seconds": 3600, + "storage": "persist", + "source": "fluent-bit" + }, + "mar": { + "mar_file_max_size_kib": 10240, + "mar_entry_max_age_seconds": 604800 + }, + "battery_monitor": null, + "metrics": { + "enable_daily_heartbeats": false, + "system_metric_collection": { + "poll_interval_seconds": 10, + "enable": false + } + } +} diff --git a/memfaultd/memfaultd.conf b/memfaultd/memfaultd.conf new file mode 100644 index 0000000..f72e2b9 --- /dev/null +++ b/memfaultd/memfaultd.conf @@ -0,0 +1,13 @@ +/* +See builtin.conf (in the source files) for configuration options. +At minimum, your override of this file should provide these keys: +{ + "software_version": "", + "software_type": "", + "project_key": "", + "persist_dir": "" +} +*/ + +{ +} diff --git a/memfaultd/src/bin/memfault-core-handler.rs b/memfaultd/src/bin/memfault-core-handler.rs new file mode 100644 index 0000000..83cdded --- /dev/null +++ b/memfaultd/src/bin/memfault-core-handler.rs @@ -0,0 +1,11 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use memfaultd::cli; + +/// memfault-core-handlers is an alias to the main function in cli.rs +/// +/// For further details, see comments on memfaultd.rs +fn main() { + cli::main() +} diff --git a/memfaultd/src/bin/memfaultctl.rs b/memfaultd/src/bin/memfaultctl.rs new file mode 100644 index 0000000..7d62156 --- /dev/null +++ b/memfaultd/src/bin/memfaultctl.rs @@ -0,0 +1,11 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use memfaultd::cli; + +/// memfaultctl is an alias to the main function in cli.rs +/// +/// For further details, see comments on memfaultd.rs +fn main() { + cli::main() +} diff --git a/memfaultd/src/bin/memfaultd.rs b/memfaultd/src/bin/memfaultd.rs new file mode 100644 index 0000000..ab835cf --- /dev/null +++ b/memfaultd/src/bin/memfaultd.rs @@ -0,0 +1,20 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use memfaultd::cli; + +/// memfaultd is an alias to the main function in cli.rs +/// +/// In the target machine, memfaultd is the only binary that remains. memfaultctl and +/// memfault-core-handler are symlinked by build scripts. In the case of Yocto, this is +/// meta-memfault/recipes-memfault/memfaultd/memfaultd.bb +/// +/// The binary setup in this crate could opt for a single bin/memfaultd.rs entrypoint, however +/// setting up three different binaries makes development easier because it mimics what build +/// systems do in target devices. +/// +/// Alternative solutions are possible, for example adding a post-build script that does the +/// symlinking manually in the build directory. +fn main() { + cli::main() +} diff --git a/memfaultd/src/bin/mfw.rs b/memfaultd/src/bin/mfw.rs new file mode 100644 index 0000000..46868bb --- /dev/null +++ b/memfaultd/src/bin/mfw.rs @@ -0,0 +1,11 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use memfaultd::cli; + +/// mfw is an alias to the main function in cli.rs +/// +/// For further details, see comments on memfaultd.rs +fn main() { + cli::main() +} diff --git a/memfaultd/src/cli/cargs.rs b/memfaultd/src/cli/cargs.rs new file mode 100644 index 0000000..8e9e5dd --- /dev/null +++ b/memfaultd/src/cli/cargs.rs @@ -0,0 +1,46 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use libc::c_char; +use libc::c_int; +use std::ffi::CString; +use std::path::Path; + +pub struct CArgs { + argv: Vec, + argv_ptr: Vec<*const libc::c_char>, +} + +impl CArgs { + pub fn new(args: impl IntoIterator) -> Self { + let argv: Vec<_> = args + .into_iter() + .map(|arg| CString::new(arg.as_str()).unwrap()) + .collect(); + let argv_ptr: Vec<_> = argv + .iter() + .map(|arg| arg.as_ptr()) + .chain(std::iter::once(std::ptr::null())) + .collect(); + Self { argv, argv_ptr } + } + + /// Returns the number of arguments, ie the C language's `argc`. + pub fn argc(&self) -> c_int { + self.argv.len() as c_int + } + + /// Returns the C language's `argv` (`*const *const c_char`). + pub fn argv(&self) -> *const *const c_char { + self.argv_ptr.as_ptr() + } + + /// Returns the name of the command invoked by the user - removing any path information. + pub fn name(&self) -> &str { + Path::new(self.argv[0].to_str().unwrap()) + .file_name() + .unwrap() + .to_str() + .unwrap() + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/arch.rs b/memfaultd/src/cli/memfault_core_handler/arch.rs new file mode 100644 index 0000000..0102688 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/arch.rs @@ -0,0 +1,72 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::cli::memfault_core_handler::elf; +use cfg_if::cfg_if; + +cfg_if! { + if #[cfg(target_arch = "aarch64")] { + pub use libc::user_regs_struct as ElfGRegSet; + pub fn get_stack_pointer(regs: &ElfGRegSet) -> usize { + regs.sp as usize + } + pub use elf::header::EM_AARCH64 as ELF_TARGET_MACHINE; + pub use elf::header::ELFCLASS64 as ELF_TARGET_CLASS; + } else if #[cfg(target_arch = "x86_64")] { + pub use libc::user_regs_struct as ElfGRegSet; + pub fn get_stack_pointer(regs: &ElfGRegSet) -> usize { + regs.rsp as usize + } + pub use elf::header::EM_X86_64 as ELF_TARGET_MACHINE; + pub use elf::header::ELFCLASS64 as ELF_TARGET_CLASS; + } else if #[cfg(target_arch = "arm")] { + pub use libc::user_regs as ElfGRegSet; + pub fn get_stack_pointer(regs: &ElfGRegSet) -> usize { + regs.arm_sp as usize + } + pub use elf::header::EM_ARM as ELF_TARGET_MACHINE; + pub use elf::header::ELFCLASS32 as ELF_TARGET_CLASS; + } else if #[cfg(target_arch = "x86")] { + pub use libc::user_regs_struct as ElfGRegSet; + pub fn get_stack_pointer(regs: &ElfGRegSet) -> usize { + regs.esp as usize + } + pub use elf::header::EM_386 as ELF_TARGET_MACHINE; + pub use elf::header::ELFCLASS32 as ELF_TARGET_CLASS; + } + else { + // Provide dummy symbols for unsupported architectures. This is preferable to + // a compile time error, as we want to be able to compile memfaultd for all + // architectures, but we don't need register access for all of them. Currently + // these registers are only used to filter out stack memory from coredumps. + pub struct ElfGRegSet; + pub fn get_stack_pointer(_regs: &ElfGRegSet) -> usize { + 0 + } + } +} + +// Function definitions for coredump thread filter support. If the target architecture +// is not supported, these functions will always return false. +cfg_if! { + if #[cfg(any( + target_arch = "aarch64", + target_arch = "arm", + target_arch = "x86", + target_arch = "x86_64" + ))] { + pub const fn coredump_thread_filter_supported() -> bool { + true + } + } else { + pub const fn coredump_thread_filter_supported() -> bool { + false + } + } +} + +#[cfg(target_endian = "little")] +pub use elf::header::ELFDATA2LSB as ELF_TARGET_ENDIANNESS; + +#[cfg(target_endian = "big")] +pub use elf::header::ELFDATA2MSB as ELF_TARGET_ENDIANNESS; diff --git a/memfaultd/src/cli/memfault_core_handler/auxv.rs b/memfaultd/src/cli/memfault_core_handler/auxv.rs new file mode 100644 index 0000000..6b24753 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/auxv.rs @@ -0,0 +1,114 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use scroll::Pread; +use std::fmt::Debug; + +#[cfg(target_pointer_width = "64")] +pub type AuxvUint = u64; + +#[cfg(target_pointer_width = "32")] +pub type AuxvUint = u32; + +#[derive(Eq, PartialEq)] +pub struct Auxv<'a> { + data: &'a [u8], +} + +#[derive(Eq, PartialEq, Debug)] +pub struct AuxvEntry { + pub key: AuxvUint, + pub value: AuxvUint, +} + +impl<'a> Auxv<'a> { + pub fn new(data: &'a [u8]) -> Self { + Self { data } + } + + pub fn iter(&'a self) -> AuxvIterator<'a> { + AuxvIterator::new(self) + } + + pub fn find_value(&self, key: AuxvUint) -> Option { + self.iter() + .find(|a| a.key == key as AuxvUint) + .map(|a| a.value) + } +} + +impl<'a> Debug for Auxv<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.iter().collect::>().fmt(f) + } +} + +pub struct AuxvIterator<'a> { + auxv: &'a Auxv<'a>, + offset: usize, +} + +impl<'a> AuxvIterator<'a> { + fn new(auxv: &'a Auxv<'a>) -> Self { + Self { auxv, offset: 0 } + } +} + +impl<'a> Iterator for AuxvIterator<'a> { + type Item = AuxvEntry; + + fn next(&mut self) -> Option { + let mut read = || self.auxv.data.gread::(&mut self.offset); + match (read(), read()) { + (Ok(key), Ok(value)) => Some(AuxvEntry { key, value }), + _ => None, + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use rstest::rstest; + use scroll::IOwrite; + use std::io::Cursor; + + #[rstest] + // Empty auxv: + #[case( + vec![], + vec![] + )] + // Happy path: + #[case( + vec![1, 2, 3, 4], + vec![AuxvEntry { key: 1, value: 2 }, AuxvEntry { key: 3, value: 4 }] + )] + // Partial entry at the end is ignored: + #[case( + vec![1, 2, 3], + vec![AuxvEntry { key: 1, value: 2 }] + )] + fn test_auxv_iterator(#[case] values: Vec, #[case] expected: Vec) { + let buffer = make_fixture(values); + let auxv = Auxv::new(buffer.as_slice()); + assert_eq!(auxv.iter().collect::>(), expected); + } + + #[test] + fn test_auxv_find_value() { + let buffer = make_fixture(vec![1, 2]); + let auxv = Auxv::new(buffer.as_slice()); + assert!(auxv.find_value(1).is_some()); + assert!(auxv.find_value(9).is_none()); + } + + fn make_fixture(values: Vec) -> Vec { + let mut cursor = Cursor::new(vec![]); + for value in values { + cursor.iowrite::(value).unwrap(); + } + + cursor.into_inner() + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/core_elf_memfault_note.rs b/memfaultd/src/cli/memfault_core_handler/core_elf_memfault_note.rs new file mode 100644 index 0000000..5f1eb0c --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/core_elf_memfault_note.rs @@ -0,0 +1,221 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Utilities and data types for writing Memfault-specific ELF notes to a core dump file. +//! +//! Currently we write two notes: +//! +//! 1. A note containing metadata about the core dump. This note is written by the +//! `CoreHandler` whenever it receives a core dump. It contains information about the device, +//! that will be used to associate the core dump with a device in the Memfault cloud. +//! 2. A note containing debug data about the core dump. Currently this note only contains +//! logs written during the coredump capture process. These logs are used by Memfault to debug +//! issues with coredump capture. +use std::time::SystemTime; + +use crate::config::CoredumpCaptureStrategy; +use crate::{build_info::VERSION, mar::LinuxLogsFormat}; + +use ciborium::{cbor, into_writer}; +use eyre::Result; +use serde::Serialize; + +use super::core_elf_note::build_elf_note; + +const NOTE_NAME: &str = "Memfault\0"; +const METADATA_NOTE_TYPE: u32 = 0x4154454d; +const DEBUG_DATA_NOTE_TYPE: u32 = 0x4154454e; +const MEMFAULT_CORE_ELF_METADATA_SCHEMA_VERSION_V1: u32 = 1; +const MEMFAULT_CORE_ELF_DEBUG_DATA_SCHEMA_VERSION_V1: u32 = 1; + +/// Map of keys used in the Memfault core ELF metadata note. +/// +/// Integer keys are used here instead of strings to reduce the size of the note. +enum MemfaultCoreElfMetadataKey { + SchemaVersion = 1, + LinuxSdkVersion = 2, + CapturedTime = 3, + DeviceSerial = 4, + HardwareVersion = 5, + SoftwareType = 6, + SoftwareVersion = 7, + CmdLine = 8, + CaptureStrategy = 9, + ApplicationLogs = 10, +} + +#[derive(Debug, Serialize)] +pub struct MemfaultMetadataLogs { + logs: Vec, + format: LinuxLogsFormat, +} + +impl MemfaultMetadataLogs { + pub fn new(logs: Vec, format: LinuxLogsFormat) -> Self { + Self { logs, format } + } +} + +/// Metadata about a core dump. +#[derive(Debug)] +pub struct CoredumpMetadata { + pub device_id: String, + pub hardware_version: String, + pub software_type: String, + pub software_version: String, + pub sdk_version: String, + pub captured_time_epoch_s: u64, + pub cmd_line: String, + pub capture_strategy: CoredumpCaptureStrategy, + pub app_logs: Option, +} + +impl CoredumpMetadata { + pub fn new(config: &crate::config::Config, cmd_line: String) -> Self { + Self { + device_id: config.device_info.device_id.clone(), + hardware_version: config.device_info.hardware_version.clone(), + software_type: config.software_type().to_string(), + software_version: config.software_version().to_string(), + sdk_version: VERSION.to_string(), + captured_time_epoch_s: SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(), + cmd_line, + capture_strategy: config.config_file.coredump.capture_strategy, + app_logs: None, + } + } +} + +/// Serialize a `CoredumpMetadata` struct as a CBOR map. +/// +/// This CBOR map uses integer keys instead of strings to reduce the size of the note. +pub fn serialize_metadata_as_map(metadata: &CoredumpMetadata) -> Result> { + let cbor_val = cbor!({ + MemfaultCoreElfMetadataKey::SchemaVersion as u32 => MEMFAULT_CORE_ELF_METADATA_SCHEMA_VERSION_V1, + MemfaultCoreElfMetadataKey::LinuxSdkVersion as u32 => metadata.sdk_version, + MemfaultCoreElfMetadataKey::CapturedTime as u32 => metadata.captured_time_epoch_s, + MemfaultCoreElfMetadataKey::DeviceSerial as u32 => metadata.device_id, + MemfaultCoreElfMetadataKey::HardwareVersion as u32 => metadata.hardware_version, + MemfaultCoreElfMetadataKey::SoftwareType as u32 => metadata.software_type, + MemfaultCoreElfMetadataKey::SoftwareVersion as u32 => metadata.software_version, + MemfaultCoreElfMetadataKey::CmdLine as u32 => metadata.cmd_line, + MemfaultCoreElfMetadataKey::CaptureStrategy as u32 => metadata.capture_strategy, + MemfaultCoreElfMetadataKey::ApplicationLogs as u32 => metadata.app_logs, + })?; + + let mut buffer = Vec::new(); + into_writer(&cbor_val, &mut buffer)?; + + Ok(buffer) +} + +/// Write a core ELF note containing metadata about a core dump. +/// +/// This note is written by the `CoreHandler` whenever it receives a core dump. It contains +/// information about the device, that will be used to associate the core dump with a device in the +/// Memfault cloud. +pub fn write_memfault_metadata_note(metadata: &CoredumpMetadata) -> Result> { + let description_buffer = serialize_metadata_as_map(metadata)?; + + build_elf_note(NOTE_NAME, &description_buffer, METADATA_NOTE_TYPE) +} + +/// A note containing a list of errors that occurred during coredump capture. +/// +/// This note is written by the `CoreHandlerLogWrapper` when it receives an error or warning log. +/// These logs will help us debug issues with coredump capture. +#[derive(Debug, Serialize)] +pub struct CoredumpDebugData { + pub schema_version: u32, + pub capture_logs: Vec, +} + +/// Write a core ELF note containing debug data about the coredump capture process. +/// +/// See `CoredumpDebugData` for more information. +pub fn write_memfault_debug_data_note(errors: Vec) -> Result> { + let coredump_capture_logs = CoredumpDebugData { + schema_version: MEMFAULT_CORE_ELF_DEBUG_DATA_SCHEMA_VERSION_V1, + capture_logs: errors, + }; + + let mut buffer = Vec::new(); + into_writer(&coredump_capture_logs, &mut buffer)?; + + build_elf_note(NOTE_NAME, &buffer, DEBUG_DATA_NOTE_TYPE) +} + +#[cfg(test)] +mod test { + use ciborium::{from_reader, Value}; + use rstest::rstest; + + use crate::test_utils::set_snapshot_suffix; + + use super::*; + + #[rstest] + #[case( + "kernel_selection", + CoredumpCaptureStrategy::KernelSelection, + 91, + false + )] + #[case("threads", CoredumpCaptureStrategy::Threads{ max_thread_size: 32 * 1024}, 104, false)] + #[case("app_logs", CoredumpCaptureStrategy::KernelSelection, 160, true)] + fn test_serialize_metadata_as_map( + #[case] test_name: &str, + #[case] capture_strategy: CoredumpCaptureStrategy, + #[case] expected_size: usize, + #[case] has_app_logs: bool, + ) { + let app_logs = has_app_logs.then(|| MemfaultMetadataLogs { + logs: vec![ + "Error 1".to_string(), + "Error 2".to_string(), + "Error 3".to_string(), + ], + format: LinuxLogsFormat::default(), + }); + let metadata = CoredumpMetadata { + device_id: "12345678".to_string(), + hardware_version: "evt".to_string(), + software_type: "main".to_string(), + software_version: "1.0.0".to_string(), + sdk_version: "SDK_VERSION".to_string(), + captured_time_epoch_s: 1234, + cmd_line: "binary -a -b -c".to_string(), + capture_strategy, + app_logs, + }; + + let map = serialize_metadata_as_map(&metadata).unwrap(); + let deser_map: Value = from_reader(map.as_slice()).unwrap(); + + set_snapshot_suffix!("{}", test_name); + insta::assert_debug_snapshot!(deser_map); + assert_eq!(map.len(), expected_size); + } + + #[test] + fn serialize_debug_data() { + let capture_logs = CoredumpDebugData { + schema_version: MEMFAULT_CORE_ELF_DEBUG_DATA_SCHEMA_VERSION_V1, + capture_logs: vec![ + "Error 1".to_string(), + "Error 2".to_string(), + "Error 3".to_string(), + ], + }; + + let mut capture_logs_buffer = Vec::new(); + into_writer(&capture_logs, &mut capture_logs_buffer).unwrap(); + + let deser_capture_logs: Value = from_reader(capture_logs_buffer.as_slice()).unwrap(); + + insta::assert_debug_snapshot!(deser_capture_logs); + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/core_elf_note.rs b/memfaultd/src/cli/memfault_core_handler/core_elf_note.rs new file mode 100644 index 0000000..2e5b10f --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/core_elf_note.rs @@ -0,0 +1,638 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::cmp::min; +use std::ffi::{CStr, OsStr}; +use std::mem::size_of; +use std::os::unix::ffi::OsStrExt; +use std::path::Path; + +use eyre::{eyre, Error, Result}; + +// The Nhdr64 struct is incorrect in goblin. We need to open a PR with them, +// but for now going to use the 32 bit version. +use crate::cli::memfault_core_handler::arch::ElfGRegSet; +use crate::cli::memfault_core_handler::auxv::Auxv; +use crate::cli::memfault_core_handler::ElfPtrSize; +use crate::util::math::align_up; +use goblin::elf::note::Nhdr32 as Nhdr; +use goblin::elf::note::{NT_FILE, NT_GNU_BUILD_ID, NT_PRSTATUS}; +use log::{error, warn}; +use scroll::{Pread, Pwrite}; + +/// Builds an ELF note from a given name, description, and note type. +pub fn build_elf_note(name: &str, description: &[u8], note_type: u32) -> Result> { + let name_bytes = name.as_bytes(); + let mut name_size = name_bytes.len(); + // NOTE: As per the spec, the terminating NUL byte should be included in the namesz, but 0 is + // used for an empty string: + if name_size > 0 { + name_size += 1; + } + let note_header = Nhdr { + n_namesz: name_size.try_into()?, + n_descsz: description.len().try_into()?, + n_type: note_type, + }; + let aligned_name_size = align_up(name_size, 4); + + let header_size = size_of::(); + let mut note_buffer = + vec![0u8; header_size + aligned_name_size + align_up(description.len(), 4)]; + note_buffer.pwrite(note_header, 0)?; + note_buffer[header_size..(header_size + name_bytes.len())].copy_from_slice(name_bytes); + + let desc_offset = header_size + aligned_name_size; + note_buffer[desc_offset..(desc_offset + description.len())].copy_from_slice(description); + + Ok(note_buffer) +} + +#[derive(Debug, PartialEq, Eq)] +/// Parsed ELF note. +/// +/// Contains the deserialized ELF note description for a given note type. +/// Unknown can be used in case the note type is not supported or if parsing failed. +pub enum ElfNote<'a> { + /// Parsed CORE::NT_PRSTATUS note. + ProcessStatus(&'a ProcessStatusNote), + // Parsed CORE::NT_FILE note + File(FileNote<'a>), + // Parsed GNU::NT_GNU_BUILD_ID note. + GnuBuildId(&'a [u8]), + Auxv(Auxv<'a>), + Unknown { + name: &'a [u8], + note_type: u32, + description: &'a [u8], + }, +} + +const NOTE_NAME_CORE: &[u8] = b"CORE"; +const NOTE_NAME_GNU: &[u8] = b"GNU"; + +const NT_AUXV: u32 = 6; + +impl<'a> ElfNote<'a> { + fn try_parse(name: &'a [u8], note_type: u32, description: &'a [u8]) -> Result> { + match (name, note_type) { + (NOTE_NAME_CORE, NT_PRSTATUS) => Ok(Some(Self::ProcessStatus(description.try_into()?))), + (NOTE_NAME_CORE, NT_FILE) => Ok(Some(Self::File(description.try_into()?))), + (NOTE_NAME_CORE, NT_AUXV) => Ok(Some(Self::Auxv(Auxv::new(description)))), + (NOTE_NAME_GNU, NT_GNU_BUILD_ID) => Ok(Some(Self::GnuBuildId(description))), + _ => Ok(None), + } + } + + fn parse(name: &'a [u8], note_type: u32, description: &'a [u8]) -> Self { + match Self::try_parse(name, note_type, description) { + Ok(Some(note)) => note, + r => { + if let Err(e) = r { + warn!( + "Failed to parse ELF note: name={} type={}: {}", + String::from_utf8_lossy(name), + note_type, + e + ); + } + Self::Unknown { + name, + note_type, + description, + } + } + } + } +} + +/// Iterator over ELF notes in a buffer. +/// +/// Only the current note is deserialized at a time. +/// This prevents us from having to make multiple copies of all notes. +pub struct ElfNoteIterator<'a> { + note_buffer: &'a [u8], + offset: usize, +} + +impl<'a> ElfNoteIterator<'a> { + fn new(note_buffer: &'a [u8]) -> Self { + Self { + note_buffer, + offset: 0, + } + } + + /// Try to create an ELF note from the given buffer at the given offset. + /// + /// If the note type is not supported, return `None`. + fn try_next_note(offset: &mut usize, note_buffer: &'a [u8]) -> Result> { + let note_header = note_buffer.gread::(offset)?; + let name_size = note_header.n_namesz as usize; + let aligned_name_size = align_up(name_size, 4); + let desc_size = note_header.n_descsz as usize; + let aligned_desc_size = align_up(desc_size, 4); + + if *offset + aligned_name_size + aligned_desc_size > note_buffer.len() { + return Err(eyre!("Note buffer shorter than expected")); + } + + let name = match name_size { + // NOTE: As per the spec, the terminating NUL byte is included in the namesz, + // but 0 is used for an empty string: + 0 => &[], + _ => ¬e_buffer[*offset..(*offset + name_size - 1)], + }; + let desc_offset = *offset + aligned_name_size; + let desc = ¬e_buffer[desc_offset..(desc_offset + desc_size)]; + + *offset += aligned_name_size + aligned_desc_size; + Ok(ElfNote::parse(name, note_header.n_type, desc)) + } +} + +impl<'a> Iterator for ElfNoteIterator<'a> { + type Item = ElfNote<'a>; + + fn next(&mut self) -> Option { + if self.offset >= self.note_buffer.len() { + None + } else { + match Self::try_next_note(&mut self.offset, self.note_buffer) { + Ok(note) => Some(note), + Err(e) => { + error!("{}", e); + None + } + } + } + } +} + +/// Helper function to iterate over ELF notes in a buffer. +pub fn iterate_elf_notes(note_buffer: &[u8]) -> ElfNoteIterator { + ElfNoteIterator::new(note_buffer) +} + +#[derive(Debug, PartialEq, Eq)] +#[repr(C)] +/// Time value for a process. +pub struct ProcessTimeVal { + pub tv_sec: usize, + pub tv_usec: usize, +} + +#[derive(Debug, PartialEq, Eq)] +#[repr(C)] +/// Deserialized process status note. +/// +/// This is the deserialized form of the NT_PRSTATUS note type. +/// Note that this structure is architecture-specific. +pub struct ProcessStatusNote { + pub si_signo: u32, + pub si_code: u32, + pub si_errno: u32, + pub pr_cursig: u16, + pub pad0: u16, + pub pr_sigpend: usize, + pub pr_sighold: usize, + pub pr_pid: u32, + pub pr_ppid: u32, + pub pr_pgrp: u32, + pub pr_sid: u32, + pub pr_utime: ProcessTimeVal, + pub pr_stime: ProcessTimeVal, + pub pr_cutime: ProcessTimeVal, + pub pr_cstime: ProcessTimeVal, + pub pr_reg: ElfGRegSet, + pub pr_fpvalid: u32, + // Padding only needed on 64 bit systems for proper alignment. We could + // just set alignment for this struct, but being explicit is better. + #[cfg(target_pointer_width = "64")] + pub pad1: u32, +} + +impl<'a> TryFrom<&'a [u8]> for &'a ProcessStatusNote { + type Error = Error; + + fn try_from(value: &'a [u8]) -> std::result::Result { + if value.len() != size_of::() { + return Err(eyre!( + "Invalid size for ProcessStatusNote: actual size: {} - expected size: {}", + value.len(), + size_of::(), + )); + } + + // SAFETY: ProcessStatusNote only contains scalar values, no pointers. + unsafe { (value.as_ptr() as *const ProcessStatusNote).as_ref() } + .ok_or(eyre!("Invalid pointer ProcessStatusNote")) + } +} + +/// An entry in a ElfNote::File::mapped_files vector. +#[derive(Debug, PartialEq, Eq)] +pub struct MappedFile<'a> { + pub path: Option<&'a Path>, + pub start_addr: usize, + pub end_addr: usize, + pub page_offset: usize, +} + +#[derive(Debug, PartialEq, Eq)] +/// Parsed CORE::NT_FILE note. +pub struct FileNote<'a> { + page_size: usize, + mapped_files: Vec>, + /// The input data was incomplete, so the mapped_files list is not complete. + incomplete: bool, +} + +impl<'a> FileNote<'a> { + const NT_FILE_ENTRY_SIZE: usize = size_of::() * 3; + + /// Parses a CORE::NT_FILE note's description data. + /// + /// Really tries hard to parse out as much as possible, even if the data is incomplete. + /// For example, if the string table is missing, it will still parse the header, but set the + /// path to None. + fn try_parse(data: &'a [u8]) -> Result { + // See linux/fs/binfmt_elf.c: + // https://github.com/torvalds/linux/blob/6465e260f48790807eef06b583b38ca9789b6072/fs/binfmt_elf.c#L1633-L1644 + // + // - long count -- how many files are mapped + // - long page_size -- units for file_ofs + // - array of [COUNT] elements of + // - long start + // - long end + // - long file_ofs + // - followed by COUNT filenames in ASCII: "FILE1" NUL "FILE2" NUL... + // + let mut offset = 0; + let count = data.gread::(&mut offset)? as usize; + let page_size = data.gread::(&mut offset)? as usize; + let mut mapped_files = Vec::with_capacity(count); + + let str_table_start = min(offset + count * Self::NT_FILE_ENTRY_SIZE, data.len()); + let str_table = &data[str_table_start..]; + let mut str_table_offset = 0; + + let mut incomplete = false; + + let mut get_next_path = || -> Option<&Path> { + match str_table.gread::<&CStr>(&mut str_table_offset) { + Ok(cstr) => Some(Path::new(OsStr::from_bytes(cstr.to_bytes()))), + Err(_) => { + incomplete = true; + None + } + } + }; + + let mut parse_entry = || -> Result { + let start_addr = data.gread::(&mut offset)? as usize; + let end_addr = data.gread::(&mut offset)? as usize; + let page_offset = data.gread::(&mut offset)? as usize; + Ok(MappedFile { + path: get_next_path(), + start_addr, + end_addr, + page_offset, + }) + }; + + for _ in 0..count { + match parse_entry() { + Ok(entry) => mapped_files.push(entry), + Err(_) => { + incomplete = true; + break; + } + } + } + + if incomplete { + // Log an error but keep the list with what we've gathered so far + warn!("Incomplete NT_FILE note."); + } + + Ok(Self { + page_size, + incomplete, + mapped_files, + }) + } + + // TODO: MFLT-11766 Use NT_FILE note and PT_LOADs in case /proc/pid/maps read failed + #[allow(dead_code)] + pub fn iter(&self) -> impl Iterator { + self.mapped_files.iter() + } +} + +impl<'a> TryFrom<&'a [u8]> for FileNote<'a> { + type Error = Error; + + fn try_from(value: &'a [u8]) -> std::result::Result { + Self::try_parse(value) + } +} + +#[cfg(test)] +mod test { + use hex::decode; + use insta::assert_debug_snapshot; + use rstest::rstest; + use scroll::IOwrite; + use std::fs::File; + use std::io::{Cursor, Read, Write}; + use std::path::PathBuf; + + use super::*; + + #[rstest] + // Header-only size in case there are no name and description: + #[case( + "", + 0, + "00000000\ + 00000000\ + 78563412" + )] + // Description data is padded to 4-byte alignment: + #[case( + "", + 1, + "00000000\ + 01000000\ + 78563412\ + FF000000" + )] + // Description data already 4-byte aligned: + #[case( + "", + 4, + "00000000\ + 04000000\ + 78563412\ + FFFFFFFF" + )] + // Name data and size includes NUL terminator and is padded to 4-byte alignment: + #[case( + "ABC", + 0, + "04000000\ + 00000000\ + 78563412\ + 41424300" + )] + // Both name and description: + #[case( + "A", + 1, + "02000000\ + 01000000\ + 78563412\ + 41000000\ + FF000000" + )] + fn test_build_elf_note( + #[case] name: &str, + #[case] description_size: usize, + #[case] expected_buffer_contents_hex: &str, + ) { + let note_type = 0x12345678; + let note_desc = [0xffu8; 40]; + let note = build_elf_note(name, ¬e_desc[..description_size], note_type).unwrap(); + + // compare the note buffer contents to the expected hex string: + let expected_buffer_contents = decode(expected_buffer_contents_hex).unwrap(); + assert_eq!(note, expected_buffer_contents); + } + + #[test] + fn test_elf_note_try_parse_build_id_note() { + let build_id = b"ABCD"; + assert_eq!( + ElfNote::try_parse(b"GNU", 3, build_id).unwrap().unwrap(), + ElfNote::GnuBuildId(build_id) + ); + } + + #[test] + fn test_iterate_elf_notes_with_fixture() { + let name = "sample_note"; + let input_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/cli/memfault_core_handler/fixtures") + .join(name) + .with_extension("bin"); + let mut input_file = File::open(input_path).unwrap(); + + let mut note = Vec::new(); + input_file.read_to_end(&mut note).unwrap(); + + let all_notes: Vec<_> = iterate_elf_notes(¬e).collect(); + assert_eq!(all_notes.len(), 7); + + let notes: Vec<_> = all_notes + .into_iter() + .filter(|n| match n { + ElfNote::ProcessStatus(_) => true, + ElfNote::File { .. } => true, + ElfNote::Auxv(_) => true, + ElfNote::GnuBuildId { .. } => false, // Fixture doesn't have a build ID note + ElfNote::Unknown { .. } => false, // Ignore + }) + .collect(); + + assert_debug_snapshot!(notes); + } + + #[test] + fn test_iterate_elf_notes_empty() { + let notes: Vec<_> = ElfNoteIterator::collect(iterate_elf_notes(&[])); + assert!(notes.is_empty()); + } + + #[rstest] + // Aligned desc: + #[case(b"TestNote", &[1, 2, 3, 4, 5, 6, 7, 8])] + // Non-aligned desc: + #[case(b"TestNote", &[1, 2, 3, 4, 5])] + // Empty desc: + #[case(b"TestNote", &[])] + // Aligned name: + #[case(b"TestNot", &[])] + // Empty name: + #[case(b"", &[1, 2, 3, 4, 5])] + fn test_iterate_elf_notes_basic_edge_cases( + #[case] name_value: &[u8], + #[case] note_desc: &[u8], + ) { + let note_type_value = 0x12345678; + let note_buffer = build_elf_note( + &String::from_utf8(name_value.into()).unwrap(), + note_desc, + note_type_value, + ) + .unwrap(); + let notes: Vec<_> = iterate_elf_notes(note_buffer.as_slice()).collect(); + assert_eq!(notes.len(), 1); + + match ¬es[0] { + ElfNote::Unknown { + name, + note_type, + description, + } => { + assert_eq!(*name, name_value); + assert_eq!(*note_type, note_type_value); + assert_eq!(description, ¬e_desc); + } + _ => panic!("Expected unknown note"), + }; + } + + #[test] + fn test_iterate_elf_notes_note_parsing_failure_handling() { + // 2 NT_PRSTATUS notes with a description that is 1 byte short, + // making ElfNote::try_parse fail and returning them as ElfNote::Unknown notes: + let note_desc = [0xffu8; size_of::() - 1]; + let note_buffer = [ + build_elf_note("CORE", ¬e_desc, NT_PRSTATUS).unwrap(), + build_elf_note("CORE", ¬e_desc, NT_PRSTATUS).unwrap(), + ] + .concat(); + + // Note: iteration continues, even if parsing fails for a note: + let notes: Vec<_> = iterate_elf_notes(note_buffer.as_slice()).collect(); + assert_eq!(notes.len(), 2); + for n in notes { + assert!(matches!(n, ElfNote::Unknown { .. })); + } + } + + #[rstest] + // Note header is too short: + #[case(size_of::() - 1)] + // Name is short: + #[case(size_of::() + 1)] + // Desc is short: + #[case(size_of::() + 8 /* "Hello" + padding */ + 1)] + fn test_iterate_elf_notes_note_data_short(#[case] note_size: usize) { + let note_type_value = 0x12345678; + let note_decs = [0xffu8; 40]; + let note_buffer = build_elf_note("Hello", ¬e_decs, note_type_value).unwrap(); + + // Note data is short -- iteration should stop immediately (no more data to read after): + let notes: Vec<_> = ElfNoteIterator::collect(iterate_elf_notes(¬e_buffer[..note_size])); + assert!(notes.is_empty()); + } + + const NT_FILE_HDR_SIZE: usize = size_of::() * 2; + + #[rstest] + // If we don't have the header, the parsing will fail: + #[case(0)] + #[case(NT_FILE_HDR_SIZE - 1)] + fn test_elf_note_try_parse_file_note_too_short(#[case] desc_size: usize) { + let file_note_desc = make_nt_file_note_data_fixture(); + + // Clip the description data to the given size: + let file_note_desc = &file_note_desc[..desc_size]; + + assert!(FileNote::try_parse(file_note_desc).is_err()); + } + + #[rstest] + // Header only -- no mapped_files: + #[case( + NT_FILE_HDR_SIZE, + FileNote { + mapped_files: vec![], + incomplete: true, + page_size: 0x1000, + } + )] + // Header + one entry -- one mapped_file, no paths (missing string table): + #[case( + NT_FILE_HDR_SIZE + FileNote::NT_FILE_ENTRY_SIZE, + FileNote { + mapped_files: vec![ + MappedFile { + path: None, + start_addr: 0, + end_addr: 1, + page_offset: 2, + }, + ], + incomplete: true, + page_size: 0x1000, + } + )] + // Header + one entry -- one mapped_file, no paths (missing string table): + #[case( + NT_FILE_HDR_SIZE + (2 * FileNote::NT_FILE_ENTRY_SIZE) + 16, + FileNote { + mapped_files: vec![ + MappedFile { + path: Some(Path::new("/path/to/file")), + start_addr: 0, + end_addr: 1, + page_offset: 2, + }, + MappedFile { + path: None, + start_addr: 0, + end_addr: 1, + page_offset: 2, + }, + ], + incomplete: true, + page_size: 0x1000, + } + )] + + fn test_elf_note_try_parse_file_note_incomplete_string_table( + #[case] desc_size: usize, + #[case] expected: FileNote, + ) { + let file_note_desc = make_nt_file_note_data_fixture(); + + // Clip the description data to the given size: + let file_note_desc = &file_note_desc[..desc_size]; + + let note = FileNote::try_parse(file_note_desc).unwrap(); + assert_eq!(note, expected); + } + + const FIXTURE_FILE_PATH: &[u8; 14] = b"/path/to/file\0"; + + fn make_nt_file_note_data_fixture() -> Vec { + let mut cursor = Cursor::new(vec![]); + let count = 2; + + // Header: + // Count: + cursor.iowrite::(count).unwrap(); + // Page size (0x1000): + cursor.iowrite::(0x1000).unwrap(); + + // Entries: + for _ in 0..count { + // Start, end, file offset: + for n in 0..3 { + cursor.iowrite::(n).unwrap(); + } + } + + // String table: + for _ in 0..count { + let _ = cursor.write(FIXTURE_FILE_PATH).unwrap(); + } + + let file_note_desc = cursor.into_inner(); + assert_eq!(file_note_desc.len(), 92); + file_note_desc + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/core_reader.rs b/memfaultd/src/cli/memfault_core_handler/core_reader.rs new file mode 100644 index 0000000..f7096b5 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/core_reader.rs @@ -0,0 +1,193 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{Read, Seek, SeekFrom}; + +use crate::cli::memfault_core_handler::arch::{ELF_TARGET_ENDIANNESS, ELF_TARGET_MACHINE}; +use crate::{cli::memfault_core_handler::elf, util::io::ForwardOnlySeeker}; + +use elf::header::{Header, EI_CLASS, EI_DATA, ELFCLASS, ELFMAG, EV_CURRENT, SELFMAG, SIZEOF_EHDR}; +use elf::program_header::{ProgramHeader, SIZEOF_PHDR}; +use eyre::{eyre, Result}; + +pub trait CoreReader { + /// Reads program headers from the input stream + fn read_program_headers(&mut self) -> Result>; + + /// Reads segment data from the input stream + fn read_segment_data(&mut self, program_header: &ProgramHeader) -> Result>; +} + +// Reads ELF headers and segments from a core file +pub struct CoreReaderImpl { + input_stream: ForwardOnlySeeker, + elf_header: Header, +} + +impl CoreReader for CoreReaderImpl { + fn read_program_headers(&mut self) -> Result> { + // Ignore unnecessary cast here as it is needed on 32-bit systems. + #[allow(clippy::unnecessary_cast)] + self.input_stream + .seek(SeekFrom::Start(self.elf_header.e_phoff as u64))?; + + let mut program_headers = + read_program_headers(&mut self.input_stream, self.elf_header.e_phnum as usize)?; + + // Sort, just in case the program headers are not sorted by offset. + // Otherwise the read_segment_data() calls later may fail. + program_headers.sort_by_key(|ph| ph.p_offset); + + Ok(program_headers) + } + + fn read_segment_data(&mut self, program_header: &ProgramHeader) -> Result> { + // Ignore unnecessary cast here as it is needed on 32-bit systems. + #[allow(clippy::unnecessary_cast)] + self.input_stream + .seek(SeekFrom::Start(program_header.p_offset as u64))?; + + let mut buf = vec![0; program_header.p_filesz as usize]; + self.input_stream.read_exact(&mut buf)?; + + Ok(buf) + } +} + +impl CoreReaderImpl { + /// Creates an instance of `CoreReader` from an input stream + pub fn new(input_stream: R) -> Result { + let mut input_stream = ForwardOnlySeeker::new(input_stream); + let mut header_buf = [0u8; SIZEOF_EHDR]; + input_stream.read_exact(&mut header_buf)?; + + let elf_header = *Header::from_bytes(&header_buf); + if !Self::verify_elf_header(&elf_header) { + return Err(eyre!("Invalid ELF header")); + } + + Ok(CoreReaderImpl { + input_stream, + elf_header, + }) + } + + fn verify_elf_header(header: &Header) -> bool { + &header.e_ident[0..SELFMAG] == ELFMAG + && header.e_ident[EI_CLASS] == ELFCLASS + && header.e_ident[EI_DATA] == ELF_TARGET_ENDIANNESS + && header.e_version == EV_CURRENT as u32 + && header.e_ehsize == SIZEOF_EHDR as u16 + && header.e_phentsize == SIZEOF_PHDR as u16 + && header.e_machine == ELF_TARGET_MACHINE + } + + pub fn elf_header(&self) -> Header { + self.elf_header + } +} + +/// Reads `count` ELF program headers from the provided input stream. +pub fn read_program_headers( + input_stream: &mut R, + count: usize, +) -> Result> { + let size = count * SIZEOF_PHDR; + let mut buffer = vec![0; size]; + input_stream.read_exact(&mut buffer)?; + Ok(ProgramHeader::from_bytes(&buffer, count)) +} + +#[cfg(test)] +mod test { + use std::fs::File; + use std::io::Cursor; + + use super::*; + + use crate::cli::memfault_core_handler::arch::ELF_TARGET_CLASS; + use crate::cli::memfault_core_handler::test_utils::build_test_header; + use elf::header::{ELFCLASSNONE, ELFDATANONE, EM_NONE}; + + use rstest::rstest; + use scroll::Pwrite; + + #[rstest] + #[case(0)] + #[case(1024)] // Test with padding between header and program headers + fn test_read_program_headers(#[case] ph_offset: usize) { + let mut elf_header = + build_test_header(ELF_TARGET_CLASS, ELF_TARGET_ENDIANNESS, ELF_TARGET_MACHINE); + elf_header.e_phnum = 2; + elf_header.e_phoff = (SIZEOF_EHDR + ph_offset).try_into().unwrap(); + let load_program_header = ProgramHeader { + p_type: elf::program_header::PT_LOAD, + p_vaddr: 0, + ..Default::default() + }; + let note_program_header = ProgramHeader { + p_type: elf::program_header::PT_NOTE, + p_offset: 0x1000, + ..Default::default() + }; + + // Build ELF input stream + let mut input_stream = vec![0; SIZEOF_EHDR + 2 * SIZEOF_PHDR + ph_offset]; + input_stream.pwrite(elf_header, 0).unwrap(); + input_stream + .pwrite(load_program_header, SIZEOF_EHDR + ph_offset) + .unwrap(); + input_stream + .pwrite(note_program_header, SIZEOF_EHDR + SIZEOF_PHDR + ph_offset) + .unwrap(); + + // Verify headers are read correctly + let mut reader = CoreReaderImpl::new(Cursor::new(input_stream)).unwrap(); + let program_headers = reader.read_program_headers().unwrap(); + assert_eq!(program_headers.len(), 2); + assert_eq!(program_headers[0], load_program_header); + assert_eq!(program_headers[1], note_program_header); + } + + #[rstest] + #[case(0, 1024)] + #[case(1024, 1024)] + fn test_read_segment_data(#[case] offset: usize, #[case] size: usize) { + const TEST_BYTE: u8 = 0x42; + + let elf_header = + build_test_header(ELF_TARGET_CLASS, ELF_TARGET_ENDIANNESS, ELF_TARGET_MACHINE); + let offset = offset + SIZEOF_EHDR; + let note_program_header = ProgramHeader { + p_type: elf::program_header::PT_NOTE, + p_offset: offset.try_into().unwrap(), + p_filesz: size.try_into().unwrap(), + ..Default::default() + }; + + let mut input_stream = vec![0u8; offset + size]; + input_stream.pwrite(elf_header, 0).unwrap(); + input_stream[offset..(offset + size)].fill(TEST_BYTE); + + let mut reader = CoreReaderImpl::new(Cursor::new(&input_stream)).unwrap(); + let segment_data = reader.read_segment_data(¬e_program_header).unwrap(); + + assert_eq!(segment_data, input_stream[offset..(offset + size)]); + } + + #[rstest] + // Mismatching class (32 vs 64 bit): + #[case(ELFCLASSNONE, ELF_TARGET_ENDIANNESS, ELF_TARGET_MACHINE)] + // Mismatching endianness: + #[case(ELF_TARGET_CLASS, ELFDATANONE, ELF_TARGET_MACHINE)] + // Mismatching machine: + #[case(ELF_TARGET_CLASS, ELF_TARGET_ENDIANNESS, EM_NONE)] + fn test_verify_elf_header_fails_for_mismatching_arch( + #[case] class: u8, + #[case] endianness: u8, + #[case] machine: u16, + ) { + let elf_header = build_test_header(class, endianness, machine); + assert!(!CoreReaderImpl::::verify_elf_header(&elf_header)); + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/core_transformer.rs b/memfaultd/src/cli/memfault_core_handler/core_transformer.rs new file mode 100644 index 0000000..906eca6 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/core_transformer.rs @@ -0,0 +1,474 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + io::{Read, Seek, SeekFrom}, + sync::mpsc::Receiver, +}; +use std::{mem::size_of, time::Duration}; + +use crate::cli::memfault_core_handler::core_elf_note::{iterate_elf_notes, ElfNote}; +use crate::cli::memfault_core_handler::core_reader::CoreReader; +use crate::cli::memfault_core_handler::core_writer::{CoreWriter, SegmentData}; +use crate::cli::memfault_core_handler::elf; +use crate::cli::memfault_core_handler::find_dynamic::find_dynamic_linker_ranges; +use crate::cli::memfault_core_handler::find_elf_headers::find_elf_headers_and_build_id_note_ranges; +use crate::cli::memfault_core_handler::find_stack::find_stack; +use crate::cli::memfault_core_handler::memory_range::{merge_memory_ranges, MemoryRange}; +use crate::cli::memfault_core_handler::procfs::ProcMaps; +use crate::cli::memfault_core_handler::ElfPtrSize; +use crate::config::CoredumpCaptureStrategy; +use crate::{ + cli::memfault_core_handler::core_elf_memfault_note::{ + write_memfault_metadata_note, CoredumpMetadata, + }, + mar::LinuxLogsFormat, +}; + +use elf::program_header::{ProgramHeader, PT_LOAD, PT_NOTE}; +use eyre::{eyre, Result}; +use libc::{AT_PHDR, AT_PHNUM}; +use log::{debug, warn}; +use procfs::process::MemoryMap; + +use super::{ + core_elf_memfault_note::{write_memfault_debug_data_note, MemfaultMetadataLogs}, + log_wrapper::CAPTURE_LOG_CHANNEL_SIZE, +}; + +#[derive(Debug)] +pub struct CoreTransformerOptions { + pub max_size: usize, + pub capture_strategy: CoredumpCaptureStrategy, + pub thread_filter_supported: bool, +} + +/// Reads segments from core elf stream and memory stream and builds a core new elf file. +pub struct CoreTransformer +where + R: CoreReader, + W: CoreWriter, + P: Read + Seek, + M: ProcMaps, +{ + core_reader: R, + core_writer: W, + proc_mem_stream: P, + metadata: CoredumpMetadata, + options: CoreTransformerOptions, + proc_maps: M, + log_fetcher: CoreTransformerLogFetcher, +} + +impl CoreTransformer +where + R: CoreReader, + W: CoreWriter, + P: Read + Seek, + M: ProcMaps, +{ + /// Creates an instance of `CoreTransformer` from an input stream and output stream + pub fn new( + core_reader: R, + core_writer: W, + proc_mem_stream: P, + options: CoreTransformerOptions, + metadata: CoredumpMetadata, + proc_maps: M, + log_fetcher: CoreTransformerLogFetcher, + ) -> Result { + Ok(Self { + core_reader, + core_writer, + proc_mem_stream, + metadata, + options, + proc_maps, + log_fetcher, + }) + } + + /// Reads segments from core elf stream and memory stream and builds a new elf file + /// + /// Reads all PT_LOAD and PT_NOTE program headers and their associated segments from the core. + /// The memory for each PT_LOAD segment will be fetched from `/proc//mem` and the + /// resulting elf file will be written to `output_stream`. + pub fn run_transformer(&mut self) -> Result<()> { + let program_headers = self.core_reader.read_program_headers()?; + let all_notes = self.read_all_note_segments(&program_headers); + + let segments_to_capture = match self.options.capture_strategy { + CoredumpCaptureStrategy::KernelSelection => { + self.kernel_selection_segments(&program_headers) + } + CoredumpCaptureStrategy::Threads { max_thread_size } => { + // Fallback to kernel selection if thread filtering is not supported. + // Support is dictated by whether we have a definition for the platform + // user regs. See arch.rs for more details. + if self.options.thread_filter_supported { + let mapped_ranges = self.proc_maps.get_process_maps()?; + self.threads_segments(&mapped_ranges, &all_notes, max_thread_size) + } else { + // Update metadata so capture strategy is reflected properly in the memfault note. + self.metadata.capture_strategy = CoredumpCaptureStrategy::KernelSelection; + self.kernel_selection_segments(&program_headers) + } + } + }; + + // Always copy over all note segments, regardless of the capturing strategy: + for (ph, data) in all_notes { + self.core_writer.add_segment(*ph, SegmentData::Buffer(data)); + } + + for ph in segments_to_capture { + self.core_writer.add_segment(ph, SegmentData::ProcessMemory); + } + + self.add_memfault_metadata_note()?; + self.add_memfault_debug_data_note()?; + self.check_output_size()?; + self.core_writer.write()?; + + Ok(()) + } + + /// Add note segments from the core elf to the output elf, verbatim. + fn read_all_note_segments<'a>( + &mut self, + program_headers: &'a [ProgramHeader], + ) -> Vec<(&'a ProgramHeader, Vec)> { + program_headers + .iter() + .filter_map(|ph| match ph.p_type { + PT_NOTE => match self.core_reader.read_segment_data(ph) { + Ok(data) => Some((ph, data)), + _ => None, + }, + _ => None, + }) + .collect::>() + } + + /// All load segments from the original/input core elf as provided by the kernel, verbatim. + fn kernel_selection_segments( + &mut self, + program_headers: &[ProgramHeader], + ) -> Vec { + program_headers + .iter() + .filter_map(|ph| match ph.p_type { + PT_LOAD => Some(*ph), + _ => None, + }) + .collect::>() + } + + /// Synthesizes segments for the stacks of all threads and adds them to the output, as well as + /// any mandatory segments (build id notes, r_debug, etc.). + fn threads_segments( + &mut self, + memory_maps: &[MemoryMap], + all_notes: &[(&ProgramHeader, Vec)], + max_thread_size: usize, + ) -> Vec { + let memory_maps_ranges: Vec = + memory_maps.iter().map(MemoryRange::from).collect(); + + let parsed_notes = all_notes + .iter() + .flat_map(|(_, data)| iterate_elf_notes(data)) + .collect::>(); + + let mut mem_ranges = Vec::new(); + let mut phdr_vaddr = None; + let mut phdr_num = None; + + for note in &parsed_notes { + match note { + ElfNote::ProcessStatus(s) => { + if let Some(stack) = find_stack(&s.pr_reg, &memory_maps_ranges, max_thread_size) + { + mem_ranges.push(stack); + } else { + warn!("Failed to collect stack for thread: {}", s.pr_pid); + } + } + ElfNote::Auxv(auxv) => { + phdr_num = auxv.find_value(AT_PHNUM); + phdr_vaddr = auxv.find_value(AT_PHDR); + } + _ => {} + } + } + + mem_ranges.extend( + memory_maps + .iter() + .filter(|mmap| mmap.offset == 0) + .flat_map( + |mmap| match self.elf_metadata_ranges_for_mapped_file(mmap.address.0) { + Ok(ranges) => ranges, + Err(e) => { + debug!( + "Failed to collect metadata for {:?} @ {:#x}: {}", + mmap.pathname, mmap.address.0, e + ); + vec![] + } + }, + ), + ); + + match (phdr_vaddr, phdr_num) { + (Some(phdr_vaddr), Some(phdr_num)) => { + if let Err(e) = find_dynamic_linker_ranges( + &mut self.proc_mem_stream, + phdr_vaddr, + phdr_num, + &memory_maps_ranges, + &mut mem_ranges, + ) { + warn!("Failed to collect dynamic linker ranges: {}", e); + } + } + _ => { + warn!("Missing AT_PHDR or AT_PHNUM auxv entry"); + } + }; + + // Merge overlapping memory ranges and turn them into PT_LOAD program headers. As a + // side-effect, this will also sort the program headers by vaddr. + let merged_ranges = merge_memory_ranges(mem_ranges); + merged_ranges.into_iter().map(ProgramHeader::from).collect() + } + + fn elf_metadata_ranges_for_mapped_file(&mut self, vaddr_base: u64) -> Result> { + // Ignore unnecessary cast here as it is needed on 32-bit systems. + #[allow(clippy::unnecessary_cast)] + self.proc_mem_stream + .seek(SeekFrom::Start(vaddr_base as u64))?; + find_elf_headers_and_build_id_note_ranges( + vaddr_base as ElfPtrSize, + &mut self.proc_mem_stream, + ) + } + + /// Check if the output file size exceeds the max size available + fn check_output_size(&self) -> Result<()> { + let output_size = self.core_writer.calc_output_size(); + if output_size > self.options.max_size { + return Err(eyre!( + "Core file size {} exceeds max size {}", + output_size, + self.options.max_size + )); + } + + Ok(()) + } + + fn add_memfault_metadata_note(&mut self) -> Result<()> { + let app_logs = self + .log_fetcher + .get_app_logs() + .map(|logs| MemfaultMetadataLogs::new(logs, LinuxLogsFormat::default())); + + self.metadata.app_logs = app_logs; + + let note_data = write_memfault_metadata_note(&self.metadata)?; + self.add_memfault_note(note_data) + } + + fn add_memfault_debug_data_note(&mut self) -> Result<()> { + let mut capture_logs = self.log_fetcher.get_capture_logs(); + if capture_logs.is_empty() { + return Ok(()); + } + + if capture_logs.len() == CAPTURE_LOG_CHANNEL_SIZE { + capture_logs.push("Log overflow, some logs may have been dropped".to_string()); + } + + let buffer = write_memfault_debug_data_note(capture_logs)?; + self.add_memfault_note(buffer) + } + + /// Add memfault note to the core elf + /// + /// Contains CBOR encoded information about the system capturing the coredump. See + /// [`CoredumpMetadata`] for more information. + fn add_memfault_note(&mut self, desc: Vec) -> Result<()> { + let program_header = ProgramHeader { + p_type: PT_NOTE, + p_filesz: desc.len().try_into()?, + ..Default::default() + }; + self.core_writer + .add_segment(program_header, SegmentData::Buffer(desc)); + + Ok(()) + } +} + +/// Convenience struct to hold the log receivers for the core transformer. +/// +/// This is used to receive logs from the `CoreHandlerLogWrapper` and the the circular buffer +/// capture logs. +#[derive(Debug)] +pub struct CoreTransformerLogFetcher { + capture_logs_rx: Receiver, + app_logs_rx: Receiver>>, +} + +impl CoreTransformerLogFetcher { + /// A timeout of 500s is used to prevent the coredump handler from blocking indefinitely. + /// This is to prevent a slow http request from blocking the coredump handler indefinitely. + const APPLICATION_LOGS_TIMEOUT: Duration = Duration::from_millis(500); + + pub fn new( + capture_logs_rx: Receiver, + app_logs_rx: Receiver>>, + ) -> Self { + Self { + capture_logs_rx, + app_logs_rx, + } + } + + /// Fetches all logs that were captured in the coredump handler during execution. + pub fn get_capture_logs(&self) -> Vec { + self.capture_logs_rx.try_iter().collect() + } + + /// Fetches all application logs that were in the circular buffer at time of crash. + /// + /// This method will block for a maximum of 500ms before returning. This is to prevent a + /// slow http request from blocking the coredump handler indefinitely. + pub fn get_app_logs(&self) -> Option> { + let logs = self + .app_logs_rx + .recv_timeout(Self::APPLICATION_LOGS_TIMEOUT); + + match logs { + Ok(logs) => logs, + Err(e) => { + debug!("Failed to fetch application logs within timeout: {}", e); + None + } + } + } +} + +impl From for ProgramHeader { + fn from(range: MemoryRange) -> Self { + ProgramHeader { + p_type: PT_LOAD, + p_vaddr: range.start, + p_filesz: range.size(), + p_memsz: range.size(), + p_align: size_of::() as ElfPtrSize, + ..Default::default() + } + } +} + +#[cfg(test)] +mod test { + use crate::cli::memfault_core_handler::test_utils::{ + FakeProcMaps, FakeProcMem, MockCoreWriter, + }; + use crate::test_utils::setup_logger; + use crate::{ + cli::memfault_core_handler::core_reader::CoreReaderImpl, test_utils::set_snapshot_suffix, + }; + use insta::assert_debug_snapshot; + use rstest::rstest; + use std::fs::File; + use std::path::PathBuf; + use std::sync::mpsc::sync_channel; + + use super::*; + + #[rstest] + #[case("kernel_selection", CoredumpCaptureStrategy::KernelSelection, true)] + #[case("threads_32k", CoredumpCaptureStrategy::Threads { max_thread_size: 32 * 1024 }, true)] + #[case( + "threads_32k_no_filter_support", + CoredumpCaptureStrategy::Threads { max_thread_size: 32 * 1024 }, + false + )] + fn test_transform( + #[case] test_case_name: &str, + #[case] capture_strategy: CoredumpCaptureStrategy, + #[case] thread_filter_supported: bool, + _setup_logger: (), + ) { + let input_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/cli/memfault_core_handler/fixtures/elf-core-runtime-ld-paths.elf"); + let input_stream = File::open(&input_path).unwrap(); + let proc_mem_stream = FakeProcMem::new_from_path(&input_path).unwrap(); + let proc_maps = FakeProcMaps::new_from_path(&input_path).unwrap(); + let opts = CoreTransformerOptions { + max_size: 1024 * 1024, + capture_strategy, + thread_filter_supported, + }; + let metadata = CoredumpMetadata { + device_id: "12345678".to_string(), + hardware_version: "evt".to_string(), + software_type: "main".to_string(), + software_version: "1.0.0".to_string(), + sdk_version: "SDK_VERSION".to_string(), + captured_time_epoch_s: 1234, + cmd_line: "binary -a -b -c".to_string(), + capture_strategy, + app_logs: None, + }; + + let (_capture_logs_tx, capture_logs_rx) = sync_channel(32); + let (app_logs_tx, app_logs_rx) = sync_channel(1); + app_logs_tx + .send(Some(vec![ + "Error 1".to_string(), + "Error 2".to_string(), + "Error 3".to_string(), + ])) + .unwrap(); + + let core_reader = CoreReaderImpl::new(input_stream).unwrap(); + let mut segments = vec![]; + let mock_core_writer = MockCoreWriter::new(&mut segments); + let log_rx = CoreTransformerLogFetcher { + capture_logs_rx, + app_logs_rx, + }; + let mut transformer = CoreTransformer::new( + core_reader, + mock_core_writer, + proc_mem_stream, + opts, + metadata, + proc_maps, + log_rx, + ) + .unwrap(); + + transformer.run_transformer().unwrap(); + + // Omit the actual data from the notes: + let segments = segments + .iter() + .map(|(ph, seg)| { + let seg = match seg { + SegmentData::ProcessMemory => SegmentData::ProcessMemory, + SegmentData::Buffer(_) => SegmentData::Buffer(vec![]), + }; + (ph, seg) + }) + .collect::>(); + + set_snapshot_suffix!("{}", test_case_name); + assert_debug_snapshot!(segments); + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/core_writer.rs b/memfaultd/src/cli/memfault_core_handler/core_writer.rs new file mode 100644 index 0000000..1b266d4 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/core_writer.rs @@ -0,0 +1,392 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{copy, repeat, Read, Seek, SeekFrom, Write}; + +use crate::cli::memfault_core_handler::elf; +use crate::util::io::StreamPosition; +use crate::util::math::align_up; + +use elf::header::{ + Header, EI_CLASS, EI_DATA, EI_VERSION, ELFMAG, ET_CORE, EV_CURRENT, SELFMAG, SIZEOF_EHDR, +}; +use elf::program_header::{ProgramHeader, SIZEOF_PHDR}; +use eyre::Result; +use scroll::Pwrite; + +const FILL_BYTE: u8 = 0xEF; + +pub trait CoreWriter { + // Adds a data segment to the writer. + fn add_segment(&mut self, program_header: ProgramHeader, data: SegmentData); + + /// Writes the core elf to the output stream. + fn write(&mut self) -> Result<()>; + + /// Calculate output coredump size. + /// + /// The max size is calculated as the size of the elf header, program header table, and all + /// segment data. We take the conservative route here and calculate the size of the ELF + /// uncompressed, even if compression is enabled. It's likely that the compressed file will + /// be smaller, but this at least gives us a worst case estimate. + fn calc_output_size(&self) -> usize; +} + +#[derive(Debug)] +pub enum SegmentData { + Buffer(Vec), + ProcessMemory, +} + +#[derive(Debug)] +pub struct Segment { + program_header: ProgramHeader, + data: SegmentData, +} + +/// Creates a new ELF core file from a set of program headers and associated segment data. +pub struct CoreWriterImpl +where + W: Write + StreamPosition, + R: Read + Seek, +{ + elf_header: Header, + data_segments: Vec, + process_memory: R, + output_stream: W, +} + +impl CoreWriter for CoreWriterImpl +where + W: Write + StreamPosition, + R: Read + Seek, +{ + fn add_segment(&mut self, program_header: ProgramHeader, data: SegmentData) { + self.data_segments.push(Segment { + program_header, + data, + }); + } + + fn write(&mut self) -> Result<()> { + self.write_elf_header()?; + self.write_data_segments()?; + Ok(()) + } + + fn calc_output_size(&self) -> usize { + let initial = (self.elf_header.e_phnum * self.elf_header.e_phentsize + + self.elf_header.e_ehsize) as usize; + self.data_segments.iter().fold(initial, |acc, s| { + align_up( + acc + s.program_header.p_filesz as usize, + s.program_header.p_align as usize, + ) + }) + } +} +impl CoreWriterImpl +where + W: Write + StreamPosition, + R: Read + Seek, +{ + /// Creates a new instance of `CoreWriter` + pub fn new(elf_header: Header, output_stream: W, process_memory: R) -> Self { + Self { + elf_header, + data_segments: Vec::new(), + process_memory, + output_stream, + } + } + + /// Write ELF header to output stream. + fn write_elf_header(&mut self) -> Result<()> { + let mut e_ident = [0u8; 16]; + e_ident[..SELFMAG].copy_from_slice(ELFMAG); + e_ident[EI_CLASS] = self.elf_header.e_ident[EI_CLASS]; + e_ident[EI_DATA] = self.elf_header.e_ident[EI_DATA]; + e_ident[EI_VERSION] = EV_CURRENT; + + let segment_count = self.data_segments.len(); + let (pheader_size, pheader_offset) = if segment_count > 0 { + (SIZEOF_PHDR as u16, SIZEOF_EHDR) + } else { + (0, 0) + }; + + let header = Header { + e_ident, + e_type: ET_CORE, + e_machine: self.elf_header.e_machine, + e_version: EV_CURRENT as u32, + e_ehsize: SIZEOF_EHDR as u16, + e_phentsize: pheader_size, + e_phnum: segment_count as u16, + e_phoff: pheader_offset.try_into()?, + ..Default::default() + }; + + let mut bytes: Vec = vec![0; self.elf_header.e_ehsize as usize]; + self.elf_header.e_phnum = self.data_segments.len().try_into()?; + bytes.pwrite(header, 0)?; + + Self::write_to_output(&mut self.output_stream, &bytes)?; + Ok(()) + } + + /// Write program header table and associated segment data + /// + /// The program header table is written first, followed by the segment data. The segment data + /// is written in the same order as the program headers, so that we don't have to seek through + /// the output data stream. + fn write_data_segments(&mut self) -> Result<()> { + // Write the program header table first. For each program header, calculate the offset + // for the associated segment data, calculate padding if necessary. We calculate the + // offset here so that we can later write all data segments sequentially without + // seeking. + let mut segment_data_offset = self.elf_header.e_phoff as usize + + self.elf_header.e_phentsize as usize * self.data_segments.len(); + for segment in &mut self.data_segments { + let padding = calc_padding( + segment_data_offset, + segment.program_header.p_align.try_into()?, + ); + segment.program_header.p_offset = (segment_data_offset + padding).try_into()?; + let mut bytes: Vec = vec![0; self.elf_header.e_phentsize as usize]; + segment_data_offset += segment.program_header.p_filesz as usize + padding; + + bytes.pwrite(segment.program_header, 0)?; + Self::write_to_output(&mut self.output_stream, &bytes)?; + } + + // Iterate through all segments and write the data to the output stream. Zeroed padding + // is written if the file offset is less than expected segment data offset. + for segment in &self.data_segments { + let cur_position = self.output_stream.stream_position(); + let padding = segment.program_header.p_offset as usize - cur_position; + Self::write_padding(&mut self.output_stream, padding)?; + + match &segment.data { + SegmentData::Buffer(data) => { + Self::write_to_output(&mut self.output_stream, data)?; + } + SegmentData::ProcessMemory => { + let header = &segment.program_header; + + if Self::read_process_memory( + header.p_vaddr as usize, + header.p_filesz as usize, + &mut self.process_memory, + &mut self.output_stream, + ) + .is_err() + { + let segment_end = (header.p_offset + header.p_filesz) as usize; + Self::fill_remaining_bytes(segment_end, &mut self.output_stream)?; + } + } + } + } + + Ok(()) + } + + /// Fill remaining bytes of segment when reading process memory fails. + fn fill_remaining_bytes(segment_end: usize, output_stream: &mut W) -> Result<()> { + let fill_size = segment_end.checked_sub(output_stream.stream_position()); + if let Some(fill_size) = fill_size { + let mut fill_stream = repeat(FILL_BYTE).take(fill_size as u64); + copy(&mut fill_stream, output_stream)?; + } + Ok(()) + } + + fn read_process_memory( + addr: usize, + len: usize, + process_memory: &mut R, + output_stream: &mut W, + ) -> Result { + process_memory.seek(SeekFrom::Start(addr as u64))?; + let mut mem_reader = process_memory.take(len as u64); + let bytes_read = copy(&mut mem_reader, output_stream)?; + + Ok(bytes_read) + } + + /// Write padding if necessary + fn write_padding(output_stream: &mut W, padding: usize) -> Result<()> { + if padding > 0 { + let mut padding_stream = repeat(0u8).take(padding as u64); + copy(&mut padding_stream, output_stream)?; + } + Ok(()) + } + + /// Write to output stream and increment cursor + fn write_to_output(output_stream: &mut W, bytes: &[u8]) -> Result<()> { + output_stream.write_all(bytes)?; + Ok(()) + } +} + +fn calc_padding(offset: usize, alignment: usize) -> usize { + if alignment <= 1 { + return 0; + } + + let next_addr = align_up(offset, alignment); + next_addr - offset +} + +#[cfg(test)] +mod test { + use std::io; + + use crate::cli::memfault_core_handler::arch::{ + ELF_TARGET_CLASS, ELF_TARGET_ENDIANNESS, ELF_TARGET_MACHINE, + }; + use crate::cli::memfault_core_handler::test_utils::build_test_header; + use crate::util::io::StreamPositionTracker; + + use super::*; + + use rstest::rstest; + use std::io::Cursor; + + const PROC_MEM_READ_CHUNK_SIZE: usize = 1024; + + #[rstest] + #[case(SegmentData::Buffer(vec![0xa5; 1024]), vec![])] + #[case(SegmentData::ProcessMemory, vec![0xaa; PROC_MEM_READ_CHUNK_SIZE])] + #[case(SegmentData::ProcessMemory, vec![0xaa; PROC_MEM_READ_CHUNK_SIZE + PROC_MEM_READ_CHUNK_SIZE / 4])] + fn test_added_segments(#[case] segment_data: SegmentData, #[case] mem_buffer: Vec) { + let mem_stream = Cursor::new(mem_buffer.clone()); + let mut output_buf = Vec::new(); + let mut core_writer = build_test_writer(mem_stream, &mut output_buf); + + let segment_buffer = match &segment_data { + SegmentData::Buffer(data) => data.clone(), + SegmentData::ProcessMemory => mem_buffer, + }; + + core_writer.add_segment( + ProgramHeader { + p_type: elf::program_header::PT_LOAD, + p_offset: 0, + p_filesz: segment_buffer.len().try_into().unwrap(), + p_align: 0, + ..Default::default() + }, + segment_data, + ); + + core_writer.write().expect("Failed to write core"); + + let elf_header_buf = output_buf[..SIZEOF_EHDR].try_into().unwrap(); + let elf_header = Header::from_bytes(elf_header_buf); + assert_eq!(elf_header.e_phnum, 1); + + // Build program header table and verify correct number of headers + let ph_table_sz = elf_header.e_phnum as usize * elf_header.e_phentsize as usize; + let ph_header_buf = &output_buf[SIZEOF_EHDR..(SIZEOF_EHDR + ph_table_sz)]; + let ph_headers = ProgramHeader::from_bytes(ph_header_buf, elf_header.e_phnum as usize); + assert_eq!(ph_headers.len(), 1); + + // Verify correct program header for added segment + assert_eq!(ph_headers[0].p_type, elf::program_header::PT_LOAD); + assert_eq!(ph_headers[0].p_filesz as usize, segment_buffer.len()); + + // Verify segment data starts after elf header and program header table + let segment_data_offset = ph_headers[0].p_offset as usize; + assert_eq!(segment_data_offset, SIZEOF_EHDR + ph_table_sz); + + // Verify correct segment data + let serialized_segment_data = &output_buf[ph_headers[0].p_offset as usize + ..(ph_headers[0].p_offset + ph_headers[0].p_filesz) as usize]; + assert_eq!(&segment_buffer, serialized_segment_data); + } + + #[rstest] + #[case(vec![1024, 1024], 1024, 3072)] + #[case(vec![2048, 1024], 512, 3584)] + #[case(vec![2048, 1024], 0, 3136)] + #[case(vec![2048, 1024], 1, 3136)] + fn test_output_size_calculation( + #[case] segment_sizes: Vec, + #[case] alignment: usize, + #[case] expected_size: usize, + ) { + let mem_stream = Vec::new(); + let mut output_buf = Vec::new(); + let mut core_writer = build_test_writer(Cursor::new(mem_stream), &mut output_buf); + + segment_sizes.iter().for_each(|size| { + core_writer.add_segment( + ProgramHeader { + p_type: elf::program_header::PT_LOAD, + p_filesz: *size as u64, + p_align: alignment.try_into().unwrap(), + ..Default::default() + }, + SegmentData::ProcessMemory, + ); + }); + + let output_size = core_writer.calc_output_size(); + assert_eq!(output_size, expected_size); + } + + #[test] + fn test_read_fail() { + let mut output_buf = Vec::new(); + let mut writer = build_test_writer(FailReader, &mut output_buf); + let segment_size = 1024usize; + + writer.add_segment( + ProgramHeader { + p_type: elf::program_header::PT_LOAD, + p_filesz: segment_size.try_into().unwrap(), + ..Default::default() + }, + SegmentData::ProcessMemory, + ); + + writer.write().unwrap(); + + let header = writer.elf_header; + let segment_offset = + header.e_phoff as usize + header.e_phentsize as usize * header.e_phnum as usize; + let segment_end = segment_offset + segment_size; + assert_eq!( + output_buf[segment_offset..segment_end], + vec![FILL_BYTE; segment_size] + ); + } + + fn build_test_writer( + mem_stream: T, + output_buf: &mut Vec, + ) -> CoreWriterImpl>> { + let elf_header = + build_test_header(ELF_TARGET_CLASS, ELF_TARGET_ENDIANNESS, ELF_TARGET_MACHINE); + let output_stream = StreamPositionTracker::new(output_buf); + CoreWriterImpl::new(elf_header, output_stream, mem_stream) + } + + /// Test reader that always returns failure when reading. + struct FailReader; + + impl Read for FailReader { + fn read(&mut self, _: &mut [u8]) -> io::Result { + Err(io::Error::new(io::ErrorKind::Other, "read failed")) + } + } + + impl Seek for FailReader { + fn seek(&mut self, _: SeekFrom) -> io::Result { + Ok(0) + } + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/find_dynamic.rs b/memfaultd/src/cli/memfault_core_handler/find_dynamic.rs new file mode 100644 index 0000000..5cbfda9 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/find_dynamic.rs @@ -0,0 +1,392 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{Read, Seek, SeekFrom}; +use std::mem::size_of; + +use elf::dynamic::{Dyn, DT_DEBUG}; +use elf::program_header::{ProgramHeader, PT_DYNAMIC, PT_PHDR, SIZEOF_PHDR}; +use eyre::{eyre, Result}; +use itertools::Itertools; +use libc::PATH_MAX; +use log::{debug, warn}; +use scroll::Pread; + +use crate::cli::memfault_core_handler::core_reader::read_program_headers; +use crate::cli::memfault_core_handler::elf; +use crate::cli::memfault_core_handler::memory_range::MemoryRange; +use crate::cli::memfault_core_handler::procfs::read_proc_mem; +use crate::cli::memfault_core_handler::r_debug::{LinkMap, RDebug, RDebugIter}; +use crate::cli::memfault_core_handler::ElfPtrSize; + +/// This function attempts to find the ranges of memory in which the dynamic linker has stored the +/// information about the loaded shared objects. This is needed for our backend processing and +/// debuggers like GDB to be able to find and load the symbol files (with debug info) for those +/// loaded shared objects. In more detail, this function does the following: +/// +/// - The function takes the AT_PHDR address from the auxiliary vector as input, the number of +/// program headers via AT_PHNUM, as well as the /proc//mem stream. The AT_PHDR address is +/// the virtual address of the program headers of the main executable. +/// - The function then reads the main executable's program headers, finds the dynamic segment, and +/// reads the DT_DEBUG entry from it. +/// - The DT_DEBUG entry contains the virtual address of the r_debug structure. This is the +/// "rendezvous structure" used by the dynamic linker to communicate details of shared object +/// loading to the debugger. +/// - The r_debug structure contains the head of a linked list of link_map structures. Each link_map +/// represents a loaded shared object. The link_map structure contains the virtual address of the +/// string buffer containing the path to the shared object. The linked list is traversed to the +/// end, collecting the memory regions of the link_map structures and the string buffers along the +/// way. +/// +/// # Arguments +/// * `memory_maps` - The list of the memory regions of the process's memory mappings. This +/// is used to bound the size reads from /proc//mem, when determining the size of path strings. +/// * `output` - Memory regions for all of the aforementioned structures are added to this vector. +pub fn find_dynamic_linker_ranges( + proc_mem_stream: &mut P, + phdr_vaddr: ElfPtrSize, + phdr_num: ElfPtrSize, + memory_maps: &[MemoryRange], + output: &mut Vec, +) -> Result<()> { + debug!( + "Detecting dynamic linker ranges from vaddr 0x{:x}", + phdr_vaddr + ); + + let phdr = read_main_executable_phdr(proc_mem_stream, phdr_vaddr, phdr_num)?; + let main_reloc_addr = calc_relocation_addr(phdr_vaddr, phdr); + + // Add the main executable's program headers. + // Note: find_elf_headers_and_build_id_note_ranges() may also find this range, but that's OK. + output.push(MemoryRange::from_start_and_size( + main_reloc_addr + phdr.p_vaddr, + phdr.p_memsz, + )); + + let main_program_headers = + read_main_exec_program_headers(proc_mem_stream, &phdr, main_reloc_addr)?; + let dynamic_ph = find_dynamic_program_header(&main_program_headers)?; + // Add the dynamic segment contents: + output.push(MemoryRange::from_start_and_size( + main_reloc_addr + dynamic_ph.p_vaddr, + dynamic_ph.p_memsz, + )); + + let r_debug_addr = find_r_debug_addr(proc_mem_stream, main_reloc_addr, dynamic_ph)?; + // Add the r_debug struct itself: + output.push(MemoryRange::from_start_and_size( + r_debug_addr, + size_of::() as ElfPtrSize, + )); + + let mut name_vaddrs: Vec = vec![]; + + RDebugIter::new(proc_mem_stream, r_debug_addr)?.for_each(|(vaddr, link_map)| { + // Add the link_map node itself: + output.push(MemoryRange::from_start_and_size( + vaddr, + size_of::() as ElfPtrSize, + )); + + // Stash the vaddr to the C-string with the path name of the shared object. + // Because the proc_mem_stream is already mutably borrowed, we can't read the string buffer + // here, so we'll do it after. + name_vaddrs.push(link_map.l_name); + }); + + // Add the memory region for each string buffer containing the shared object's path: + name_vaddrs.into_iter().for_each(|name_vaddr| { + output.push(find_c_string_region( + proc_mem_stream, + memory_maps, + name_vaddr, + )); + }); + + Ok(()) +} + +fn find_c_string_region( + proc_mem_stream: &mut P, + memory_maps: &[MemoryRange], + c_string_vaddr: ElfPtrSize, +) -> MemoryRange { + // Read up to PATH_MAX bytes from the given virtual address, or until the end of the memory of + // the memory mapping in which the address resides, whichever is smaller: + let read_size = memory_maps + .iter() + .find(|r| r.contains(c_string_vaddr)) + .map_or(PATH_MAX as ElfPtrSize, |r| r.end - c_string_vaddr) + .min(PATH_MAX as ElfPtrSize); + + read_proc_mem(proc_mem_stream, c_string_vaddr, read_size) + .map(|data| { + data.iter().find_position(|b| **b == 0).map_or_else( + || MemoryRange::from_start_and_size(c_string_vaddr, read_size), + |(idx, _)| { + // +1 for the NUL terminator: + let string_size = (idx + 1).min(read_size as usize); + MemoryRange::from_start_and_size(c_string_vaddr, string_size as ElfPtrSize) + }, + ) + }) + .unwrap_or_else(|e| { + warn!("Failed to read C-string at 0x{:x}: {}", c_string_vaddr, e); + MemoryRange::from_start_and_size(c_string_vaddr, read_size) + }) +} + +fn find_r_debug_addr( + proc_mem_stream: &mut P, + main_reloc_addr: ElfPtrSize, + dynamic_ph: &ProgramHeader, +) -> Result { + let dyn_data = read_proc_mem( + proc_mem_stream, + main_reloc_addr + dynamic_ph.p_vaddr, + dynamic_ph.p_memsz, + ) + .map_err(|e| eyre!("Failed to read dynamic segment: {}", e))?; + + let mut dyn_iter = DynIter::new(dyn_data); + + match find_dt_debug(&mut dyn_iter) { + Some(addr) => Ok(addr), + None => Err(eyre!("Missing DT_DEBUG entry")), + } +} + +/// Finds the virtual address of the r_debug structure in the DT_DEBUG entry, given a dynamic segment iterator. +fn find_dt_debug(dyn_iter: &mut impl Iterator) -> Option { + dyn_iter + .find(|dyn_entry| dyn_entry.d_tag == DT_DEBUG as ElfPtrSize) + .map(|dyn_entry| dyn_entry.d_val) +} + +/// Iterator over the entries in a dynamic segment. +struct DynIter { + data: Vec, + offset: usize, +} + +impl DynIter { + fn new(data: Vec) -> Self { + Self { data, offset: 0 } + } +} +impl Iterator for DynIter { + type Item = Dyn; + + fn next(&mut self) -> Option { + self.data.gread::(&mut self.offset).ok() + } +} + +/// Finds the PT_DYNAMIC program header in the main executable's program headers. +fn find_dynamic_program_header(program_headers: &[ProgramHeader]) -> Result<&ProgramHeader> { + match program_headers.iter().find(|ph| ph.p_type == PT_DYNAMIC) { + Some(ph) => Ok(ph), + None => Err(eyre!("No PT_DYNAMIC found")), + } +} + +/// Reads the main executable's program headers, given the PHDR program header, relocation address, +/// and the /proc//mem stream. +fn read_main_exec_program_headers( + proc_mem_stream: &mut P, + phdr: &ProgramHeader, + main_reloc_addr: ElfPtrSize, +) -> Result> { + // Ignore unnecessary cast here as it is needed on 32-bit systems. + #[allow(clippy::unnecessary_cast)] + proc_mem_stream.seek(SeekFrom::Start((main_reloc_addr + phdr.p_vaddr) as u64))?; + let count = phdr.p_memsz / (SIZEOF_PHDR as ElfPtrSize); + read_program_headers(proc_mem_stream, count as usize) +} + +/// Reads the program header table from the main executable and searches for the `PT_PHDR` +/// header. From the ELF spec: +/// +/// "The array element, if present, specifies the location and size of the program header +/// table itself, both in the file and in the memory image of the program. This segment +/// type may not occur more than once in a file. Moreover, it may occur only if the +/// program header table is part of the memory image of the program. If it is present, +/// it must precede any loadable segment entry." +fn read_main_executable_phdr( + proc_mem_stream: &mut P, + phdr_vaddr: ElfPtrSize, + phdr_num: ElfPtrSize, +) -> Result { + // Ignore unnecessary cast here as it is needed on 32-bit systems. + #[allow(clippy::unnecessary_cast)] + proc_mem_stream.seek(SeekFrom::Start(phdr_vaddr as u64))?; + read_program_headers(proc_mem_stream, phdr_num as usize)? + .into_iter() + .find(|ph| ph.p_type == PT_PHDR) + .ok_or_else(|| eyre!("Main executable PT_PHDR not found")) +} + +/// Calculates the relocation address given the PHDR virtual address and PHDR program header. +fn calc_relocation_addr(phdr_vaddr: ElfPtrSize, phdr: ProgramHeader) -> ElfPtrSize { + phdr_vaddr - phdr.p_vaddr +} + +#[cfg(test)] +mod test { + use std::io::Cursor; + use std::path::PathBuf; + + use insta::assert_debug_snapshot; + use rstest::rstest; + use scroll::{IOwrite, Pwrite}; + + use crate::cli::memfault_core_handler::procfs::ProcMaps; + use crate::cli::memfault_core_handler::test_utils::{FakeProcMaps, FakeProcMem}; + + use super::*; + + #[test] + fn test_phdr_not_first_header() { + let pdyn_header = build_test_program_header(PT_DYNAMIC); + let phdr_header = build_test_program_header(PT_PHDR); + + let mut phdr_bytes = [0; SIZEOF_PHDR * 2]; + phdr_bytes.pwrite::(pdyn_header, 0).unwrap(); + phdr_bytes + .pwrite::(phdr_header, SIZEOF_PHDR) + .unwrap(); + + let mut proc_mem_stream = Cursor::new(phdr_bytes); + let actual_phdr = read_main_executable_phdr(&mut proc_mem_stream, 0, 2).unwrap(); + + assert_eq!(actual_phdr, phdr_header); + } + + #[test] + fn test_find_dynamic_linker_ranges() { + let input_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/cli/memfault_core_handler/fixtures/elf-core-runtime-ld-paths.elf"); + let mut proc_mem_stream = FakeProcMem::new_from_path(&input_path).unwrap(); + + // The coredump fixture contains part of the main ELF executable at offset 0x2000: + // Type Offset VirtAddr PhysAddr FileSiz MemSiz Flg Align + // ... + // LOAD 0x002000 0x00005587ae8bd000 0x0000000000000000 0x001000 0x001000 R 0x1000 + + // The embedded, partial ELF executable has this program table: + // Type Offset VirtAddr PhysAddr FileSiz MemSiz Flg Align + // PHDR 0x000040 0x0000000000000040 0x0000000000000040 0x0002d8 0x0002d8 R 0x8 + let phdr_vaddr = 0x5587ae8bd000 + 0x40; + let phdr_num = 0x02d8 / SIZEOF_PHDR as ElfPtrSize; + + let mut fake_proc_maps = FakeProcMaps::new_from_path(&input_path).unwrap(); + let memory_maps = fake_proc_maps.get_process_maps().unwrap(); + let memory_maps_ranges: Vec = + memory_maps.iter().map(MemoryRange::from).collect(); + + let mut output = vec![]; + find_dynamic_linker_ranges( + &mut proc_mem_stream, + phdr_vaddr, + phdr_num, + &memory_maps_ranges, + &mut output, + ) + .unwrap(); + assert_debug_snapshot!(output); + } + + #[rstest] + // Happy case: + #[case( + b"1hello\0brave\0new\0world!!\0", + vec![MemoryRange::from_start_and_size(0, 25)], + 1, + MemoryRange::from_start_and_size(1, 6), // hello\0 -> 6 bytes + )] + // Cannot find NUL terminator, clips to end of memory mapping region: + #[case( + b"1hello", + vec![MemoryRange::from_start_and_size(0, 6)], + 1, + MemoryRange::from_start_and_size(1, 5), + )] + // Clips to the end of memory mapping regions: + #[case( + b"1hello\0brave\0new\0world!!\0", + vec![MemoryRange::new(0, 4)], + 1, + MemoryRange::new(1, 4), + )] + // Falls back to PATH_MAX if no memory mapping region is found: + #[case( + b"1hello\0", + vec![], + 1, + MemoryRange::from_start_and_size(1, PATH_MAX as ElfPtrSize), + )] + // Clips to PATH_MAX if memory mapping region is longer than PATH_MAX and NUL is not found: + #[case( + &[b'A'; PATH_MAX as usize + 1], + vec![MemoryRange::from_start_and_size(0, PATH_MAX as ElfPtrSize + 1)], + 0, + MemoryRange::from_start_and_size(0, PATH_MAX as ElfPtrSize), + )] + fn test_find_c_string_region( + #[case] proc_mem: &[u8], + #[case] mmap_regions: Vec, + #[case] c_string_vaddr: ElfPtrSize, + #[case] expected: MemoryRange, + ) { + let mut proc_mem_stream = Cursor::new(proc_mem); + assert_eq!( + find_c_string_region(&mut proc_mem_stream, &mmap_regions, c_string_vaddr), + expected + ); + } + + #[rstest] + // Empty + #[case(vec![], vec![])] + // Some items + #[case( + vec![1, 2, 3, 4], + vec![ + Dyn { d_tag: 1, d_val: 2 }, + Dyn { d_tag: 3, d_val: 4 } + ], + )] + // Partial item at the end + #[case( + vec![1, 2, 3], + vec![ + Dyn { d_tag: 1, d_val: 2 }, + ], + )] + fn test_dyn_iter(#[case] input: Vec, #[case] expected: Vec) { + let data = make_dyn_fixture(input); + assert_eq!(DynIter::new(data).collect::>(), expected); + } + + fn make_dyn_fixture(values: Vec) -> Vec { + let mut cursor = Cursor::new(vec![]); + for value in values { + cursor.iowrite::(value).unwrap(); + } + cursor.into_inner() + } + + fn build_test_program_header(p_type: u32) -> ProgramHeader { + ProgramHeader { + p_type, + p_flags: 1, + p_offset: 2, + p_vaddr: 3, + p_paddr: 4, + p_filesz: 5, + p_memsz: 6, + p_align: 0, + } + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/find_elf_headers.rs b/memfaultd/src/cli/memfault_core_handler/find_elf_headers.rs new file mode 100644 index 0000000..6e95953 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/find_elf_headers.rs @@ -0,0 +1,168 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{Read, Seek}; + +use elf::program_header::{ProgramHeader, PT_NOTE}; +use eyre::Result; +use log::debug; + +use crate::cli::memfault_core_handler::core_elf_note::{iterate_elf_notes, ElfNote}; +use crate::cli::memfault_core_handler::core_reader::{CoreReader, CoreReaderImpl}; +use crate::cli::memfault_core_handler::memory_range::MemoryRange; +use crate::cli::memfault_core_handler::{elf, ElfPtrSize}; + +/// Detects whether the supplied stream is at an ELF file and if so, returns the ranges in the +/// stream, that contain the ELF header, program headers and build id note. The ranges are +/// relative to the stream's position when calling the function. +/// GDB requires the headers and note to be present, for debuginfod and build id based symbol file +/// lookups to work. +pub fn find_elf_headers_and_build_id_note_ranges( + vaddr_base: ElfPtrSize, + stream: &mut P, +) -> Result> { + debug!( + "Detecting ELF headers and build ID note ranges from vaddr 0x{:x}", + vaddr_base + ); + + let mut elf_reader = CoreReaderImpl::new(stream)?; + let elf_header = elf_reader.elf_header(); + let program_headers = elf_reader.read_program_headers()?; + let build_id_note_ph = program_headers + .iter() + .find(|ph| ph.p_type == PT_NOTE && contains_gnu_build_id_note(&mut elf_reader, ph)); + + match build_id_note_ph { + Some(build_id_note_ph) => { + let ranges = [ + // ELF header: + MemoryRange::from_start_and_size(vaddr_base, elf_header.e_ehsize as ElfPtrSize), + // Program header table: + MemoryRange::from_start_and_size( + vaddr_base + elf_header.e_phoff, + (elf_header.e_phentsize as ElfPtrSize) * (elf_header.e_phnum as ElfPtrSize), + ), + // Note segment containing GNU Build ID: + MemoryRange::from_start_and_size( + vaddr_base + build_id_note_ph.p_offset, + build_id_note_ph.p_filesz, + ), + ]; + + // FIXME: MFLT-11635 CoreElf .py requires ELF header + build ID in single segment + Ok(vec![MemoryRange::new( + ranges.iter().map(|r| r.start).min().unwrap(), + ranges.iter().map(|r| r.end).max().unwrap(), + )]) + } + None => Err(eyre::eyre!("Build ID note missing")), + } +} + +fn contains_gnu_build_id_note( + elf_reader: &mut CoreReaderImpl<&mut P>, + program_header: &ProgramHeader, +) -> bool { + match elf_reader.read_segment_data(program_header) { + Ok(data) => iterate_elf_notes(&data).any(|note| matches!(note, ElfNote::GnuBuildId(_))), + _ => false, + } +} + +#[cfg(test)] +mod test { + use std::fs::File; + use std::io::{Cursor, SeekFrom}; + use std::path::PathBuf; + + use super::*; + + #[test] + fn test_err_result_if_no_elf() { + let mut stream = Cursor::new(vec![0u8; 100]); + assert_eq!( + find_elf_headers_and_build_id_note_ranges(0, &mut stream) + .unwrap_err() + .to_string(), + "Invalid ELF header" + ); + } + + #[test] + fn test_err_result_if_missing_build_id() { + // A core.elf itself does not have a build id note: + let mut stream = core_elf_stream(); + assert_eq!( + find_elf_headers_and_build_id_note_ranges(0, &mut stream) + .unwrap_err() + .to_string(), + "Build ID note missing" + ); + } + + #[test] + #[ignore = "FIXME: MFLT-11635 CoreElf .py requires ELF header + build ID in single segment"] + fn test_ok_result_with_list_of_ranges() { + let mut stream = exe_elf_stream(); + assert_eq!( + find_elf_headers_and_build_id_note_ranges(0x1000, &mut stream).unwrap(), + // Note: see offsets/sizes from the readelf -Wl output in exe_elf_stream(). + vec![ + // ELF header + MemoryRange::from_start_and_size(0x1000, 0x40), + // Program header table: + MemoryRange::from_start_and_size(0x1000 + 0x40, 0x2d8), + // Note segment with GNU Build ID + MemoryRange::from_start_and_size(0x1000 + 0x358, 0x44), + ] + ); + } + + #[test] + fn test_ok_result_with_list_of_ranges_mflt_11635_work_around() { + let mut stream = exe_elf_stream(); + assert_eq!( + find_elf_headers_and_build_id_note_ranges(0x1000, &mut stream).unwrap(), + // Note: see offsets/sizes from the readelf -Wl output in exe_elf_stream(). + + // FIXME: MFLT-11635 CoreElf .py requires ELF header + build ID in single segment + vec![MemoryRange::new(0x1000, 0x1000 + 0x358 + 0x44),] + ); + } + + fn core_elf_stream() -> File { + File::open(core_elf_path()).unwrap() + } + + fn exe_elf_stream() -> File { + let mut stream = core_elf_stream(); + // The coredump fixture contains part of a captured ELF executable at offset 0x2000: + // Type Offset VirtAddr PhysAddr FileSiz MemSiz Flg Align + // ... + // LOAD 0x002000 0x00005587ae8bd000 0x0000000000000000 0x001000 0x001000 R 0x1000 + + // The embedded, partial ELF executable has this program table: + // Type Offset VirtAddr PhysAddr FileSiz MemSiz Flg Align + // PHDR 0x000040 0x0000000000000040 0x0000000000000040 0x0002d8 0x0002d8 R 0x8 + // INTERP 0x000318 0x0000000000000318 0x0000000000000318 0x00001c 0x00001c R 0x1 + // LOAD 0x000000 0x0000000000000000 0x0000000000000000 0x000648 0x000648 R 0x1000 + // LOAD 0x001000 0x0000000000001000 0x0000000000001000 0x000199 0x000199 R E 0x1000 + // LOAD 0x002000 0x0000000000002000 0x0000000000002000 0x0000e4 0x0000e4 R 0x1000 + // LOAD 0x002db0 0x0000000000003db0 0x0000000000003db0 0x000268 0x000270 RW 0x1000 + // DYNAMIC 0x002dc0 0x0000000000003dc0 0x0000000000003dc0 0x000200 0x000200 RW 0x8 + // NOTE 0x000338 0x0000000000000338 0x0000000000000338 0x000020 0x000020 R 0x8 + // NOTE 0x000358 0x0000000000000358 0x0000000000000358 0x000044 0x000044 R 0x4 + // GNU_PROPERTY 0x000338 0x0000000000000338 0x0000000000000338 0x000020 0x000020 R 0x8 + // GNU_EH_FRAME 0x002008 0x0000000000002008 0x0000000000002008 0x00002c 0x00002c R 0x4 + // GNU_STACK 0x000000 0x0000000000000000 0x0000000000000000 0x000000 0x000000 RW 0x10 + // GNU_RELRO 0x002db0 0x0000000000003db0 0x0000000000003db0 0x000250 0x000250 R 0x1 + stream.seek(SeekFrom::Start(0x2000)).unwrap(); + stream + } + + fn core_elf_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/cli/memfault_core_handler/fixtures/elf-core-runtime-ld-paths.elf") + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/find_stack.rs b/memfaultd/src/cli/memfault_core_handler/find_stack.rs new file mode 100644 index 0000000..1cf5c18 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/find_stack.rs @@ -0,0 +1,133 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::cmp::min; + +use crate::cli::memfault_core_handler::arch::{get_stack_pointer, ElfGRegSet}; +use crate::cli::memfault_core_handler::memory_range::MemoryRange; +use crate::cli::memfault_core_handler::ElfPtrSize; + +use psm::StackDirection; + +/// Attempts to find a MemoryRange for the stack based on the supplied register set. +/// The returned range is bound by the max_thread_size and the end of the segment in which +/// the stack is found. The assumption herein is that an anonymous memory mapping is created as +/// stack (and used exclusively as such). If the stack pointer is not contained in any segment, +/// None is returned. +pub fn find_stack( + regs: &ElfGRegSet, + mapped_memory_ranges: &[MemoryRange], + max_thread_size: usize, +) -> Option { + let stack_pointer = get_stack_pointer(regs) as ElfPtrSize; + let stack_direction = StackDirection::new(); + + find_stack_inner( + stack_pointer, + mapped_memory_ranges, + max_thread_size, + stack_direction, + ) +} + +fn find_stack_inner( + stack_pointer: ElfPtrSize, + mapped_memory_ranges: &[MemoryRange], + max_thread_size: usize, + stack_direction: StackDirection, +) -> Option { + // Iterate over all PT_LOAD segments and find the one that contains the stack + // pointer. If the stack pointer is not contained in any PT_LOAD segment, we + // ignore the thread. + // + // NOTE: This is an M*N operation, but both M(#segments) and N(#threads) are + // likely quite small so this should be fine. + for memory_range in mapped_memory_ranges { + if memory_range.contains(stack_pointer as ElfPtrSize) { + let (stack_start, stack_end) = match stack_direction { + StackDirection::Ascending => { + let stack_size = min( + stack_pointer.saturating_sub(memory_range.start), + max_thread_size as ElfPtrSize, + ); + + let stack_base = stack_pointer - stack_size; + (stack_base, stack_pointer) + } + StackDirection::Descending => { + let stack_size = min( + memory_range.end.saturating_sub(stack_pointer), + max_thread_size as ElfPtrSize, + ); + + let stack_base = stack_pointer + stack_size; + (stack_pointer, stack_base) + } + }; + + return Some(MemoryRange::new(stack_start, stack_end)); + } + } + + None +} + +#[cfg(test)] +mod test { + use super::*; + + use rstest::rstest; + + #[rstest] + #[case::stack_ascending( + StackDirection::Ascending, + 0x1500, + MemoryRange::new(0x500, 0x1500), + 0x1000 + )] + #[case::stack_descending( + StackDirection::Descending, + 0x1500, + MemoryRange::new(0x1500, 0x2500), + 0x1000 + )] + fn test_stack_calculation( + #[case] stack_direction: StackDirection, + #[case] stack_pointer: ElfPtrSize, + #[case] expected_stack: MemoryRange, + #[case] max_thread_size: usize, + ) { + let mapped_regions = program_header_fixture(); + + let stack = find_stack_inner( + stack_pointer, + &mapped_regions, + max_thread_size, + stack_direction, + ); + assert_eq!(stack, Some(expected_stack)); + } + + #[rstest] + #[case::below_regions(0x0050)] + #[case::between_regions(0x0400)] + #[case::above_regions(0x3000)] + fn test_stack_not_found(#[case] stack_pointer: ElfPtrSize) { + let mapped_ranges = program_header_fixture(); + + let stack = find_stack_inner( + stack_pointer, + &mapped_ranges, + 0x1000, + StackDirection::Ascending, + ); + assert!(stack.is_none()); + } + + fn program_header_fixture() -> Vec { + vec![ + MemoryRange::from_start_and_size(0x0100, 0x0250), + MemoryRange::from_start_and_size(0x0500, 0x2500), + ] + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/fixtures/elf-core-runtime-ld-paths.elf b/memfaultd/src/cli/memfault_core_handler/fixtures/elf-core-runtime-ld-paths.elf new file mode 100644 index 0000000..4b6d6cb Binary files /dev/null and b/memfaultd/src/cli/memfault_core_handler/fixtures/elf-core-runtime-ld-paths.elf differ diff --git a/memfaultd/src/cli/memfault_core_handler/fixtures/sample_note.bin b/memfaultd/src/cli/memfault_core_handler/fixtures/sample_note.bin new file mode 100644 index 0000000..f14c2c1 Binary files /dev/null and b/memfaultd/src/cli/memfault_core_handler/fixtures/sample_note.bin differ diff --git a/memfaultd/src/cli/memfault_core_handler/log_wrapper.rs b/memfaultd/src/cli/memfault_core_handler/log_wrapper.rs new file mode 100644 index 0000000..33d86c6 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/log_wrapper.rs @@ -0,0 +1,123 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Logger wrapper to capture logs that happen during coredump capture. +//! +//! These logs are sent to a channel that is read by the `CoreElfTransformer` and written to a note +//! in the coredump. + +use std::sync::mpsc::SyncSender; + +use log::{LevelFilter, Log, Metadata, Record}; + +pub const CAPTURE_LOG_CHANNEL_SIZE: usize = 128; +pub const CAPTURE_LOG_MAX_LEVEL: LevelFilter = LevelFilter::Debug; + +/// Logger wrapper to capture all error and warning logs that happen during coredump capture. +pub struct CoreHandlerLogWrapper { + log: Box, + capture_logs_tx: SyncSender, + level: LevelFilter, +} + +impl CoreHandlerLogWrapper { + pub fn new(log: Box, capture_logs_tx: SyncSender, level: LevelFilter) -> Self { + Self { + log, + capture_logs_tx, + level, + } + } +} + +impl Log for CoreHandlerLogWrapper { + fn enabled(&self, metadata: &Metadata) -> bool { + self.log.enabled(metadata) + } + + fn log(&self, record: &Record) { + if record.level() <= CAPTURE_LOG_MAX_LEVEL { + let entry = build_log_string(record); + + // Errors are ignored here because the only options are to panic or log the error. + // Panicking is not a great option because this isn't critical functionality. Logging + // the error isn't an option because we'd risk infinite recursion since we're already + // inside the logger. + let _ = self.capture_logs_tx.try_send(entry); + } + + if record.level() <= self.level { + self.log.log(record) + } + } + + fn flush(&self) { + self.log.flush() + } +} + +/// Build a log string from a log record. +/// +/// The log string is formatted as follows: +/// +/// ```text +/// : - +/// ``` +fn build_log_string(record: &Record) -> String { + match record.line() { + Some(line) => format!( + "{} {}:{} - {}", + record.level(), + record.target(), + line, + record.args() + ), + None => format!("{} {} - {}", record.level(), record.target(), record.args()), + } +} + +#[cfg(test)] +mod test { + use super::*; + + use std::sync::mpsc::sync_channel; + + use insta::assert_json_snapshot; + use log::Level; + + #[test] + fn test_log_saving() { + let mut logger = stderrlog::new(); + logger.module("memfaultd").verbosity(10); + + let (capture_logs_tx, capture_logs_rx) = sync_channel(5); + let wrapper = + CoreHandlerLogWrapper::new(Box::new(logger), capture_logs_tx, CAPTURE_LOG_MAX_LEVEL); + + let error_record = build_log_record(Level::Error); + let warn_record = build_log_record(Level::Warn); + let info_record = build_log_record(Level::Info); + let debug_record = build_log_record(Level::Debug); + let trace_record = build_log_record(Level::Trace); + + wrapper.log(&error_record); + wrapper.log(&warn_record); + wrapper.log(&info_record); + wrapper.log(&debug_record); + wrapper.log(&trace_record); + + let errors: Vec = capture_logs_rx.try_iter().collect(); + assert_json_snapshot!(errors); + } + + fn build_log_record(level: Level) -> Record<'static> { + Record::builder() + .args(format_args!("Test message")) + .level(level) + .target("test") + .file(Some("log_wrapper.rs")) + .line(Some(71)) + .module_path(Some("core_handler")) + .build() + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/memory_range.rs b/memfaultd/src/cli/memfault_core_handler/memory_range.rs new file mode 100644 index 0000000..c6a57a5 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/memory_range.rs @@ -0,0 +1,127 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::cli::memfault_core_handler::ElfPtrSize; +use std::cmp::max; +use std::fmt::{Debug, Formatter}; + +/// Convenience struct to manage memory address ranges +#[derive(PartialEq, Eq, Clone)] +pub struct MemoryRange { + pub start: ElfPtrSize, + pub end: ElfPtrSize, +} + +impl MemoryRange { + pub fn new(start: ElfPtrSize, end: ElfPtrSize) -> Self { + Self { start, end } + } + + pub fn from_start_and_size(start: ElfPtrSize, size: ElfPtrSize) -> Self { + Self { + start, + end: start + size, + } + } + + /// Returns true if the two ranges overlap + pub fn overlaps(&self, other: &Self) -> bool { + self.start <= other.end && self.end > other.start + } + + pub fn size(&self) -> ElfPtrSize { + self.end - self.start + } + + pub fn contains(&self, addr: ElfPtrSize) -> bool { + self.start <= addr && addr < self.end + } +} + +impl Debug for MemoryRange { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MemoryRange") + .field("start", &format!("{:#x}", self.start)) + .field("end", &format!("{:#x}", self.end)) + .finish() + } +} + +/// Merge overlapping memory ranges. +/// +/// This is used to merge memory ranges before turning them into PT_LOAD program +/// headers. +pub fn merge_memory_ranges(mut ranges: Vec) -> Vec { + // First, sort by start address. This lets us merge overlapping ranges in a single pass + // by knowing that we only need to check the last range in the merged list. + ranges.sort_by_key(|r| r.start); + + // Next, iterate over the sorted ranges and merge overlapping ranges. If the current range + // overlaps with the last range in the merged list, we extend the last range to include the + // current range. Otherwise, we add the current range to the merged list. + let mut merged_ranges: Vec = Vec::new(); + for range in ranges { + if let Some(last) = merged_ranges.last_mut() { + if last.overlaps(&range) { + last.end = max(last.end, range.end); + continue; + } + } + merged_ranges.push(range); + } + + merged_ranges +} + +#[cfg(test)] +mod test { + use rstest::rstest; + + use super::*; + + #[rstest] + // Two ranges with matching boundaries + #[case( + vec![MemoryRange::new(0x1000, 0x2000), MemoryRange::new(0x2000, 0x3000)], + vec![MemoryRange::new(0x1000, 0x2000), MemoryRange::new(0x2000, 0x3000)], + )] + // Two ranges with overlapping boundaries + #[case( + vec![MemoryRange::new(0x1000, 0x2000), MemoryRange::new(0x1500, 0x3000)], + vec![MemoryRange::new(0x1000, 0x3000)], + )] + // Two ranges with non-overlapping boundaries + #[case( + vec![MemoryRange::new(0x1000, 0x2000), MemoryRange::new(0x3000, 0x4000)], + vec![MemoryRange::new(0x1000, 0x2000), MemoryRange::new(0x3000, 0x4000)], + )] + // Three overlapping regions, unsorted + #[case( + vec![ + MemoryRange::new(0x1500, 0x3000), + MemoryRange::new(0x1000, 0x2000), + MemoryRange::new(0x3000, 0x5000), + ], + vec![MemoryRange::new(0x1000, 0x3000), MemoryRange::new(0x3000, 0x5000)] + )] + fn test_memory_range_merge( + #[case] input: Vec, + #[case] expected: Vec, + ) { + let merged = merge_memory_ranges(input); + assert_eq!(merged, expected); + } + + #[rstest] + #[case(MemoryRange::new(0x1000, 0x2000), 0x1000, true)] + #[case(MemoryRange::new(0x1000, 0x2000), 0x2000, false)] + #[case(MemoryRange::new(0x1000, 0x2000), 0x0fff, false)] + #[case(MemoryRange::new(0x1000, 0x2000), 0x2001, false)] + fn test_memory_range_contains( + #[case] input_range: MemoryRange, + #[case] addr: ElfPtrSize, + #[case] contains: bool, + ) { + assert_eq!(input_range.contains(addr), contains); + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/mod.rs b/memfaultd/src/cli/memfault_core_handler/mod.rs new file mode 100644 index 0000000..d4425df --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/mod.rs @@ -0,0 +1,306 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +mod arch; +mod auxv; +mod core_elf_memfault_note; +mod core_elf_note; +mod core_reader; +mod core_transformer; +mod core_writer; +mod find_dynamic; +mod find_elf_headers; +mod find_stack; +mod log_wrapper; +mod memory_range; +mod procfs; +mod r_debug; +#[cfg(test)] +mod test_utils; + +use self::log_wrapper::CoreHandlerLogWrapper; +use self::procfs::{proc_mem_stream, read_proc_cmdline, ProcMapsImpl}; +use self::{arch::coredump_thread_filter_supported, log_wrapper::CAPTURE_LOG_CHANNEL_SIZE}; +use self::{core_elf_memfault_note::CoredumpMetadata, core_transformer::CoreTransformerOptions}; +use self::{core_reader::CoreReaderImpl, core_transformer::CoreTransformerLogFetcher}; +use self::{core_writer::CoreWriterImpl, log_wrapper::CAPTURE_LOG_MAX_LEVEL}; +use crate::cli; +use crate::config::{Config, CoredumpCompression}; +use crate::mar::manifest::{CompressionAlgorithm, Metadata}; +use crate::mar::mar_entry_builder::MarEntryBuilder; +use crate::network::NetworkConfig; +use crate::util::disk_size::get_disk_space; +use crate::util::io::{ForwardOnlySeeker, StreamPositionTracker}; +use crate::util::persistent_rate_limiter::PersistentRateLimiter; +use argh::FromArgs; +use eyre::{eyre, Result, WrapErr}; +use flate2::write::GzEncoder; +use kernlog::KernelLog; +use log::{debug, error, info, warn, LevelFilter, Log}; +use prctl::set_dumpable; +use std::io::BufWriter; +use std::path::Path; +use std::thread::scope; +use std::{cmp::max, io::BufReader}; +use std::{cmp::min, fs::File}; +use std::{ + env::{set_var, var}, + sync::mpsc::SyncSender, +}; +use std::{io::Write, sync::mpsc::sync_channel}; +use uuid::Uuid; + +#[cfg(target_pointer_width = "64")] +pub use goblin::elf64 as elf; + +#[cfg(target_pointer_width = "64")] +pub type ElfPtrSize = u64; + +#[cfg(target_pointer_width = "32")] +pub use goblin::elf32 as elf; + +use super::MemfaultdClient; + +#[cfg(target_pointer_width = "32")] +pub type ElfPtrSize = u32; + +#[derive(FromArgs)] +/// Accepts a kernel-generated core.elf from stdin and processes it. +/// This is intended to be called by the kernel as the coredump handler. It is not intended to be +/// called by users directly. memfaultd is expected to set up the handler with the kernel by writing +/// the appropriate configuration to /proc/sys/kernel/core_pattern. +/// See https://mflt.io/linux-coredumps for more information. +struct MemfaultCoreHandlerArgs { + /// use configuration file + #[argh(option, short = 'c')] + config_file: Option, + + #[argh(positional)] + pid: i32, + + /// verbose output + #[argh(switch, short = 'V')] + verbose: bool, +} + +pub fn main() -> Result<()> { + // Disable coredumping of this process + let dumpable_result = set_dumpable(false); + + let args: MemfaultCoreHandlerArgs = argh::from_env(); + + let (capture_logs_tx, capture_logs_rx) = sync_channel(CAPTURE_LOG_CHANNEL_SIZE); + let log_level = if args.verbose { + LevelFilter::Trace + } else { + LevelFilter::Info + }; + // When the kernel executes a core dump handler, the stdout/stderr go nowhere. + // Let's log to the kernel log to aid debugging: + init_kernel_logger(log_level, capture_logs_tx); + + if let Err(e) = dumpable_result { + warn!("Failed to set dumpable: {}", e); + }; + + let config_path = args.config_file.as_ref().map(Path::new); + let config = + Config::read_from_system(config_path).wrap_err(eyre!("Unable to load configuration"))?; + + if !config.config_file.enable_data_collection { + error!("Data collection disabled, not processing corefile"); + return Ok(()); + } + + let (app_logs_tx, app_logs_rx) = sync_channel(1); + let log_fetcher = CoreTransformerLogFetcher::new(capture_logs_rx, app_logs_rx); + + // Asynchronously notify memfaultd that a crash occurred and fetch any crash logs. + scope(|s| { + s.spawn(|| { + let client = MemfaultdClient::from_config(&config); + match client { + Ok(client) => { + if let Err(e) = client.notify_crash() { + debug!("Failed to notify memfaultd of crash: {:?}", e); + } + + debug!("Getting crash logs"); + match client.get_crash_logs() { + Ok(logs) => { + if let Err(e) = app_logs_tx.send(logs) { + debug!("Application logs channel rx already dropped: {:?}", e); + } + } + Err(e) => { + debug!("Failed to get crash logs: {:?}", e); + } + } + } + Err(e) => { + debug!("Failed to create memfaultd client: {:?}", e); + } + } + }); + process_corefile(&config, args.pid, log_fetcher) + .wrap_err(format!("Error processing coredump for PID {}", args.pid)) + }) +} + +pub fn process_corefile( + config: &Config, + pid: i32, + log_fetcher: CoreTransformerLogFetcher, +) -> Result<()> { + let rate_limiter = if !config.config_file.enable_dev_mode { + config.coredump_rate_limiter_file_path(); + let mut rate_limiter = PersistentRateLimiter::load( + config.coredump_rate_limiter_file_path(), + config.config_file.coredump.rate_limit_count, + chrono::Duration::from_std(config.config_file.coredump.rate_limit_duration)?, + ) + .with_context(|| { + format!( + "Unable to open coredump rate limiter {}", + config.coredump_rate_limiter_file_path().display() + ) + })?; + if !rate_limiter.check() { + info!("Coredumps limit reached, not processing corefile"); + return Ok(()); + } + Some(rate_limiter) + } else { + None + }; + + let max_size = calculate_available_space(config)?; + if max_size == 0 { + error!("Not processing corefile, disk usage limits exceeded"); + return Ok(()); + } + + let mar_staging_path = config.mar_staging_path(); + let mar_builder = MarEntryBuilder::new(&mar_staging_path)?; + let compression = config.config_file.coredump.compression; + let capture_strategy = config.config_file.coredump.capture_strategy; + let output_file_name = generate_tmp_file_name(compression); + let output_file_path = mar_builder.make_attachment_path_in_entry_dir(&output_file_name); + + let cmd_line_file_name = format!("/proc/{}/cmdline", pid); + let mut cmd_line_file = File::open(cmd_line_file_name)?; + let cmd_line = read_proc_cmdline(&mut cmd_line_file)?; + let metadata = CoredumpMetadata::new(config, cmd_line); + let thread_filter_supported = coredump_thread_filter_supported(); + let transformer_options = CoreTransformerOptions { + max_size, + capture_strategy, + thread_filter_supported, + }; + + let output_file = BufWriter::new(File::create(&output_file_path)?); + let output_stream: Box = match compression { + CoredumpCompression::Gzip => { + Box::new(GzEncoder::new(output_file, flate2::Compression::default())) + } + CoredumpCompression::None => Box::new(output_file), + }; + let output_stream = StreamPositionTracker::new(output_stream); + + let input_stream = ForwardOnlySeeker::new(BufReader::new(std::io::stdin())); + let proc_maps = ProcMapsImpl::new(pid); + let core_reader = CoreReaderImpl::new(input_stream)?; + let core_writer = CoreWriterImpl::new( + core_reader.elf_header(), + output_stream, + proc_mem_stream(pid)?, + ); + let mut core_transformer = core_transformer::CoreTransformer::new( + core_reader, + core_writer, + proc_mem_stream(pid)?, + transformer_options, + metadata, + proc_maps, + log_fetcher, + )?; + + match core_transformer.run_transformer() { + Ok(()) => { + info!("Successfully captured coredump"); + let network_config = NetworkConfig::from(config); + let mar_entry = mar_builder + .set_metadata(Metadata::new_coredump(output_file_name, compression.into())) + .add_attachment(output_file_path) + .save(&network_config)?; + + debug!("Coredump MAR entry generated: {}", mar_entry.path.display()); + + if let Some(rate_limiter) = rate_limiter { + rate_limiter.save()?; + } + + Ok(()) + } + Err(e) => Err(eyre!("Failed to capture coredump: {}", e)), + } +} + +fn generate_tmp_file_name(compression: CoredumpCompression) -> String { + let id = Uuid::new_v4(); + let extension = match compression { + CoredumpCompression::Gzip => "elf.gz", + CoredumpCompression::None => "elf", + }; + format!("core-{}.{}", id, extension) +} + +fn calculate_available_space(config: &Config) -> Result { + let min_headroom = config.tmp_dir_min_headroom(); + let available = get_disk_space(&config.tmp_dir())?; + let has_headroom = available.exceeds(&min_headroom); + if !has_headroom { + return Ok(0); + } + Ok(min( + (available.bytes - min_headroom.bytes) as usize, + config.config_file.coredump.coredump_max_size, + )) +} + +impl From for CompressionAlgorithm { + fn from(compression: CoredumpCompression) -> Self { + match compression { + CoredumpCompression::Gzip => CompressionAlgorithm::Gzip, + CoredumpCompression::None => CompressionAlgorithm::None, + } + } +} + +fn init_kernel_logger(level: LevelFilter, capture_logs_tx: SyncSender) { + // kernlog::init() reads from the KERNLOG_LEVEL to set the level. There's no public interface + // to set it otherwise, so: if this environment variable is not set, set it according to the + // --verbose flag: + if var("KERNLOG_LEVEL").is_err() { + set_var("KERNLOG_LEVEL", level.as_str()); + } + // We fallback to standard output if verbose mode is enabled or if kernel is not available. + + let logger: Box = match KernelLog::from_env() { + Ok(logger) => Box::new(logger), + Err(_) => Box::new(cli::build_logger(level)), + }; + + let logger = Box::new(CoreHandlerLogWrapper::new( + logger, + capture_logs_tx, + CAPTURE_LOG_MAX_LEVEL, + )); + log::set_boxed_logger(logger).unwrap(); + + // Set the max log level to the max of the log level and the capture log level. + // This is necessary because the log macros will completely disable calls if the level + // is below the max level. + let max_level = max(level, CAPTURE_LOG_MAX_LEVEL); + log::set_max_level(max_level); +} diff --git a/memfaultd/src/cli/memfault_core_handler/procfs.rs b/memfaultd/src/cli/memfault_core_handler/procfs.rs new file mode 100644 index 0000000..e697f91 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/procfs.rs @@ -0,0 +1,79 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::fs::File; +use std::io::{Read, Seek, SeekFrom}; + +use eyre::{eyre, Result}; +use procfs::process::{MemoryMap, MemoryMaps}; + +use crate::cli::memfault_core_handler::memory_range::MemoryRange; +use crate::cli::memfault_core_handler::ElfPtrSize; + +/// Opens /proc//mem for reading. +pub fn proc_mem_stream(pid: i32) -> Result { + let proc_mem_stream = File::open(format!("/proc/{}/mem", pid))?; + Ok(proc_mem_stream) +} + +/// Reads memory from /proc//mem into a buffer. +pub fn read_proc_mem( + proc_mem_stream: &mut P, + vaddr: ElfPtrSize, + size: ElfPtrSize, +) -> Result> { + // Ignore unnecessary cast here as it is needed on 32-bit systems. + #[allow(clippy::unnecessary_cast)] + proc_mem_stream.seek(SeekFrom::Start(vaddr as u64))?; + let mut buf = vec![0; size as usize]; + proc_mem_stream.read_exact(&mut buf)?; + Ok(buf) +} + +pub fn read_proc_cmdline(cmd_line_stream: &mut P) -> Result { + let mut cmd_line_buf = Vec::new(); + cmd_line_stream.read_to_end(&mut cmd_line_buf)?; + + Ok(String::from_utf8_lossy(&cmd_line_buf).into_owned()) +} + +/// Wrapper trait for reading /proc//maps. +/// +/// Provides a useful abstraction that can be mocked out for testing. +pub trait ProcMaps { + fn get_process_maps(&mut self) -> Result>; +} + +/// Direct implementation of ProcMaps that reads from /proc//maps file. +/// +/// This is the default implementation used in production. It simply reads directly from +/// the file and returns the parsed memory ranges. +#[derive(Debug)] +pub struct ProcMapsImpl { + pid: i32, +} + +impl ProcMapsImpl { + pub fn new(pid: i32) -> Self { + Self { pid } + } +} + +impl ProcMaps for ProcMapsImpl { + fn get_process_maps(&mut self) -> Result> { + let maps_file_name = format!("/proc/{}/maps", self.pid); + + Ok(MemoryMaps::from_path(maps_file_name) + .map_err(|e| eyre!("Failed to read /proc/{}/maps: {}", self.pid, e))? + .memory_maps) + } +} + +impl From<&MemoryMap> for MemoryRange { + fn from(m: &MemoryMap) -> Self { + MemoryRange { + start: m.address.0 as ElfPtrSize, + end: m.address.1 as ElfPtrSize, + } + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/r_debug.rs b/memfaultd/src/cli/memfault_core_handler/r_debug.rs new file mode 100644 index 0000000..cb0442b --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/r_debug.rs @@ -0,0 +1,97 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{Read, Seek, SeekFrom}; + +use eyre::{eyre, Result}; + +use crate::cli::memfault_core_handler::ElfPtrSize; +use crate::util::mem::AsBytes; + +/// "Rendezvous structures used by the run-time dynamic linker to +/// communicate details of shared object loading to the debugger." +/// See glibc's elf/link.h +/// https://sourceware.org/git/?p=glibc.git;a=blob;f=elf/link.h;h=3b5954d9818e8ea9f35638c55961f861f6ae6057 + +// TODO: MFLT-11643 Add support for r_debug_extended + +/// The r_debug C structure from elf/link.h +#[derive(Debug, Clone, Copy, Default)] +#[repr(C)] +pub struct RDebug { + pub version: u32, + pub r_map: ElfPtrSize, + pub r_brk: ElfPtrSize, + pub r_state: u32, + pub r_ldbase: ElfPtrSize, +} + +/// The link_map C structure from elf/link.h +#[derive(Debug, Clone, Copy, Default)] +#[repr(C)] +pub struct LinkMap { + pub l_addr: ElfPtrSize, + /// Pointer to C-string. + pub l_name: ElfPtrSize, + pub l_ld: ElfPtrSize, + /// Pointer to next link map. + pub l_next: ElfPtrSize, + pub l_prev: ElfPtrSize, +} + +pub struct RDebugIter<'a, P: Read + Seek> { + proc_mem_stream: &'a mut P, + l_next: ElfPtrSize, +} + +impl<'a, P: Read + Seek> RDebugIter<'a, P> { + pub fn new(proc_mem_stream: &'a mut P, r_debug_addr: ElfPtrSize) -> Result { + // Ignore unnecessary cast here as it is needed on 32-bit systems. + #[allow(clippy::unnecessary_cast)] + proc_mem_stream.seek(SeekFrom::Start(r_debug_addr as u64))?; + let mut r_debug = RDebug::default(); + // SAFETY: From the point of view of this program, + // RDebug only contains scalar values where any value is allowed. + let data = unsafe { r_debug.as_mut_bytes() }; + proc_mem_stream + .read_exact(data) + .map_err(|e| eyre!("Failed to read r_debug: {}", e))?; + Ok(Self { + proc_mem_stream, + l_next: r_debug.r_map, + }) + } + + /// Returns the next tuple of the link map's virtual address and the link map itself, + /// or None if the end of the linked list has been reached. + fn read_next(&mut self) -> Result<(ElfPtrSize, LinkMap)> { + let vaddr = self.l_next; + // Ignore unnecessary cast here as it is needed on 32-bit systems. + #[allow(clippy::unnecessary_cast)] + self.proc_mem_stream.seek(SeekFrom::Start(vaddr as u64))?; + let mut link_map = LinkMap::default(); + // SAFETY: From the point of view of this program, + // LinkMap only contains scalar values where any value is allowed. + let data = unsafe { link_map.as_mut_bytes() }; + self.proc_mem_stream + .read_exact(data) + .map_err(|e| eyre!("Failed to read link_map: {}", e))?; + Ok((vaddr, link_map)) + } +} + +impl<'a, P: Read + Seek> Iterator for RDebugIter<'a, P> { + /// Tuple of the link map's virtual address and the link map itself. + type Item = (ElfPtrSize, LinkMap); + + fn next(&mut self) -> Option { + if self.l_next == 0 { + return None; + } + + self.read_next().ok().map(|(vaddr, link_map)| { + self.l_next = link_map.l_next; + (vaddr, link_map) + }) + } +} diff --git a/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_debug_data.snap b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_debug_data.snap new file mode 100644 index 0000000..fbc0986 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_debug_data.snap @@ -0,0 +1,36 @@ +--- +source: memfaultd/src/cli/memfault_core_handler/core_elf_memfault_note.rs +expression: deser_capture_logs +--- +Map( + [ + ( + Text( + "schema_version", + ), + Integer( + Integer( + 1, + ), + ), + ), + ( + Text( + "capture_logs", + ), + Array( + [ + Text( + "Error 1", + ), + Text( + "Error 2", + ), + Text( + "Error 3", + ), + ], + ), + ), + ], +) diff --git a/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_metadata_as_map@app_logs.snap b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_metadata_as_map@app_logs.snap new file mode 100644 index 0000000..03d4449 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_metadata_as_map@app_logs.snap @@ -0,0 +1,165 @@ +--- +source: memfaultd/src/cli/memfault_core_handler/core_elf_memfault_note.rs +expression: deser_map +--- +Map( + [ + ( + Integer( + Integer( + 1, + ), + ), + Integer( + Integer( + 1, + ), + ), + ), + ( + Integer( + Integer( + 2, + ), + ), + Text( + "SDK_VERSION", + ), + ), + ( + Integer( + Integer( + 3, + ), + ), + Integer( + Integer( + 1234, + ), + ), + ), + ( + Integer( + Integer( + 4, + ), + ), + Text( + "12345678", + ), + ), + ( + Integer( + Integer( + 5, + ), + ), + Text( + "evt", + ), + ), + ( + Integer( + Integer( + 6, + ), + ), + Text( + "main", + ), + ), + ( + Integer( + Integer( + 7, + ), + ), + Text( + "1.0.0", + ), + ), + ( + Integer( + Integer( + 8, + ), + ), + Text( + "binary -a -b -c", + ), + ), + ( + Integer( + Integer( + 9, + ), + ), + Map( + [ + ( + Text( + "type", + ), + Text( + "kernel_selection", + ), + ), + ], + ), + ), + ( + Integer( + Integer( + 10, + ), + ), + Map( + [ + ( + Text( + "logs", + ), + Array( + [ + Text( + "Error 1", + ), + Text( + "Error 2", + ), + Text( + "Error 3", + ), + ], + ), + ), + ( + Text( + "format", + ), + Map( + [ + ( + Text( + "id", + ), + Text( + "v1", + ), + ), + ( + Text( + "serialization", + ), + Text( + "json-lines", + ), + ), + ], + ), + ), + ], + ), + ), + ], +) diff --git a/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_metadata_as_map@kernel_selection.snap b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_metadata_as_map@kernel_selection.snap new file mode 100644 index 0000000..690ef76 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_metadata_as_map@kernel_selection.snap @@ -0,0 +1,119 @@ +--- +source: memfaultd/src/cli/memfault_core_handler/core_elf_memfault_note.rs +expression: deser_map +--- +Map( + [ + ( + Integer( + Integer( + 1, + ), + ), + Integer( + Integer( + 1, + ), + ), + ), + ( + Integer( + Integer( + 2, + ), + ), + Text( + "SDK_VERSION", + ), + ), + ( + Integer( + Integer( + 3, + ), + ), + Integer( + Integer( + 1234, + ), + ), + ), + ( + Integer( + Integer( + 4, + ), + ), + Text( + "12345678", + ), + ), + ( + Integer( + Integer( + 5, + ), + ), + Text( + "evt", + ), + ), + ( + Integer( + Integer( + 6, + ), + ), + Text( + "main", + ), + ), + ( + Integer( + Integer( + 7, + ), + ), + Text( + "1.0.0", + ), + ), + ( + Integer( + Integer( + 8, + ), + ), + Text( + "binary -a -b -c", + ), + ), + ( + Integer( + Integer( + 9, + ), + ), + Map( + [ + ( + Text( + "type", + ), + Text( + "kernel_selection", + ), + ), + ], + ), + ), + ( + Integer( + Integer( + 10, + ), + ), + Null, + ), + ], +) diff --git a/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_metadata_as_map@threads.snap b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_metadata_as_map@threads.snap new file mode 100644 index 0000000..65d4c87 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_memfault_note__test__serialize_metadata_as_map@threads.snap @@ -0,0 +1,129 @@ +--- +source: memfaultd/src/cli/memfault_core_handler/core_elf_memfault_note.rs +expression: deser_map +--- +Map( + [ + ( + Integer( + Integer( + 1, + ), + ), + Integer( + Integer( + 1, + ), + ), + ), + ( + Integer( + Integer( + 2, + ), + ), + Text( + "SDK_VERSION", + ), + ), + ( + Integer( + Integer( + 3, + ), + ), + Integer( + Integer( + 1234, + ), + ), + ), + ( + Integer( + Integer( + 4, + ), + ), + Text( + "12345678", + ), + ), + ( + Integer( + Integer( + 5, + ), + ), + Text( + "evt", + ), + ), + ( + Integer( + Integer( + 6, + ), + ), + Text( + "main", + ), + ), + ( + Integer( + Integer( + 7, + ), + ), + Text( + "1.0.0", + ), + ), + ( + Integer( + Integer( + 8, + ), + ), + Text( + "binary -a -b -c", + ), + ), + ( + Integer( + Integer( + 9, + ), + ), + Map( + [ + ( + Text( + "type", + ), + Text( + "threads", + ), + ), + ( + Text( + "max_thread_size_kib", + ), + Integer( + Integer( + 32, + ), + ), + ), + ], + ), + ), + ( + Integer( + Integer( + 10, + ), + ), + Null, + ), + ], +) diff --git a/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_note__test__iterate_elf_notes_with_fixture.snap b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_note__test__iterate_elf_notes_with_fixture.snap new file mode 100644 index 0000000..5812d39 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_elf_note__test__iterate_elf_notes_with_fixture.snap @@ -0,0 +1,756 @@ +--- +source: memfaultd/src/cli/memfault_core_handler/core_elf_note.rs +expression: notes +--- +[ + ProcessStatus( + ProcessStatusNote { + si_signo: 11, + si_code: 0, + si_errno: 0, + pr_cursig: 11, + pad0: 0, + pr_sigpend: 0, + pr_sighold: 0, + pr_pid: 1757063, + pr_ppid: 4514, + pr_pgrp: 1756783, + pr_sid: 1706201, + pr_utime: ProcessTimeVal { + tv_sec: 0, + tv_usec: 0, + }, + pr_stime: ProcessTimeVal { + tv_sec: 0, + tv_usec: 0, + }, + pr_cutime: ProcessTimeVal { + tv_sec: 0, + tv_usec: 0, + }, + pr_cstime: ProcessTimeVal { + tv_sec: 0, + tv_usec: 0, + }, + pr_reg: user_regs_struct { + r15: 140723164295904, + r14: 1, + r13: 140723164295824, + r12: 140723155918848, + rbp: 0, + rbx: 140723155914752, + r11: 582, + r10: 140192614273224, + r9: 140192606909824, + r8: 0, + rax: 0, + rcx: 42, + rdx: 21885, + rsi: 0, + rdi: 0, + orig_rax: 18446744073709551615, + rip: 93999345134665, + cs: 51, + eflags: 66118, + rsp: 140723164290544, + ss: 43, + fs_base: 140192606909824, + gs_base: 0, + ds: 0, + es: 0, + fs: 0, + gs: 0, + }, + pr_fpvalid: 1, + pad1: 0, + }, + ), + Auxv( + [ + AuxvEntry { + key: 33, + value: 140723164557312, + }, + AuxvEntry { + key: 51, + value: 3632, + }, + AuxvEntry { + key: 16, + value: 3219913727, + }, + AuxvEntry { + key: 6, + value: 4096, + }, + AuxvEntry { + key: 17, + value: 100, + }, + AuxvEntry { + key: 3, + value: 93999343464512, + }, + AuxvEntry { + key: 4, + value: 56, + }, + AuxvEntry { + key: 5, + value: 14, + }, + AuxvEntry { + key: 7, + value: 140192614031360, + }, + AuxvEntry { + key: 8, + value: 0, + }, + AuxvEntry { + key: 9, + value: 93999344771504, + }, + AuxvEntry { + key: 11, + value: 1000, + }, + AuxvEntry { + key: 12, + value: 1000, + }, + AuxvEntry { + key: 13, + value: 1000, + }, + AuxvEntry { + key: 14, + value: 1000, + }, + AuxvEntry { + key: 23, + value: 0, + }, + AuxvEntry { + key: 25, + value: 140723164297705, + }, + AuxvEntry { + key: 26, + value: 2, + }, + AuxvEntry { + key: 31, + value: 140723164307423, + }, + AuxvEntry { + key: 15, + value: 140723164297721, + }, + AuxvEntry { + key: 0, + value: 0, + }, + ], + ), + File( + FileNote { + page_size: 4096, + mapped_files: [ + MappedFile { + path: Some( + "/home/blake/git/memfault-linux-sdk-internal/meta-memfault/recipes-memfault/memfaultd/files/target/debug/memfaultctl", + ), + start_addr: 93999343464448, + end_addr: 93999344754688, + page_offset: 0, + }, + MappedFile { + path: Some( + "/home/blake/git/memfault-linux-sdk-internal/meta-memfault/recipes-memfault/memfaultd/files/target/debug/memfaultctl", + ), + start_addr: 93999344754688, + end_addr: 93999353466880, + page_offset: 315, + }, + MappedFile { + path: Some( + "/home/blake/git/memfault-linux-sdk-internal/meta-memfault/recipes-memfault/memfaultd/files/target/debug/memfaultctl", + ), + start_addr: 93999353466880, + end_addr: 93999355916288, + page_offset: 2442, + }, + MappedFile { + path: Some( + "/home/blake/git/memfault-linux-sdk-internal/meta-memfault/recipes-memfault/memfaultd/files/target/debug/memfaultctl", + ), + start_addr: 93999355916288, + end_addr: 93999356571648, + page_offset: 3039, + }, + MappedFile { + path: Some( + "/home/blake/git/memfault-linux-sdk-internal/meta-memfault/recipes-memfault/memfaultd/files/target/debug/memfaultctl", + ), + start_addr: 93999356571648, + end_addr: 93999356583936, + page_offset: 3199, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgpg-error.so.0.32.1", + ), + start_addr: 140192606912512, + end_addr: 140192606928896, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgpg-error.so.0.32.1", + ), + start_addr: 140192606928896, + end_addr: 140192607019008, + page_offset: 4, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgpg-error.so.0.32.1", + ), + start_addr: 140192607019008, + end_addr: 140192607059968, + page_offset: 26, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgpg-error.so.0.32.1", + ), + start_addr: 140192607059968, + end_addr: 140192607064064, + page_offset: 35, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgpg-error.so.0.32.1", + ), + start_addr: 140192607064064, + end_addr: 140192607068160, + page_offset: 36, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcrypt.so.20.3.4", + ), + start_addr: 140192607068160, + end_addr: 140192607129600, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcrypt.so.20.3.4", + ), + start_addr: 140192607129600, + end_addr: 140192608071680, + page_offset: 15, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcrypt.so.20.3.4", + ), + start_addr: 140192608071680, + end_addr: 140192608325632, + page_offset: 245, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcrypt.so.20.3.4", + ), + start_addr: 140192608325632, + end_addr: 140192608329728, + page_offset: 307, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcrypt.so.20.3.4", + ), + start_addr: 140192608329728, + end_addr: 140192608342016, + page_offset: 307, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcrypt.so.20.3.4", + ), + start_addr: 140192608342016, + end_addr: 140192608366592, + page_offset: 310, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libcap.so.2.44", + ), + start_addr: 140192608370688, + end_addr: 140192608382976, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libcap.so.2.44", + ), + start_addr: 140192608382976, + end_addr: 140192608399360, + page_offset: 3, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libcap.so.2.44", + ), + start_addr: 140192608399360, + end_addr: 140192608407552, + page_offset: 7, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libcap.so.2.44", + ), + start_addr: 140192608407552, + end_addr: 140192608411648, + page_offset: 8, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libcap.so.2.44", + ), + start_addr: 140192608411648, + end_addr: 140192608415744, + page_offset: 9, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblz4.so.1.9.3", + ), + start_addr: 140192608415744, + end_addr: 140192608423936, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblz4.so.1.9.3", + ), + start_addr: 140192608423936, + end_addr: 140192608526336, + page_offset: 2, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblz4.so.1.9.3", + ), + start_addr: 140192608526336, + end_addr: 140192608534528, + page_offset: 27, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblz4.so.1.9.3", + ), + start_addr: 140192608534528, + end_addr: 140192608538624, + page_offset: 29, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblz4.so.1.9.3", + ), + start_addr: 140192608538624, + end_addr: 140192608542720, + page_offset: 29, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblz4.so.1.9.3", + ), + start_addr: 140192608542720, + end_addr: 140192608546816, + page_offset: 30, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libzstd.so.1.4.8", + ), + start_addr: 140192608546816, + end_addr: 140192608587776, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libzstd.so.1.4.8", + ), + start_addr: 140192608587776, + end_addr: 140192609316864, + page_offset: 10, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libzstd.so.1.4.8", + ), + start_addr: 140192609316864, + end_addr: 140192609386496, + page_offset: 188, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libzstd.so.1.4.8", + ), + start_addr: 140192609386496, + end_addr: 140192609390592, + page_offset: 204, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libzstd.so.1.4.8", + ), + start_addr: 140192609390592, + end_addr: 140192609394688, + page_offset: 205, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblzma.so.5.2.5", + ), + start_addr: 140192609394688, + end_addr: 140192609406976, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblzma.so.5.2.5", + ), + start_addr: 140192609406976, + end_addr: 140192609517568, + page_offset: 3, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblzma.so.5.2.5", + ), + start_addr: 140192609517568, + end_addr: 140192609562624, + page_offset: 30, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblzma.so.5.2.5", + ), + start_addr: 140192609562624, + end_addr: 140192609566720, + page_offset: 40, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/liblzma.so.5.2.5", + ), + start_addr: 140192609566720, + end_addr: 140192609570816, + page_offset: 41, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libm.so.6", + ), + start_addr: 140192609570816, + end_addr: 140192609628160, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libm.so.6", + ), + start_addr: 140192609628160, + end_addr: 140192610136064, + page_offset: 14, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libm.so.6", + ), + start_addr: 140192610136064, + end_addr: 140192610508800, + page_offset: 138, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libm.so.6", + ), + start_addr: 140192610508800, + end_addr: 140192610512896, + page_offset: 228, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libm.so.6", + ), + start_addr: 140192610512896, + end_addr: 140192610516992, + page_offset: 229, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libc.so.6", + ), + start_addr: 140192610516992, + end_addr: 140192610680832, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libc.so.6", + ), + start_addr: 140192610680832, + end_addr: 140192612339712, + page_offset: 40, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libc.so.6", + ), + start_addr: 140192612339712, + end_addr: 140192612700160, + page_offset: 445, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libc.so.6", + ), + start_addr: 140192612700160, + end_addr: 140192612716544, + page_offset: 532, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libc.so.6", + ), + start_addr: 140192612716544, + end_addr: 140192612724736, + page_offset: 536, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcc_s.so.1", + ), + start_addr: 140192612790272, + end_addr: 140192612802560, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcc_s.so.1", + ), + start_addr: 140192612802560, + end_addr: 140192612896768, + page_offset: 3, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcc_s.so.1", + ), + start_addr: 140192612896768, + end_addr: 140192612913152, + page_offset: 26, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcc_s.so.1", + ), + start_addr: 140192612913152, + end_addr: 140192612917248, + page_offset: 29, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libgcc_s.so.1", + ), + start_addr: 140192612917248, + end_addr: 140192612921344, + page_offset: 30, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libconfig.so.9.2.0", + ), + start_addr: 140192612929536, + end_addr: 140192612941824, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libconfig.so.9.2.0", + ), + start_addr: 140192612941824, + end_addr: 140192612966400, + page_offset: 3, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libconfig.so.9.2.0", + ), + start_addr: 140192612966400, + end_addr: 140192612978688, + page_offset: 9, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libconfig.so.9.2.0", + ), + start_addr: 140192612978688, + end_addr: 140192612982784, + page_offset: 11, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libconfig.so.9.2.0", + ), + start_addr: 140192612982784, + end_addr: 140192612986880, + page_offset: 12, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libsystemd.so.0.32.0", + ), + start_addr: 140192612986880, + end_addr: 140192613064704, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libsystemd.so.0.32.0", + ), + start_addr: 140192613064704, + end_addr: 140192613584896, + page_offset: 19, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libsystemd.so.0.32.0", + ), + start_addr: 140192613584896, + end_addr: 140192613761024, + page_offset: 146, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libsystemd.so.0.32.0", + ), + start_addr: 140192613761024, + end_addr: 140192613765120, + page_offset: 189, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libsystemd.so.0.32.0", + ), + start_addr: 140192613765120, + end_addr: 140192613793792, + page_offset: 189, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libsystemd.so.0.32.0", + ), + start_addr: 140192613793792, + end_addr: 140192613797888, + page_offset: 196, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libz.so.1.2.11", + ), + start_addr: 140192613801984, + end_addr: 140192613810176, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libz.so.1.2.11", + ), + start_addr: 140192613810176, + end_addr: 140192613879808, + page_offset: 2, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libz.so.1.2.11", + ), + start_addr: 140192613879808, + end_addr: 140192613904384, + page_offset: 19, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libz.so.1.2.11", + ), + start_addr: 140192613904384, + end_addr: 140192613908480, + page_offset: 25, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libz.so.1.2.11", + ), + start_addr: 140192613908480, + end_addr: 140192613912576, + page_offset: 25, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/libz.so.1.2.11", + ), + start_addr: 140192613912576, + end_addr: 140192613916672, + page_offset: 26, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2", + ), + start_addr: 140192614031360, + end_addr: 140192614039552, + page_offset: 0, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2", + ), + start_addr: 140192614039552, + end_addr: 140192614211584, + page_offset: 2, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2", + ), + start_addr: 140192614211584, + end_addr: 140192614256640, + page_offset: 44, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2", + ), + start_addr: 140192614260736, + end_addr: 140192614268928, + page_offset: 55, + }, + MappedFile { + path: Some( + "/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2", + ), + start_addr: 140192614268928, + end_addr: 140192614277120, + page_offset: 57, + }, + ], + incomplete: false, + }, + ), +] diff --git a/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_transformer__test__transform@kernel_selection.snap b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_transformer__test__transform@kernel_selection.snap new file mode 100644 index 0000000..dc66b7e --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_transformer__test__transform@kernel_selection.snap @@ -0,0 +1,387 @@ +--- +source: memfaultd/src/cli/memfault_core_handler/core_transformer.rs +expression: segments +--- +[ + ( + ProgramHeader { + p_type: "PT_NOTE", + p_flags: 0x0, + p_offset: 0x660, + p_vaddr: 0x0, + p_paddr: 0x0, + p_filesz: 0xdec, + p_memsz: 0x0, + p_align: 0, + }, + Buffer( + [], + ), + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x2000, + p_vaddr: 0x5587ae8bd000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x5, + p_offset: 0x3000, + p_vaddr: 0x5587ae8be000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x3000, + p_vaddr: 0x5587ae8bf000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x3000, + p_vaddr: 0x5587ae8c0000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x4000, + p_vaddr: 0x5587ae8c1000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x5000, + p_vaddr: 0x7f6e38cb1000, + p_paddr: 0x0, + p_filesz: 0x3000, + p_memsz: 0x3000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x8000, + p_vaddr: 0x7f6e38cb4000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x26000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x5, + p_offset: 0x9000, + p_vaddr: 0x7f6e38cda000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x155000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x9000, + p_vaddr: 0x7f6e38e2f000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x53000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x9000, + p_vaddr: 0x7f6e38e82000, + p_paddr: 0x0, + p_filesz: 0x4000, + p_memsz: 0x4000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0xd000, + p_vaddr: 0x7f6e38e86000, + p_paddr: 0x0, + p_filesz: 0x2000, + p_memsz: 0x2000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0xf000, + p_vaddr: 0x7f6e38e88000, + p_paddr: 0x0, + p_filesz: 0xd000, + p_memsz: 0xd000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x1c000, + p_vaddr: 0x7f6e38e99000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x10000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x5, + p_offset: 0x1d000, + p_vaddr: 0x7f6e38ea9000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x73000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x1d000, + p_vaddr: 0x7f6e38f1c000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x5a000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x1d000, + p_vaddr: 0x7f6e38f76000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x1e000, + p_vaddr: 0x7f6e38f77000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x1f000, + p_vaddr: 0x7f6e38f78000, + p_paddr: 0x0, + p_filesz: 0x2000, + p_memsz: 0x2000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x21000, + p_vaddr: 0x7f6e38f7a000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x5, + p_offset: 0x22000, + p_vaddr: 0x7f6e38f7b000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x25000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x22000, + p_vaddr: 0x7f6e38fa0000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0xa000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x22000, + p_vaddr: 0x7f6e38faa000, + p_paddr: 0x0, + p_filesz: 0x2000, + p_memsz: 0x2000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x24000, + p_vaddr: 0x7f6e38fac000, + p_paddr: 0x0, + p_filesz: 0x2000, + p_memsz: 0x2000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x26000, + p_vaddr: 0x7ffe4ba5b000, + p_paddr: 0x0, + p_filesz: 0x21000, + p_memsz: 0x21000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x47000, + p_vaddr: 0x7ffe4bb1b000, + p_paddr: 0x0, + p_filesz: 0x4000, + p_memsz: 0x4000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x5, + p_offset: 0x4b000, + p_vaddr: 0x7ffe4bb1f000, + p_paddr: 0x0, + p_filesz: 0x2000, + p_memsz: 0x2000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x1, + p_offset: 0x4d000, + p_vaddr: 0xffffffffff600000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_NOTE", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x0, + p_paddr: 0x0, + p_filesz: 0xb8, + p_memsz: 0x0, + p_align: 0, + }, + Buffer( + [], + ), + ), +] diff --git a/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_transformer__test__transform@threads_32k.snap b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_transformer__test__transform@threads_32k.snap new file mode 100644 index 0000000..451f0ba --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_transformer__test__transform@threads_32k.snap @@ -0,0 +1,244 @@ +--- +source: memfaultd/src/cli/memfault_core_handler/core_transformer.rs +expression: segments +--- +[ + ( + ProgramHeader { + p_type: "PT_NOTE", + p_flags: 0x0, + p_offset: 0x660, + p_vaddr: 0x0, + p_paddr: 0x0, + p_filesz: 0xdec, + p_memsz: 0x0, + p_align: 0, + }, + Buffer( + [], + ), + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x5587ae8bd000, + p_paddr: 0x0, + p_filesz: 0x39c, + p_memsz: 0x39c, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x5587ae8c0dc0, + p_paddr: 0x0, + p_filesz: 0x200, + p_memsz: 0x200, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38cb4000, + p_paddr: 0x0, + p_filesz: 0x3b4, + p_memsz: 0x3b4, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38e99000, + p_paddr: 0x0, + p_filesz: 0x30c, + p_memsz: 0x30c, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38f781d0, + p_paddr: 0x0, + p_filesz: 0xc, + p_memsz: 0xc, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38f781e0, + p_paddr: 0x0, + p_filesz: 0x28, + p_memsz: 0x28, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38f78740, + p_paddr: 0x0, + p_filesz: 0x20, + p_memsz: 0x20, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38f78760, + p_paddr: 0x0, + p_filesz: 0x28, + p_memsz: 0x28, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38f7a000, + p_paddr: 0x0, + p_filesz: 0x25c, + p_memsz: 0x25c, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38facad0, + p_paddr: 0x0, + p_filesz: 0x28, + p_memsz: 0x28, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38fad118, + p_paddr: 0x0, + p_filesz: 0x28, + p_memsz: 0x28, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38fad2e0, + p_paddr: 0x0, + p_filesz: 0x28, + p_memsz: 0x28, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38fad8a0, + p_paddr: 0x0, + p_filesz: 0x1, + p_memsz: 0x1, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7f6e38fad8b0, + p_paddr: 0x0, + p_filesz: 0x28, + p_memsz: 0x28, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7ffe4ba7a3c0, + p_paddr: 0x0, + p_filesz: 0x1c40, + p_memsz: 0x1c40, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x7ffe4bb1f000, + p_paddr: 0x0, + p_filesz: 0x554, + p_memsz: 0x554, + p_align: 8, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_NOTE", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x0, + p_paddr: 0x0, + p_filesz: 0xc8, + p_memsz: 0x0, + p_align: 0, + }, + Buffer( + [], + ), + ), +] diff --git a/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_transformer__test__transform@threads_32k_no_filter_support.snap b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_transformer__test__transform@threads_32k_no_filter_support.snap new file mode 100644 index 0000000..dc66b7e --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__core_transformer__test__transform@threads_32k_no_filter_support.snap @@ -0,0 +1,387 @@ +--- +source: memfaultd/src/cli/memfault_core_handler/core_transformer.rs +expression: segments +--- +[ + ( + ProgramHeader { + p_type: "PT_NOTE", + p_flags: 0x0, + p_offset: 0x660, + p_vaddr: 0x0, + p_paddr: 0x0, + p_filesz: 0xdec, + p_memsz: 0x0, + p_align: 0, + }, + Buffer( + [], + ), + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x2000, + p_vaddr: 0x5587ae8bd000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x5, + p_offset: 0x3000, + p_vaddr: 0x5587ae8be000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x3000, + p_vaddr: 0x5587ae8bf000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x3000, + p_vaddr: 0x5587ae8c0000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x4000, + p_vaddr: 0x5587ae8c1000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x5000, + p_vaddr: 0x7f6e38cb1000, + p_paddr: 0x0, + p_filesz: 0x3000, + p_memsz: 0x3000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x8000, + p_vaddr: 0x7f6e38cb4000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x26000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x5, + p_offset: 0x9000, + p_vaddr: 0x7f6e38cda000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x155000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x9000, + p_vaddr: 0x7f6e38e2f000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x53000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x9000, + p_vaddr: 0x7f6e38e82000, + p_paddr: 0x0, + p_filesz: 0x4000, + p_memsz: 0x4000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0xd000, + p_vaddr: 0x7f6e38e86000, + p_paddr: 0x0, + p_filesz: 0x2000, + p_memsz: 0x2000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0xf000, + p_vaddr: 0x7f6e38e88000, + p_paddr: 0x0, + p_filesz: 0xd000, + p_memsz: 0xd000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x1c000, + p_vaddr: 0x7f6e38e99000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x10000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x5, + p_offset: 0x1d000, + p_vaddr: 0x7f6e38ea9000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x73000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x1d000, + p_vaddr: 0x7f6e38f1c000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x5a000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x1d000, + p_vaddr: 0x7f6e38f76000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x1e000, + p_vaddr: 0x7f6e38f77000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x1f000, + p_vaddr: 0x7f6e38f78000, + p_paddr: 0x0, + p_filesz: 0x2000, + p_memsz: 0x2000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x21000, + p_vaddr: 0x7f6e38f7a000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x5, + p_offset: 0x22000, + p_vaddr: 0x7f6e38f7b000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0x25000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x22000, + p_vaddr: 0x7f6e38fa0000, + p_paddr: 0x0, + p_filesz: 0x0, + p_memsz: 0xa000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x22000, + p_vaddr: 0x7f6e38faa000, + p_paddr: 0x0, + p_filesz: 0x2000, + p_memsz: 0x2000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x24000, + p_vaddr: 0x7f6e38fac000, + p_paddr: 0x0, + p_filesz: 0x2000, + p_memsz: 0x2000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x6, + p_offset: 0x26000, + p_vaddr: 0x7ffe4ba5b000, + p_paddr: 0x0, + p_filesz: 0x21000, + p_memsz: 0x21000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x4, + p_offset: 0x47000, + p_vaddr: 0x7ffe4bb1b000, + p_paddr: 0x0, + p_filesz: 0x4000, + p_memsz: 0x4000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x5, + p_offset: 0x4b000, + p_vaddr: 0x7ffe4bb1f000, + p_paddr: 0x0, + p_filesz: 0x2000, + p_memsz: 0x2000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_LOAD", + p_flags: 0x1, + p_offset: 0x4d000, + p_vaddr: 0xffffffffff600000, + p_paddr: 0x0, + p_filesz: 0x1000, + p_memsz: 0x1000, + p_align: 4096, + }, + ProcessMemory, + ), + ( + ProgramHeader { + p_type: "PT_NOTE", + p_flags: 0x0, + p_offset: 0x0, + p_vaddr: 0x0, + p_paddr: 0x0, + p_filesz: 0xb8, + p_memsz: 0x0, + p_align: 0, + }, + Buffer( + [], + ), + ), +] diff --git a/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__find_dynamic__test__find_dynamic_linker_ranges.snap b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__find_dynamic__test__find_dynamic_linker_ranges.snap new file mode 100644 index 0000000..9848a1f --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__find_dynamic__test__find_dynamic_linker_ranges.snap @@ -0,0 +1,58 @@ +--- +source: memfaultd/src/cli/memfault_core_handler/find_dynamic.rs +expression: output +--- +[ + MemoryRange { + start: "0x5587ae8bd040", + end: "0x5587ae8bd318", + }, + MemoryRange { + start: "0x5587ae8c0dc0", + end: "0x5587ae8c0fc0", + }, + MemoryRange { + start: "0x7f6e38fad118", + end: "0x7f6e38fad140", + }, + MemoryRange { + start: "0x7f6e38fad2e0", + end: "0x7f6e38fad308", + }, + MemoryRange { + start: "0x7f6e38fad8b0", + end: "0x7f6e38fad8d8", + }, + MemoryRange { + start: "0x7f6e38f781e0", + end: "0x7f6e38f78208", + }, + MemoryRange { + start: "0x7f6e38f78760", + end: "0x7f6e38f78788", + }, + MemoryRange { + start: "0x7f6e38facad0", + end: "0x7f6e38facaf8", + }, + MemoryRange { + start: "0x7f6e38fad8a0", + end: "0x7f6e38fad8a1", + }, + MemoryRange { + start: "0x7ffe4bb1f371", + end: "0x7ffe4bb1f381", + }, + MemoryRange { + start: "0x7f6e38f781d0", + end: "0x7f6e38f781dc", + }, + MemoryRange { + start: "0x7f6e38f78740", + end: "0x7f6e38f78760", + }, + MemoryRange { + start: "0x5587ae8bd318", + end: "0x5587ae8bd334", + }, +] diff --git a/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__log_wrapper__test__log_saving.snap b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__log_wrapper__test__log_saving.snap new file mode 100644 index 0000000..415c90b --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/snapshots/memfaultd__cli__memfault_core_handler__log_wrapper__test__log_saving.snap @@ -0,0 +1,10 @@ +--- +source: memfaultd/src/cli/memfault_core_handler/log_wrapper.rs +expression: errors +--- +[ + "ERROR test:71 - Test message", + "WARN test:71 - Test message", + "INFO test:71 - Test message", + "DEBUG test:71 - Test message" +] diff --git a/memfaultd/src/cli/memfault_core_handler/test_utils.rs b/memfaultd/src/cli/memfault_core_handler/test_utils.rs new file mode 100644 index 0000000..f5d4a51 --- /dev/null +++ b/memfaultd/src/cli/memfault_core_handler/test_utils.rs @@ -0,0 +1,231 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::cli::memfault_core_handler::core_writer::{CoreWriter, SegmentData}; +use crate::cli::memfault_core_handler::elf; +use crate::cli::memfault_core_handler::procfs::ProcMaps; +use elf::{header::Header, program_header::ProgramHeader}; +// NOTE: Using the "universal" (width-agnostic types in the test): +use goblin::elf::{ + program_header::{PF_R, PF_W, PF_X, PT_LOAD}, + Elf, ProgramHeader as UniversalProgramHeader, +}; +use procfs::process::{MMPermissions, MMapPath, MemoryMap}; +use std::fs::read; +use std::io::{Cursor, Error, ErrorKind, Read, Seek, SeekFrom, Take}; +use std::path::Path; +use take_mut::take; + +pub fn build_test_header(class: u8, endianness: u8, machine: u16) -> Header { + let mut e_ident = [0u8; 16]; + e_ident[..elf::header::SELFMAG].copy_from_slice(elf::header::ELFMAG); + e_ident[elf::header::EI_CLASS] = class; + e_ident[elf::header::EI_DATA] = endianness; + e_ident[elf::header::EI_VERSION] = elf::header::EV_CURRENT; + + Header { + e_phoff: elf::header::SIZEOF_EHDR.try_into().unwrap(), + e_phentsize: elf::program_header::SIZEOF_PHDR.try_into().unwrap(), + e_ehsize: elf::header::SIZEOF_EHDR.try_into().unwrap(), + e_version: elf::header::EV_CURRENT.try_into().unwrap(), + e_phnum: 0, + e_machine: machine, + e_ident, + ..Default::default() + } +} + +pub struct MockCoreWriter<'a> { + pub output_size: usize, + pub segments: &'a mut Vec<(ProgramHeader, SegmentData)>, +} + +impl<'a> MockCoreWriter<'a> { + pub fn new(segments: &mut Vec<(ProgramHeader, SegmentData)>) -> MockCoreWriter { + MockCoreWriter { + output_size: 0, + segments, + } + } +} + +impl<'a> CoreWriter for MockCoreWriter<'a> { + fn add_segment(&mut self, program_header: ProgramHeader, data: SegmentData) { + self.segments.push((program_header, data)); + } + + fn write(&mut self) -> eyre::Result<()> { + Ok(()) + } + + fn calc_output_size(&self) -> usize { + self.output_size + } +} + +/// A fake `ProcMaps` implementation that uses the memory ranges from PT_LOAD segments. +pub struct FakeProcMaps { + maps: Vec, +} + +impl FakeProcMaps { + /// Creates a new `FakeProcMaps` from the given core.elf file. + pub fn new_from_path>(core_elf_path: P) -> eyre::Result { + let data = &read(core_elf_path)?; + + let load_segments = load_segments_from_buffer(data)?; + let maps = load_segments + .iter() + .map(|ph| MemoryMap { + address: (ph.p_vaddr, ph.p_vaddr + ph.p_memsz), + perms: ph_flags_to_perms(ph.p_flags), + // All below is (likely) incorrect, but we cannot get this info from the ELF file: + offset: 0, + dev: (0, 0), + inode: 0, + pathname: MMapPath::Other("Unknown".into()), + extension: Default::default(), + }) + .collect(); + + Ok(Self { maps }) + } +} + +impl ProcMaps for FakeProcMaps { + fn get_process_maps(&mut self) -> eyre::Result> { + Ok(self.maps.clone()) + } +} + +/// A fake `/proc//mem` stream that uses the memory contents from PT_LOAD segments of a +/// core.elf as the data in the `/proc//mem` file. +pub struct FakeProcMem { + // Note: as per the ELF spec, are expected to be sorted by v_addr: + load_segments: Vec, + inner: Take>>, +} + +impl FakeProcMem { + pub fn new_from_path>(core_elf_path: P) -> eyre::Result { + let data = read(core_elf_path)?; + Self::new(data) + } + pub fn new(data: Vec) -> eyre::Result { + let load_segments = load_segments_from_buffer(&data)?; + + Ok(FakeProcMem { + inner: FakeProcMem::make_inner(data, &load_segments[0])?, + load_segments, + }) + } + + /// Creates a Read stream that corresponds to the given program header. + /// The stream is a "view" into the segment in the coredump data buffer. + fn make_inner( + data: Vec, + ph: &UniversalProgramHeader, + ) -> eyre::Result>>> { + let mut cursor = Cursor::new(data); + // Ignore unnecessary cast here as it is needed on 32-bit systems. + #[allow(clippy::unnecessary_cast)] + cursor.seek(SeekFrom::Start(ph.p_offset as u64))?; + Ok(cursor.take(ph.p_filesz)) + } + + fn file_offset_to_vaddr(&self, offset: u64) -> Option { + self.load_segments + .iter() + .find(|ph| { + let start = ph.p_offset; + let end = start + ph.p_filesz; + offset >= start && offset < end + }) + .map(|ph| ph.p_vaddr + (offset - ph.p_offset)) + } +} + +impl Read for FakeProcMem { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.inner.read(buf) + } +} + +impl Seek for FakeProcMem { + fn seek(&mut self, pos: SeekFrom) -> std::io::Result { + if let SeekFrom::End(_) = pos {} + + // The seek offset in /proc//mem is the virtual address of the process memory: + let vaddr = match pos { + SeekFrom::Start(pos) => Ok(pos), + SeekFrom::Current(pos) => self + .stream_position() + .map(|p| pos.checked_add(p as i64).unwrap() as u64), + SeekFrom::End(_) => Err(Error::new(ErrorKind::Other, "Not implemented")), + } + .unwrap(); + + // Find the PT_LOAD segment's program header that contains the requested vaddr: + let ph = self.load_segments.iter().find(|ph| { + let start = ph.p_vaddr; + let end = start + ph.p_memsz; + vaddr >= start && vaddr < end + }); + + match ph { + Some(ph) => { + // When seek() is called, always create a new inner stream that contains the + // requested seek position (vaddr), even if the seek position is within the current + // inner: + take(&mut self.inner, |inner| { + let data = inner.into_inner().into_inner(); + FakeProcMem::make_inner(data, ph).unwrap() + }); + // Seek within the new inner stream to the requested vaddr: + self.inner + .get_mut() + .seek(SeekFrom::Start(ph.p_offset + vaddr - ph.p_vaddr)) + } + None => Err(Error::new( + ErrorKind::Other, + format!("Invalid seek position: {:#x}", vaddr), + )), + } + } + + fn stream_position(&mut self) -> std::io::Result { + let inner_pos = self.inner.get_mut().stream_position()?; + self.file_offset_to_vaddr(inner_pos) + .ok_or_else(|| Error::new(ErrorKind::Other, "Invalid stream position")) + } +} + +fn load_segments_from_buffer(data: &[u8]) -> eyre::Result> { + let elf = Elf::parse(data)?; + let load_segments: Vec = elf + .program_headers + .iter() + .filter_map(|ph| { + if ph.p_type == PT_LOAD { + Some(ph.clone()) + } else { + None + } + }) + .collect(); + Ok(load_segments) +} + +fn ph_flags_to_perms(flags: u32) -> MMPermissions { + let mut perms = MMPermissions::empty(); + if flags & PF_R != 0 { + perms |= MMPermissions::READ; + } + if flags & PF_W != 0 { + perms |= MMPermissions::WRITE; + } + if flags & PF_X != 0 { + perms |= MMPermissions::EXECUTE; + } + perms +} diff --git a/memfaultd/src/cli/memfault_watch/buffer.rs b/memfaultd/src/cli/memfault_watch/buffer.rs new file mode 100644 index 0000000..5715289 --- /dev/null +++ b/memfaultd/src/cli/memfault_watch/buffer.rs @@ -0,0 +1,120 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + io::{BufRead, Result, Write}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }, +}; + +use log::error; + +pub(crate) static STOP_THREADS: AtomicBool = AtomicBool::new(false); + +/// This function monitors the reader and calls `read_and_double_write` on the two inputs. +/// It handles the buffering and the quitting in the case where the `read()` call returns +/// an error (in the case of an OS-level problem retrieving the process's stream) and +/// also the case where the program no longer as any more bytes to read. +pub(crate) fn monitor_and_buffer( + reader: impl BufRead, + mut io_writer: &mut impl Write, + file_writer: Arc>, +) { + // Create an initial buffer with a large starting capacity, to avoid the rapid + // allocation that can occur when inserting over and over again one by one + // Refs: https://blog.mozilla.org/nnethercote/2020/08/05/how-to-speed-up-the-rust-compiler-some-more-in-2020/ + // (under miscellaneous, discusses how rust-lang solved an adjacent issue by + // making the default allocation a minimum of 4) + + // The default value of 240 was chosen as a rough estimate of the average line size + // of a log. If there are lines significantly longer, the higher capacity will + // persist after clearing. + let mut buffer = Vec::with_capacity(240); + let mut reader = reader; + loop { + // Here we catch the case where we need to stop the threads because of an error + // reaching the process. We call join() in the upper loop, but this just waits + // for each thread to stop naturally. + if STOP_THREADS.load(Ordering::Relaxed) { + break; + } + + // Utilize the Take adapter to temporarily limit the amount we + // can ever read from *this limiter*. This avoids the problem later on in + // `read_and_double_write`, which calls `read_until` in order to read + // uninterrupted until reaching a \n. If a program's output prints extremely long + // lines, without ever printing a \n (for example, if a print subroutine breaks), + // we still want to handle that by limiting to this value. + let mut temporary_limiter = reader.take(8192); + let read_result = read_and_double_write( + &mut buffer, + &mut temporary_limiter, + &mut io_writer, + file_writer.clone(), + ); + + //Here we catch the case where we successfully read zero bytes. This should + // _only_ happen when there is _no more_ data to be read - because otherwise, + // any implementation of `Read` should _always block_ upon trying to read, + // waiting until there's _something_ to return (even Ok(1)). + + // TLDR: Ok(0) should only occur if the program exited. So, break. + if let Ok(0) = read_result { + break; + } + // Here we catch any `io`-related error when reading. + if let Err(e) = read_result { + error!("Error reading data: {e}"); + break; + } + // Clear the buffer - zeroes out the contents but does not affect the capacity + // Leaving the capacity untouched for two reasons: + // 1. Avoiding reallocating if lines are consistently longer than 240 bytes + // 2. When considering the allocation, it's unlikely for some extremely long + // line to hold a large buffer + buffer.clear(); + // Here, we reset the reader to remove the limiter, by consuming the Take and + // saving it back as the wrapped reader. + reader = temporary_limiter.into_inner(); + } + + if let Err(e) = file_writer + .lock() + .expect("Failed to lock file writer") + .flush() + { + error!("Error flushing file writer. {e}"); + } +} + +/// This function is meant to be called in a loop - it reads as much as it can from the +/// `reader` into the buffer, and writes to each of the writers. It handles the case of +/// no data currently available, as well as a broken reader, by returning an +/// std::io::Result. The contained usize success value is the number of bytes read. +/// Returns an error in the case of a generic io::Error, whether in the read, or in +/// either of the two writes/flushes. +/// The file writer is not flushed, as it's wrapped in a BufWriter and only needs to be +/// flushed at the end. +#[inline(always)] +pub(crate) fn read_and_double_write( + buffer: &mut Vec, + reader: &mut impl BufRead, + io_writer: &mut impl Write, + file_writer: Arc>, +) -> Result { + // We use read_until here to continuously read bytes until we reach a newline. + match reader.read_until(b'\n', buffer)? { + 0 => Ok(0), + bytes_read => { + io_writer.write_all(buffer)?; + io_writer.flush()?; + file_writer + .lock() + .expect("Failed to lock file writer") + .write_all(buffer)?; + Ok(bytes_read) + } + } +} diff --git a/memfaultd/src/cli/memfault_watch/mod.rs b/memfaultd/src/cli/memfault_watch/mod.rs new file mode 100644 index 0000000..2d63479 --- /dev/null +++ b/memfaultd/src/cli/memfault_watch/mod.rs @@ -0,0 +1,218 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + fs::File, + io::{stderr, stdout, BufReader, BufWriter}, + path::Path, + process::{Command, ExitStatus, Stdio}, + sync::{atomic::Ordering, Arc, Mutex}, + time::{Duration, Instant}, +}; + +use argh::{FromArgs, TopLevelCommand}; +use buffer::{monitor_and_buffer, STOP_THREADS}; +use chrono::Local; +use eyre::{eyre, Result}; +use flate2::{write::ZlibEncoder, Compression}; +use log::{error, info, trace, LevelFilter}; + +use crate::{ + cli::{init_logger, MemfaultdClient}, + config::Config, + mar::{CompressionAlgorithm, Metadata}, +}; + +use super::memfaultctl::WrappedArgs; + +mod buffer; + +#[derive(FromArgs)] +/// A command line utility to run a specified command and send its output to our backend +struct MemfaultWatchArgs { + /// use configuration file + #[argh(option, short = 'c')] + config_file: Option, + /// verbose output + #[argh(switch, short = 'V')] + verbose: bool, + + /// read in positional command argument + #[argh(positional)] + command: Vec, +} + +pub fn main() -> Result<()> { + let args: MemfaultWatchArgs = from_env(); + let exit_status = run_from_args(args)?; + std::process::exit(exit_status) +} + +fn run_from_args(args: MemfaultWatchArgs) -> Result { + init_logger(if args.verbose { + LevelFilter::Trace + } else { + LevelFilter::Info + }); + + let config_path = args.config_file.as_ref().map(Path::new); + let config = Config::read_from_system(config_path)?; + + // Set up log files + let file_name = format!("mfw-log-{}", Local::now().to_rfc3339()); + let stdio_log_file_name = format!("{file_name}.zlib"); + let stdio_log_file = File::create(&stdio_log_file_name) + .map_err(|_| eyre!("Failed to create output file on filesystem!"))?; + + let (command, additional_args) = args + .command + .split_first() + .ok_or_else(|| eyre!("No command given!"))?; + + trace!("Running command: {:?}", command); + + let start = Instant::now(); + + let mut child = Command::new(command) + .args(additional_args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| eyre!("Failed to run command! Does it exist in path?\nError: {e}"))?; + + let child_stdout = child + .stdout + .take() + .ok_or_else(|| eyre!("Failed to take stdout handle from child process."))?; + let child_stderr = child + .stderr + .take() + .ok_or_else(|| eyre!("Failed to take stderr handle from child process."))?; + + let mut child_stdout_reader = BufReader::new(child_stdout); + let mut child_stderr_reader = BufReader::new(child_stderr); + + let compression_encoder_stdio = ZlibEncoder::new(stdio_log_file, Compression::fast()); + + let stdio_file_writer = Arc::new(Mutex::new(BufWriter::new(compression_encoder_stdio))); + + let child_stdout_writer = stdio_file_writer; + let child_stderr_writer = child_stdout_writer.clone(); + + let child_stdout_monitor = std::thread::spawn(move || { + monitor_and_buffer(&mut child_stdout_reader, &mut stdout(), child_stdout_writer); + }); + + let child_stderr_monitor = std::thread::spawn(move || { + monitor_and_buffer(&mut child_stderr_reader, &mut stderr(), child_stderr_writer); + }); + + let mut get_status = || match child.try_wait() { + Ok(Some(status)) => { + trace!("Command completed with status {status}!"); + Ok(ProcessStatus::Exited(status)) + } + Ok(None) => Ok(ProcessStatus::Running), + Err(e) => { + error!("Failed to check command status! {e}"); + Err(e) + } + }; + let either_thread_still_running = + || !child_stdout_monitor.is_finished() || !child_stderr_monitor.is_finished(); + + // Check condition of process + while matches!(get_status(), Ok(ProcessStatus::Running)) { + std::thread::sleep(Duration::from_millis(100)); + } + + // Now that the process is no longer running, send message to threads + STOP_THREADS.store(true, Ordering::Relaxed); + + // While threads still cleaning up + while either_thread_still_running() { + std::thread::sleep(Duration::from_millis(100)); + } + + let child_stdout_monitor = child_stdout_monitor.join(); + let child_stderr_monitor = child_stderr_monitor.join(); + + let status = match get_status() { + Ok(ProcessStatus::Exited(status)) => status, + _ => return Err(eyre!("Failed to retrieve exit status!")), + }; + + let exit_code = status + .code() + .ok_or_else(|| eyre!("Failed to retrieve exit status code!"))?; + + match (&child_stdout_monitor, &child_stderr_monitor) { + (Ok(_), Ok(_)) => { + trace!("Execution completed and monitor threads shut down.") + } + _ => { + error!( + "Error shutting down monitor threads. \n{:?} | {:?}", + &child_stdout_monitor, &child_stderr_monitor + ) + } + } + + let duration = start.elapsed(); + + trace!("Command completed in {} ms", duration.as_millis()); + + let _metadata = Metadata::LinuxMemfaultWatch { + cmdline: args.command, + exit_code, + duration, + stdio_log_file_name, + compression: CompressionAlgorithm::Zlib, + }; + + if !status.success() { + info!("Command failed with exit code {exit_code}!"); + let client = MemfaultdClient::from_config(&config) + .map_err(|report| eyre!("Failed to create Memfaultd client from config! {report}"))?; + + if client.notify_crash().is_err() { + error!("Unable to contact memfaultd. Is it running?"); + } + } + + Ok(exit_code) +} + +/// Utilizes the WrappedArgs which provide version information +pub fn from_env() -> T { + argh::from_env::>().0 +} + +enum ProcessStatus { + Running, + Exited(ExitStatus), +} + +use sealed_test::prelude::*; + +#[sealed_test] +fn test_success_propagates() { + let args: MemfaultWatchArgs = MemfaultWatchArgs { + config_file: None, + verbose: false, + command: vec!["ls".into()], + }; + + assert_eq!(run_from_args(args).unwrap(), 0); +} + +#[sealed_test] +fn test_error_propagates() { + let args: MemfaultWatchArgs = MemfaultWatchArgs { + config_file: None, + verbose: false, + command: vec!["bash".into(), "-c".into(), "exit 7".into()], + }; + + assert_eq!(run_from_args(args).unwrap(), 7); +} diff --git a/memfaultd/src/cli/memfaultctl/add_battery_reading.rs b/memfaultd/src/cli/memfaultctl/add_battery_reading.rs new file mode 100644 index 0000000..24c8d50 --- /dev/null +++ b/memfaultd/src/cli/memfaultctl/add_battery_reading.rs @@ -0,0 +1,19 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::config::Config; +use eyre::{eyre, Result}; + +use crate::cli::memfaultd_client::MemfaultdClient; + +pub fn add_battery_reading(config: &Config, reading_string: &str) -> Result<()> { + let client = MemfaultdClient::from_config(config)?; + + match client.add_battery_reading(reading_string) { + Ok(()) => { + eprintln!("Successfully published battery reading to memfaultd"); + Ok(()) + } + Err(e) => Err(eyre!("add-battery-reading failed: {:#}", e)), + } +} diff --git a/memfaultd/src/cli/memfaultctl/config_file.rs b/memfaultd/src/cli/memfaultctl/config_file.rs new file mode 100644 index 0000000..236858f --- /dev/null +++ b/memfaultd/src/cli/memfaultctl/config_file.rs @@ -0,0 +1,176 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Cli commands for modifying the Memfaultd config file. + +use crate::util::string::capitalize; +use crate::{config::Config, service_manager::MemfaultdServiceManager}; + +use eyre::Result; +use urlencoding::encode; + +/// Set the developer mode flag in the config file and restart memfaultd. +pub fn set_developer_mode( + config: &mut Config, + service_manager: &impl MemfaultdServiceManager, + enable_dev_mode: bool, +) -> Result<()> { + let already_set = check_already_set( + "developer mode", + config.config_file.enable_dev_mode, + enable_dev_mode, + ); + + print_server_side_developer_mode_url(config); + + if already_set { + return Ok(()); + } + + config.config_file.enable_dev_mode = enable_dev_mode; + write_bool_to_config_and_restart_memfaultd( + config, + "enable_dev_mode", + enable_dev_mode, + service_manager, + ) +} + +fn print_server_side_developer_mode_url(config: &Config) { + let device_serial = encode(&config.device_info.device_id); + let project_key = encode(&config.config_file.project_key); + println!( + "⚠️ Enable 'server-side developer mode' to bypass rate limits in Memfault cloud:\n\ + https://mflt.io/developer-mode?d={device_serial}&p={project_key}" + ); +} + +/// Set the data collection flag in the config file and restart memfaultd. +pub fn set_data_collection( + config: &mut Config, + service_manager: &impl MemfaultdServiceManager, + enable_data_collection: bool, +) -> Result<()> { + if check_already_set( + "data collection", + config.config_file.enable_data_collection, + enable_data_collection, + ) { + return Ok(()); + } + + config.config_file.enable_data_collection = enable_data_collection; + write_bool_to_config_and_restart_memfaultd( + config, + "enable_data_collection", + enable_data_collection, + service_manager, + ) +} + +fn check_already_set(module: &str, config_val: bool, new_value: bool) -> bool { + let is_set = config_val == new_value; + if is_set { + let enable_string = if new_value { "enabled" } else { "disabled" }; + println!("{} is already {}", capitalize(module), enable_string); + } else { + let enable_string = if new_value { "Enabling" } else { "Disabling" }; + println!("{} {}", enable_string, module); + } + + is_set +} + +fn write_bool_to_config_and_restart_memfaultd( + config: &Config, + key: &str, + value: bool, + service_manager: &impl MemfaultdServiceManager, +) -> Result<()> { + config + .config_file + .set_and_write_bool_to_runtime_config(key, value)?; + + service_manager.restart_memfaultd_if_running() +} + +#[cfg(test)] +mod test { + use super::*; + + use crate::{service_manager::MockMemfaultdServiceManager, util::path::AbsolutePath}; + + use rstest::rstest; + use tempfile::tempdir; + + struct TestContext { + config: Config, + mock_service_manager: MockMemfaultdServiceManager, + _tmpdir: tempfile::TempDir, + } + + impl TestContext { + fn new() -> Self { + let mut config = Config::test_fixture(); + let mut mock_service_manager = MockMemfaultdServiceManager::new(); + + let tmpdir = tempdir().unwrap(); + config.config_file.persist_dir = + AbsolutePath::try_from(tmpdir.path().to_path_buf()).unwrap(); + + mock_service_manager + .expect_restart_memfaultd_if_running() + .returning(|| Ok(())); + + Self { + config, + mock_service_manager, + _tmpdir: tmpdir, + } + } + } + + #[rstest] + #[case(true, false)] + #[case(false, true)] + #[case(false, false)] + #[case(true, true)] + fn test_set_developer_mode(#[case] enable_developer_mode: bool, #[case] initial_state: bool) { + let mut test_context = TestContext::new(); + test_context.config.config_file.enable_dev_mode = initial_state; + + set_developer_mode( + &mut test_context.config, + &test_context.mock_service_manager, + enable_developer_mode, + ) + .unwrap(); + + assert_eq!( + test_context.config.config_file.enable_dev_mode, + enable_developer_mode, + ); + } + + #[rstest] + #[case(true, false)] + #[case(false, true)] + #[case(false, false)] + #[case(true, true)] + fn test_set_data_collection(#[case] enable_data_collection: bool, #[case] initial_state: bool) { + let mut test_context = TestContext::new(); + test_context.config.config_file.enable_data_collection = initial_state; + + set_data_collection( + &mut test_context.config, + &test_context.mock_service_manager, + enable_data_collection, + ) + .unwrap(); + + assert_eq!( + test_context.config.config_file.enable_data_collection, + enable_data_collection, + ); + } +} diff --git a/memfaultd/src/cli/memfaultctl/coredump.rs b/memfaultd/src/cli/memfaultctl/coredump.rs new file mode 100644 index 0000000..0f5fe20 --- /dev/null +++ b/memfaultd/src/cli/memfaultctl/coredump.rs @@ -0,0 +1,87 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use argh::FromArgValue; +use eyre::Result; + +use crate::config::Config; + +/// Strategy to trigger a crash. +/// +/// This is used to test the crash reporting functionality. +#[derive(Debug)] +pub enum ErrorStrategy { + SegFault, + FPException, +} + +impl FromArgValue for ErrorStrategy { + fn from_arg_value(value: &str) -> Result { + match value { + "segfault" => Ok(ErrorStrategy::SegFault), + "divide-by-zero" => Ok(ErrorStrategy::FPException), + _ => Err("valid strategies are 'segfault' and 'divide-by-zero'".to_string()), + } + } +} + +pub fn trigger_coredump(config: &Config, error_type: ErrorStrategy) -> Result<()> { + trigger_coredump_inner(config, error_type) +} + +#[cfg(feature = "coredump")] +fn trigger_coredump_inner(config: &Config, error_type: ErrorStrategy) -> Result<()> { + use crate::util::ipc::send_flush_signal; + + trigger_crash(error_type)?; + + if config.config_file.enable_dev_mode { + println!("Signaling memfaultd to upload coredump event..."); + + // Give the kernel and memfault-core-handler time to process the coredump + std::thread::sleep(std::time::Duration::from_secs(3)); + + send_flush_signal()?; + } + + Ok(()) +} + +#[cfg(not(feature = "coredump"))] +fn trigger_coredump_inner(_config: &Config, _error_type: ErrorStrategy) -> Result<()> { + println!( + "You must enable the coredump feature when building memfault SDK to report coredumps." + ); + + Ok(()) +} + +#[cfg(feature = "coredump")] +fn trigger_crash(error_type: ErrorStrategy) -> Result<()> { + use memfaultc_sys::coredump::memfault_trigger_fp_exception; + use nix::unistd::{fork, ForkResult}; + + println!("Triggering coredump ..."); + match unsafe { fork() } { + Ok(ForkResult::Parent { .. }) => Ok(()), + Ok(ForkResult::Child) => { + match error_type { + ErrorStrategy::FPException => { + // This still needs to be implemented in C because Rust automatically + // generates code to prevent divide-by-zero errors. This can be moved to Rust when + // [unchecked_div](https://doc.rust-lang.org/std/intrinsics/fn.unchecked_div.html) + // is stabilized. + unsafe { + memfault_trigger_fp_exception(); + } + } + ErrorStrategy::SegFault => { + unsafe { std::ptr::null_mut::().write(42) }; + } + } + + unreachable!("Child process should have crashed"); + } + Err(e) => Err(eyre::eyre!("Failed to fork process: {}", e)), + } +} diff --git a/memfaultd/src/cli/memfaultctl/export.rs b/memfaultd/src/cli/memfaultctl/export.rs new file mode 100644 index 0000000..831eb1a --- /dev/null +++ b/memfaultd/src/cli/memfaultctl/export.rs @@ -0,0 +1,54 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{copy, BufWriter}; + +use crate::config::Config; +use eyre::{eyre, Context, Result}; + +use crate::cli::memfaultctl::ExportArgs; + +use crate::cli::memfaultd_client::{ExportDeleteResponse, ExportGetResponse, MemfaultdClient}; + +pub fn export(config: &Config, args: &ExportArgs) -> Result<()> { + let client = MemfaultdClient::from_config(config)?; + + let delete_token = match client + .export_get(&args.format) + .wrap_err("Unable to fetch latest export")? + { + ExportGetResponse::NoData => { + eprintln!("Nothing to export right now. You may want to try `memfaultctl sync`."); + return Ok(()); + } + ExportGetResponse::Data { + delete_token, + mut data, + } => { + let mut file = BufWriter::new(args.output.get_output_stream()?); + copy(&mut data, &mut file).wrap_err("Unable to write server response")?; + delete_token + } + }; + + if !args.do_not_delete { + match client + .export_delete(delete_token) + .wrap_err("Error while deleting data")? + { + ExportDeleteResponse::Ok => { + eprintln!("Export saved and data cleared from memfaultd."); + Ok(()) + } + ExportDeleteResponse::ErrorWrongDeleteToken => { + Err(eyre!("Unexpected response: wrong hash")) + } + ExportDeleteResponse::Error404 => { + Err(eyre!("Unexpected response: 404 (no data to delete)")) + } + } + } else { + eprintln!("Export saved. Data preserved in memfaultd."); + Ok(()) + } +} diff --git a/memfaultd/src/cli/memfaultctl/mod.rs b/memfaultd/src/cli/memfaultctl/mod.rs new file mode 100644 index 0000000..0d0be11 --- /dev/null +++ b/memfaultd/src/cli/memfaultctl/mod.rs @@ -0,0 +1,322 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use argh::{FromArgs, TopLevelCommand}; +use std::{path::Path, str::FromStr}; + +mod add_battery_reading; +mod config_file; +mod coredump; +mod export; +mod report_sync; +mod session; +mod sync; +mod write_attributes; + +use crate::{ + cli::version::format_version, + mar::{DeviceAttribute, ExportFormat, Metadata}, + metrics::{KeyedMetricReading, SessionName}, + reboot::{write_reboot_reason_and_reboot, RebootReason}, + service_manager::get_service_manager, +}; +use crate::{mar::MarEntryBuilder, util::output_arg::OutputArg}; + +use crate::cli::init_logger; +use crate::cli::memfaultctl::add_battery_reading::add_battery_reading; +use crate::cli::memfaultctl::config_file::{set_data_collection, set_developer_mode}; +use crate::cli::memfaultctl::coredump::{trigger_coredump, ErrorStrategy}; +use crate::cli::memfaultctl::export::export; +use crate::cli::memfaultctl::report_sync::report_sync; +use crate::cli::memfaultctl::sync::sync; +use crate::cli::show_settings::show_settings; +use crate::config::Config; +use crate::network::NetworkConfig; +use eyre::{eyre, Context, Result}; +use log::LevelFilter; + +use self::session::{end_session, start_session}; + +#[derive(FromArgs)] +/// A command line utility to adjust memfaultd configuration and trigger specific events for +/// testing purposes. For further reference, see: +/// https://docs.memfault.com/docs/linux/reference-memfaultctl-cli +struct MemfaultctlArgs { + #[argh(subcommand)] + command: MemfaultctlCommand, + + /// use configuration file + #[argh(option, short = 'c')] + config_file: Option, + + /// show version information + #[argh(switch, short = 'v')] + #[allow(dead_code)] + version: bool, + + /// verbose output + #[argh(switch, short = 'V')] + verbose: bool, +} + +/// Wrapper around argh to support flags acting as subcommands, like --version. +/// Inspired by https://gist.github.com/suluke/e0c672492126be0a4f3b4f0e1115d77c +pub struct WrappedArgs(pub T); +impl TopLevelCommand for WrappedArgs {} +impl FromArgs for WrappedArgs { + fn from_args(command_name: &[&str], args: &[&str]) -> Result { + /// Pseudo subcommands that look like flags. + #[derive(FromArgs)] + struct CommandlikeFlags { + /// show version information + #[argh(switch, short = 'v')] + version: bool, + } + + match CommandlikeFlags::from_args(command_name, args) { + Ok(CommandlikeFlags { version: true }) => Err(argh::EarlyExit { + output: format_version(), + status: Ok(()), + }), + _ => T::from_args(command_name, args).map(Self), + } + } +} + +pub fn from_env() -> T { + argh::from_env::>().0 +} + +#[derive(FromArgs)] +#[argh(subcommand)] +enum MemfaultctlCommand { + EnableDataCollection(EnableDataCollectionArgs), + DisableDataCollection(DisableDataCollectionArgs), + EnableDevMode(EnableDevModeArgs), + DisableDevMode(DisableDevModeArgs), + Export(ExportArgs), + Reboot(RebootArgs), + RequestMetrics(RequestMetricsArgs), + ShowSettings(ShowSettingsArgs), + Synchronize(SyncArgs), + TriggerCoredump(TriggerCoredumpArgs), + WriteAttributes(WriteAttributesArgs), + AddBatteryReading(AddBatteryReadingArgs), + ReportSyncSuccess(ReportSyncSuccessArgs), + ReportSyncFailure(ReportSyncFailureArgs), + StartSession(StartSessionArgs), + EndSession(EndSessionArgs), +} + +#[derive(FromArgs)] +/// enable data collection and restart memfaultd +#[argh(subcommand, name = "enable-data-collection")] +struct EnableDataCollectionArgs {} + +#[derive(FromArgs)] +/// disable data collection and restart memfaultd +#[argh(subcommand, name = "disable-data-collection")] +struct DisableDataCollectionArgs {} + +#[derive(FromArgs)] +/// enable developer mode and restart memfaultd +#[argh(subcommand, name = "enable-dev-mode")] +struct EnableDevModeArgs {} + +#[derive(FromArgs)] +/// disable developer mode and restart memfaultd +#[argh(subcommand, name = "disable-dev-mode")] +struct DisableDevModeArgs {} + +#[derive(FromArgs)] +/// export (and delete) memfault data +#[argh(subcommand, name = "export")] +pub struct ExportArgs { + #[argh(switch, short = 'n')] + /// do not delete the data from memfault mar_staging + do_not_delete: bool, + #[argh(option, short = 'o')] + /// where to write the MAR data (or '-' for standard output) + output: OutputArg, + + #[argh(option, short = 'f', default = "ExportFormat::Mar")] + /// output format (mar, chunk or chunk-wrapped) + format: ExportFormat, +} + +#[derive(FromArgs)] +/// register reboot reason and call 'reboot' +#[argh(subcommand, name = "reboot")] +struct RebootArgs { + /// a reboot reason ID from https://docs.memfault.com/docs/platform/reference-reboot-reason-ids + #[argh(option)] + reason: String, +} + +#[derive(FromArgs)] +/// flush collectd metrics to Memfault now +#[argh(subcommand, name = "request-metrics")] +struct RequestMetricsArgs {} + +#[derive(FromArgs)] +/// show memfaultd settings +#[argh(subcommand, name = "show-settings")] +struct ShowSettingsArgs {} + +#[derive(FromArgs)] +/// Upload all pending data to Memfault now +#[argh(subcommand, name = "sync")] +struct SyncArgs {} + +#[derive(FromArgs)] +/// trigger a coredump and immediately reports it to Memfault (defaults to segfault) +#[argh(subcommand, name = "trigger-coredump")] +struct TriggerCoredumpArgs { + /// a strategy, either 'segfault' or 'divide-by-zero' + #[argh(positional, default = "ErrorStrategy::SegFault")] + strategy: ErrorStrategy, +} + +#[derive(FromArgs)] +/// write device attribute(s) to memfaultd +#[argh(subcommand, name = "write-attributes")] +struct WriteAttributesArgs { + /// attributes to write, in the format + #[argh(positional)] + attributes: Vec, +} + +#[derive(FromArgs)] +/// add a reading to memfaultd's battery metrics in format "[status string]:[0.0-100.0]". +#[argh(subcommand, name = "add-battery-reading")] +struct AddBatteryReadingArgs { + // Valid status strings are "Charging", "Not charging", "Discharging", "Unknown", and "Full" + // These are based off the values that can appear in /sys/class/power_supply//status + // See: https://www.kernel.org/doc/Documentation/ABI/testing/sysfs-class-power + #[argh(positional)] + reading_string: String, +} + +#[derive(FromArgs)] +/// Report a successful sync for connectivity metrics +#[argh(subcommand, name = "report-sync-success")] +struct ReportSyncSuccessArgs {} + +#[derive(FromArgs)] +/// Report a failed sync for connectivity metrics +#[argh(subcommand, name = "report-sync-failure")] +struct ReportSyncFailureArgs {} + +#[derive(FromArgs)] +/// Begin a session and start capturing metrics for it +#[argh(subcommand, name = "start-session")] +struct StartSessionArgs { + // session name (needs to be defined in memfaultd.conf) + #[argh(positional)] + session_name: SessionName, + // List of metric key value pairs to write in the format + #[argh(positional)] + readings: Vec, +} + +#[derive(FromArgs)] +/// End a session and dump its metrics to MAR staging directory +#[argh(subcommand, name = "end-session")] +struct EndSessionArgs { + // session name (needs to be defined in memfaultd.conf) + #[argh(positional)] + session_name: SessionName, + // List of metric key value pairs to write in the format + #[argh(positional)] + readings: Vec, +} + +fn check_data_collection_enabled(config: &Config, do_what: &str) -> Result<()> { + match config.config_file.enable_data_collection { + true => Ok(()), + false => { + let msg = format!( + "Cannot {} because data collection is disabled. \ + Hint: enable it with 'memfaultctl enable-data-collection'.", + do_what + ); + Err(eyre!(msg)) + } + } +} + +pub fn main() -> Result<()> { + let args: MemfaultctlArgs = from_env(); + + init_logger(if args.verbose { + LevelFilter::Trace + } else { + LevelFilter::Info + }); + + let config_path = args.config_file.as_ref().map(Path::new); + let mut config = Config::read_from_system(config_path)?; + let network_config = NetworkConfig::from(&config); + let mar_staging_path = config.mar_staging_path(); + + let service_manager = get_service_manager(); + + match args.command { + MemfaultctlCommand::EnableDataCollection(_) => { + set_data_collection(&mut config, &service_manager, true) + } + MemfaultctlCommand::DisableDataCollection(_) => { + set_data_collection(&mut config, &service_manager, false) + } + MemfaultctlCommand::EnableDevMode(_) => { + set_developer_mode(&mut config, &service_manager, true) + } + MemfaultctlCommand::DisableDevMode(_) => { + set_developer_mode(&mut config, &service_manager, false) + } + MemfaultctlCommand::Export(args) => export(&config, &args).wrap_err("Error exporting data"), + MemfaultctlCommand::Reboot(args) => { + let reason = RebootReason::from_str(&args.reason) + .wrap_err(eyre!("Failed to parse {}", args.reason))?; + println!("Rebooting with reason {:?}", reason); + write_reboot_reason_and_reboot( + &config.config_file.reboot.last_reboot_reason_file, + reason, + ) + } + MemfaultctlCommand::RequestMetrics(_) => sync(), + MemfaultctlCommand::ShowSettings(_) => show_settings(config_path), + MemfaultctlCommand::Synchronize(_) => sync(), + MemfaultctlCommand::TriggerCoredump(TriggerCoredumpArgs { strategy }) => { + trigger_coredump(&config, strategy) + } + MemfaultctlCommand::WriteAttributes(WriteAttributesArgs { attributes }) => { + // argh does not have a way to specify the minimum number of repeating arguments, so check here: + // https://github.com/google/argh/issues/110 + if attributes.is_empty() { + Err(eyre!( + "No attributes given. Please specify them as KEY=VALUE pairs." + )) + } else { + check_data_collection_enabled(&config, "write attributes")?; + MarEntryBuilder::new(&mar_staging_path)? + .set_metadata(Metadata::new_device_attributes(attributes)) + .save(&network_config) + .map(|_entry| ()) + } + } + MemfaultctlCommand::AddBatteryReading(AddBatteryReadingArgs { reading_string }) => { + add_battery_reading(&config, &reading_string) + } + MemfaultctlCommand::ReportSyncSuccess(_) => report_sync(&config, true), + MemfaultctlCommand::ReportSyncFailure(_) => report_sync(&config, false), + MemfaultctlCommand::StartSession(StartSessionArgs { + session_name, + readings, + }) => start_session(&config, session_name, readings), + MemfaultctlCommand::EndSession(EndSessionArgs { + session_name, + readings, + }) => end_session(&config, session_name, readings), + } +} diff --git a/memfaultd/src/cli/memfaultctl/report_sync.rs b/memfaultd/src/cli/memfaultctl/report_sync.rs new file mode 100644 index 0000000..199653f --- /dev/null +++ b/memfaultd/src/cli/memfaultctl/report_sync.rs @@ -0,0 +1,25 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::config::Config; +use eyre::{eyre, Result}; + +use crate::cli::memfaultd_client::MemfaultdClient; + +pub fn report_sync(config: &Config, success: bool) -> Result<()> { + let client = MemfaultdClient::from_config(config)?; + let status = if success { "successful" } else { "failed" }; + let command_string = if success { + "report-sync-success " + } else { + "report-sync-failure" + }; + + match client.report_sync(success) { + Ok(()) => { + eprintln!("Reported a {} sync to memfaultd", status); + Ok(()) + } + Err(e) => Err(eyre!("{} failed: {:#}", command_string, e)), + } +} diff --git a/memfaultd/src/cli/memfaultctl/session.rs b/memfaultd/src/cli/memfaultctl/session.rs new file mode 100644 index 0000000..96c2f40 --- /dev/null +++ b/memfaultd/src/cli/memfaultctl/session.rs @@ -0,0 +1,54 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::{eyre, Result}; + +use crate::config::Config; +use crate::{ + cli::memfaultd_client::MemfaultdClient, + metrics::{KeyedMetricReading, SessionName}, +}; + +pub fn start_session( + config: &Config, + session_name: SessionName, + metric_readings: Vec, +) -> Result<()> { + let client = MemfaultdClient::from_config(config)?; + if config.config_file.enable_data_collection { + match client.start_session(session_name.clone(), metric_readings) { + Ok(()) => { + eprintln!("Started new {} session", session_name); + Ok(()) + } + Err(e) => Err(eyre!("start-session failed: {:?}", e)), + } + } else { + Err(eyre!( + "Cannot start session with data collection disabled.\n\ + You can enable data collection with \"memfaultctl enable-data-collection\"" + )) + } +} + +pub fn end_session( + config: &Config, + session_name: SessionName, + readings: Vec, +) -> Result<()> { + let client = MemfaultdClient::from_config(config)?; + if config.config_file.enable_data_collection { + match client.end_session(session_name.clone(), readings) { + Ok(()) => { + eprintln!("Ended ongoing {} session", session_name); + Ok(()) + } + Err(e) => Err(eyre!("end-session failed: {:?}", e)), + } + } else { + Err(eyre!( + "Cannot end session with data collection disabled.\n\ + You can enable data collection with \"memfaultctl enable-data-collection\"" + )) + } +} diff --git a/memfaultd/src/cli/memfaultctl/sync.rs b/memfaultd/src/cli/memfaultctl/sync.rs new file mode 100644 index 0000000..87bb40d --- /dev/null +++ b/memfaultd/src/cli/memfaultctl/sync.rs @@ -0,0 +1,18 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::{eyre, Result}; + +use crate::util::ipc::send_flush_signal; + +pub fn sync() -> Result<()> { + match send_flush_signal() { + Ok(()) => Ok(()), + Err(e) => Err(eyre!( + "Error: {} If you are not running memfaultd as a daemon you \ + can force it to sync data with \ + 'killall -USR1 memfaultd'.", + e + )), + } +} diff --git a/memfaultd/src/cli/memfaultctl/write_attributes.rs b/memfaultd/src/cli/memfaultctl/write_attributes.rs new file mode 100644 index 0000000..b213c21 --- /dev/null +++ b/memfaultd/src/cli/memfaultctl/write_attributes.rs @@ -0,0 +1,82 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::mar::DeviceAttribute; +use crate::metrics::MetricStringKey; +use argh::FromArgValue; +use serde_json::Value; +use std::str::FromStr; + +impl FromArgValue for DeviceAttribute { + fn from_arg_value(value: &str) -> Result { + let (key, value_str) = value + .split_once('=') + .ok_or("Each attribute should be specified as KEY=VALUE")?; + + // Let's ensure the key is valid first: + let metric_key = MetricStringKey::from_str(key)?; + let value = match serde_json::from_str::(value_str) { + Ok(value) => { + if value.is_array() || value.is_object() { + return Err("Invalid value: arrays or objects are not allowed".to_string()); + } + value + } + // Default to string value: + Err(_) => value_str.into(), + }; + Ok(DeviceAttribute::new(metric_key, value)) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case("")] + #[case("KEY")] + #[case("KEY:VALUE")] + fn split_failed(#[case] input: &str) { + assert!(DeviceAttribute::from_arg_value(input) + .err() + .unwrap() + .contains("Each attribute should be specified as KEY=VALUE")); + } + + #[rstest] + fn invalid_key() { + assert_eq!( + DeviceAttribute::from_arg_value("\u{1F4A9}=smelly") + .err() + .unwrap(), + "Invalid key: must be ASCII" + ); + } + + #[rstest] + #[case("key=[]")] + #[case("key={}")] + fn invalid_value(#[case] input: &str) { + assert_eq!( + DeviceAttribute::from_arg_value(input).err().unwrap(), + "Invalid value: arrays or objects are not allowed" + ); + } + + #[rstest] + #[case("key=", ("key", "").try_into())] + #[case("key=my_string", ("key", "my_string").try_into())] + #[case("key=123", ("key", 123).try_into())] + #[case("key=123.456", ("key", 123.456).try_into())] + #[case("key=true", ("key", true).try_into())] + #[case("key=false", ("key", false).try_into())] + #[case("key=\"false\"", ("key", "false").try_into())] + #[case("key=\"[]\"", ("key", "[]").try_into())] + #[case("key=\"{}\"", ("key", "{}").try_into())] + fn parsed_ok(#[case] input: &str, #[case] expected: Result) { + assert_eq!(DeviceAttribute::from_arg_value(input), expected); + } +} diff --git a/memfaultd/src/cli/memfaultd.rs b/memfaultd/src/cli/memfaultd.rs new file mode 100644 index 0000000..f73db2c --- /dev/null +++ b/memfaultd/src/cli/memfaultd.rs @@ -0,0 +1,154 @@ +#![allow(clippy::print_stdout, clippy::print_stderr)] +// +// Copyright (c) Memfault, Inc. +// See License.txt for details + +use std::{ + env::args_os, fs::create_dir_all, os::unix::process::CommandExt, path::Path, process::Command, +}; + +use crate::{ + config::Config, + memfaultd::{memfaultd_loop, MemfaultLoopResult}, + util::pid_file::{get_pid_from_file, is_pid_file_about_me, remove_pid_file, write_pid_file}, +}; + +use eyre::{eyre, Context, Result}; +use log::{error, info, warn, LevelFilter}; + +use crate::cli::show_settings::show_settings; +use crate::cli::version::format_version; +use argh::FromArgs; + +use super::init_logger; + +#[derive(FromArgs)] +/// Memfault daemon. +struct MemfaultDArgs { + /// use configuration file + #[argh(option, short = 'c')] + config_file: Option, + + #[argh(switch, short = 's')] + /// show settings and exit immediately + show_settings: bool, + + #[argh(switch, short = 'Z')] + /// daemonize (fork to background) + daemonize: bool, + + #[argh(switch, short = 'v')] + /// show version + version: bool, + + #[argh(switch, short = 'V')] + /// verbose output + verbose: bool, + + #[argh(switch, short = 'q')] + /// quiet - no output + quiet: bool, +} + +pub fn main() -> Result<()> { + let args: MemfaultDArgs = argh::from_env(); + let config_path = args.config_file.as_ref().map(Path::new); + + init_logger(match (args.quiet, args.verbose) { + (true, _) => LevelFilter::Off, + (false, true) => LevelFilter::Trace, + _ => LevelFilter::Info, + }); + + if args.version { + println!("{}", format_version()); + return Ok(()); + } + + let config = + Config::read_from_system(config_path).wrap_err(eyre!("Unable to load configuration"))?; + + // Create directories early so we can fail early if we can't create them. + mkdir_if_needed(&config.config_file.persist_dir)?; + mkdir_if_needed(&config.tmp_dir())?; + + // Always show settings when starting + show_settings(config_path)?; + + if args.show_settings { + // Already printed above. We're done. + return Ok(()); + } + + if !args.daemonize && get_pid_from_file().is_ok() { + return Err(eyre!("memfaultd is already running")); + } + + if config.config_file.enable_dev_mode { + info!("memfaultd:: Starting with developer mode enabled"); + } + if !config.config_file.enable_data_collection { + warn!("memfaultd:: Starting with data collection DISABLED"); + } + + #[cfg(feature = "swupdate")] + { + use crate::swupdate::generate_swupdate_config; + + generate_swupdate_config(&config)?; + } + #[cfg(feature = "coredump")] + { + use crate::coredump::coredump_configure_kernel; + if config.config_file.enable_data_collection { + coredump_configure_kernel(&config.config_file_path)?; + } + } + + // Only daemonize when asked to AND not already running (aka don't fork when reloading) + let need_daemonize = args.daemonize && !is_pid_file_about_me(); + if need_daemonize { + daemonize()?; + } + + let result = memfaultd_loop(config, || { + if need_daemonize { + // All subcomponents are ready, write the pid file now to indicate we've started up completely. + write_pid_file()?; + } + Ok(()) + })?; + if result == MemfaultLoopResult::Relaunch { + // If reloading the config, execv ourselves (replace our program in memory by a new copy of us) + let mut args = args_os().collect::>(); + let arg0 = args.remove(0); + + let err = Command::new(arg0).args(&args).exec(); + // This next line will only be executed if we failed to exec(). + error!("Unable to restart {:?}: {:?}", args, err); + }; + + if args.daemonize { + remove_pid_file()? + } + Ok(()) +} + +fn mkdir_if_needed(path: &Path) -> Result<()> { + if path.exists() && path.is_dir() { + return Ok(()); + } + create_dir_all(path).wrap_err(eyre!("Unable to create directory {}", path.display())) +} + +fn daemonize() -> Result<()> { + #[cfg(target_os = "linux")] + { + nix::unistd::daemon(true, true).wrap_err("Unable to daemonize") + } + #[cfg(not(target_os = "linux"))] + { + warn!("Daemonizing is not supported on this platform"); + Ok(()) + } +} diff --git a/memfaultd/src/cli/memfaultd_client.rs b/memfaultd/src/cli/memfaultd_client.rs new file mode 100644 index 0000000..3afc538 --- /dev/null +++ b/memfaultd/src/cli/memfaultd_client.rs @@ -0,0 +1,206 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{io::Read, str::from_utf8, time::Duration}; + +use eyre::{eyre, Context, Result}; +use reqwest::{blocking::Client, header::ACCEPT, StatusCode}; + +use crate::{ + config::Config, + http_server::SessionRequest, + mar::{ExportFormat, EXPORT_MAR_URL}, + metrics::{KeyedMetricReading, SessionName}, +}; + +/// Client to Memfaultd localhost HTTP API +pub struct MemfaultdClient { + base_url: String, + client: Client, +} + +pub struct DeleteToken(String); + +pub enum ExportGetResponse { + Data { + delete_token: DeleteToken, + data: Box, + }, + NoData, +} + +pub enum ExportDeleteResponse { + Ok, + ErrorWrongDeleteToken, + Error404, +} + +impl MemfaultdClient { + pub fn from_config(config: &Config) -> Result { + Ok(MemfaultdClient { + client: Client::builder().timeout(Duration::from_secs(10)).build()?, + base_url: format!("http://{}", config.config_file.http_server.bind_address), + }) + } + + pub fn export_get(&self, format: &ExportFormat) -> Result { + let r = self + .client + .get(format!("{}{}", self.base_url, EXPORT_MAR_URL)) + .header(ACCEPT, format.to_content_type()) + .send() + .wrap_err_with(|| { + eyre!(format!( + "Error fetching {}/{}", + self.base_url, EXPORT_MAR_URL + )) + })?; + match r.status() { + StatusCode::OK => Ok(ExportGetResponse::Data { + delete_token: DeleteToken( + r.headers() + .iter() + .find(|h| h.0.as_str() == "etag") + .ok_or(eyre!("No ETag header included on the response")) + .map(|etag| etag.1.to_str())?? + .trim_matches('"') + .to_owned(), + ), + data: Box::new(r), + }), + StatusCode::NO_CONTENT => Ok(ExportGetResponse::NoData), + StatusCode::NOT_ACCEPTABLE => Err(eyre!("Requested format not supported")), + _ => Err(eyre!("Unexpected status code {}", r.status().as_u16())), + } + } + + pub fn export_delete(&self, delete_token: DeleteToken) -> Result { + let r = self + .client + .delete(format!("{}{}", self.base_url, EXPORT_MAR_URL)) + .header("If-Match", delete_token.0) + .send()?; + match r.status() { + StatusCode::NO_CONTENT => Ok(ExportDeleteResponse::Ok), + StatusCode::PRECONDITION_FAILED => Ok(ExportDeleteResponse::ErrorWrongDeleteToken), + StatusCode::NOT_FOUND => Ok(ExportDeleteResponse::Error404), + _ => Err(eyre!(format!( + "Unexpected status code {}", + r.status().as_u16() + ))), + } + } + + pub fn add_battery_reading(&self, battery_reading_string: &str) -> Result<()> { + let r = self + .client + .post(format!("{}{}", self.base_url, "/v1/battery/add_reading")) + .body(battery_reading_string.to_string()) + .send()?; + match r.status() { + StatusCode::OK => Ok(()), + _ => Err(eyre!( + "Unexpected status code {}: {}", + r.status().as_u16(), + from_utf8(&r.bytes()?)? + )), + } + } + + pub fn notify_crash(&self) -> Result<()> { + self.client + .post(format!("{}{}", self.base_url, "/v1/crash/report")) + .send()?; + Ok(()) + } + + pub fn report_sync(&self, success: bool) -> Result<()> { + let path = if success { + "/v1/sync/success" + } else { + "/v1/sync/failure" + }; + let r = self + .client + .post(format!("{}{}", self.base_url, path)) + .send()?; + match r.status() { + StatusCode::OK => Ok(()), + _ => Err(eyre!( + "Unexpected status code {}: {}", + r.status().as_u16(), + from_utf8(&r.bytes()?)? + )), + } + } + + pub fn start_session( + &self, + session_name: SessionName, + readings: Vec, + ) -> Result<()> { + let body = if readings.is_empty() { + session_name.to_string() + } else { + serde_json::to_string(&SessionRequest::new(session_name, readings))? + }; + let r = self + .client + .post(format!("{}/v1/session/start", self.base_url)) + .body(body) + .send()?; + match r.status() { + StatusCode::OK => Ok(()), + _ => Err(eyre!( + "Unexpected status code {}: {}", + r.status().as_u16(), + from_utf8(&r.bytes()?)? + )), + } + } + + pub fn end_session( + &self, + session_name: SessionName, + readings: Vec, + ) -> Result<()> { + let body = if readings.is_empty() { + session_name.to_string() + } else { + serde_json::to_string(&SessionRequest::new(session_name, readings))? + }; + let r = self + .client + .post(format!("{}/v1/session/end", self.base_url)) + .body(body) + .send()?; + match r.status() { + StatusCode::OK => Ok(()), + _ => Err(eyre!( + "Unexpected status code {}: {}", + r.status().as_u16(), + from_utf8(&r.bytes()?)? + )), + } + } + + #[cfg(feature = "logging")] + pub fn get_crash_logs(&self) -> Result>> { + use crate::logs::log_collector::{CrashLogs, CRASH_LOGS_URL}; + + let r = self + .client + .get(format!("{}{}", self.base_url, CRASH_LOGS_URL)) + .send()?; + + match r.status() { + StatusCode::OK => Ok(Some(r.json::()?.logs)), + _ => Err(eyre!("Unexpected status code {}", r.status().as_u16())), + } + } + + #[cfg(not(feature = "logging"))] + pub fn get_crash_logs(&self) -> Result>> { + Ok(None) + } +} diff --git a/memfaultd/src/cli/mod.rs b/memfaultd/src/cli/mod.rs new file mode 100644 index 0000000..83c6146 --- /dev/null +++ b/memfaultd/src/cli/mod.rs @@ -0,0 +1,70 @@ +#![allow(clippy::print_stdout, clippy::print_stderr)] +// +// Copyright (c) Memfault, Inc. +// See License.txt for details + +use eyre::eyre; +use log::LevelFilter; +use std::path::Path; +use stderrlog::{LogLevelNum, StdErrLog}; + +#[cfg(all(target_os = "linux", feature = "coredump"))] +mod memfault_core_handler; +#[cfg(feature = "mfw")] +mod memfault_watch; +mod memfaultctl; +mod memfaultd; +mod memfaultd_client; +mod show_settings; +mod version; + +pub use memfaultd_client::*; + +fn build_logger(level: LevelFilter) -> StdErrLog { + let mut log = stderrlog::new(); + + log.module("memfaultd"); + log.verbosity(LogLevelNum::from(level)); + + log +} + +fn init_logger(level: LevelFilter) { + build_logger(level).init().unwrap(); +} + +pub fn main() { + let arg0 = std::env::args().next().unwrap(); + let cmd_name = Path::new(&arg0) + .file_name() + .expect("") + .to_str() + .unwrap(); + + let result = match cmd_name { + #[cfg(all(target_os = "linux", feature = "coredump"))] + "memfault-core-handler" => memfault_core_handler::main(), + #[cfg(not(all(target_os = "linux", feature = "coredump")))] + "memfault-core-handler" => Err(eyre!( + "memfault-core-handler is not supported in this build" + )), + "memfaultctl" => memfaultctl::main(), + "memfaultd" => memfaultd::main(), + #[cfg(feature = "mfw")] + "mfw" => memfault_watch::main(), + #[cfg(not(feature = "mfw"))] + "mfw" => Err(eyre!("Memfault-watch is currently experimental. You must compile with the experimental flag enabled.")), + _ => Err(eyre!( + "Unknown command: {}. Should be memfaultd/memfaultctl/memfault-core-handler.", + cmd_name + )), + }; + + match result { + Ok(_) => (), + Err(e) => { + eprintln!("{:#}", e); + std::process::exit(-1); + } + } +} diff --git a/memfaultd/src/cli/show_settings.rs b/memfaultd/src/cli/show_settings.rs new file mode 100644 index 0000000..0919b0c --- /dev/null +++ b/memfaultd/src/cli/show_settings.rs @@ -0,0 +1,192 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::borrow::Cow; +use std::io::{stdout, Write}; +use std::path::Path; + +use eyre::Result; + +use crate::config::{DeviceInfo, DeviceInfoWarning, JsonConfigs, MemfaultdConfig}; +use crate::{ + build_info::{BUILD_ID, GIT_COMMIT, VERSION}, + config::Config, +}; + +fn dump_config( + writer: &mut impl Write, + configs: &JsonConfigs, + config_path: Option<&Path>, +) -> Result<()> { + let path_str = config_path + .map(Path::display) + .map(|d| Cow::Owned(d.to_string())) + .unwrap_or_else(|| Cow::Borrowed(Config::DEFAULT_CONFIG_PATH)); + writeln!(writer, "Base configuration ({}):", path_str)?; + writeln!(writer, "{}", serde_json::to_string_pretty(&configs.base)?)?; + writeln!(writer)?; + writeln!(writer, "Runtime configuration:")?; + writeln!( + writer, + "{}", + serde_json::to_string_pretty(&configs.runtime)? + )?; + Ok(()) +} + +type Device = (DeviceInfo, Vec); + +fn dump_device_info(writer: &mut impl Write, device: &Device) -> Result<()> { + let (device_info, _warnings) = device; + writeln!(writer, "Device configuration from memfault-device-info:")?; + writeln!(writer, " MEMFAULT_DEVICE_ID={}", device_info.device_id)?; + writeln!( + writer, + " MEMFAULT_HARDWARE_VERSION={}", + device_info.hardware_version + )?; + if let Some(sw_version) = device_info.software_version.as_ref() { + writeln!(writer, " MEMFAULT_SOFTWARE_VERSION={}", sw_version)?; + } + if let Some(sw_type) = device_info.software_type.as_ref() { + writeln!(writer, " MEMFAULT_SOFTWARE_TYPE={}", sw_type)?; + } + + Ok(()) +} + +struct Versions { + version: &'static str, + git_commit: &'static str, + build_id: &'static str, +} + +fn dump_version(writer: &mut impl Write, versions: &Versions) -> Result<()> { + writeln!(writer, "Memfault version:")?; + writeln!(writer, " VERSION={}", versions.version)?; + writeln!(writer, " GIT COMMIT={}", versions.git_commit)?; + writeln!(writer, " BUILD ID={}", versions.build_id)?; + Ok(()) +} + +fn dump_features(writer: &mut impl Write, features: &[&str]) -> Result<()> { + writeln!(writer, "Features enabled:")?; + for feature in features { + writeln!(writer, " {}", feature)?; + } + Ok(()) +} + +fn dump_settings( + writer: &mut impl Write, + configs: &JsonConfigs, + config_path: Option<&Path>, + device: &Device, + versions: &Versions, + features: &[&str], +) -> Result<()> { + dump_config(writer, configs, config_path)?; + writeln!(writer)?; + dump_device_info(writer, device)?; + writeln!(writer)?; + dump_version(writer, versions)?; + writeln!(writer)?; + dump_features(writer, features)?; + writeln!(writer)?; + Ok(()) +} + +pub fn show_settings(config_path: Option<&Path>) -> Result<()> { + let configs = MemfaultdConfig::parse_configs( + config_path.unwrap_or_else(|| Path::new(Config::DEFAULT_CONFIG_PATH)), + )?; + let versions = Versions { + version: VERSION, + git_commit: GIT_COMMIT, + build_id: BUILD_ID, + }; + + let enabled_features = [ + "reboot", + #[cfg(feature = "swupdate")] + "swupdate", + #[cfg(feature = "collectd")] + "collectd", + #[cfg(feature = "coredump")] + "coredump", + #[cfg(feature = "logging")] + "logging", + #[cfg(feature = "log-to-metrics")] + "log-to-metrics", + #[cfg(feature = "systemd")] + "systemd", + ]; + + dump_settings( + &mut stdout(), + &configs, + config_path, + &DeviceInfo::load()?, + &versions, + &enabled_features, + ) +} + +#[cfg(test)] +mod tests { + use std::io::Cursor; + use std::path::PathBuf; + + use insta::assert_snapshot; + use serde_json::json; + + use super::*; + use crate::config::MockDeviceInfoDefaults; + + #[test] + fn test() { + let configs = JsonConfigs { + base: json!({"project_key": "xyz"}), + runtime: json!({"enable_data_collection": true}), + }; + let config_path = PathBuf::from("/etc/memfaultd.conf"); + + let mut di_defaults = MockDeviceInfoDefaults::new(); + di_defaults.expect_software_type().returning(|| Ok(None)); + di_defaults.expect_software_version().returning(|| Ok(None)); + di_defaults + .expect_hardware_version() + .returning(|| Ok("Hardware".into())); + di_defaults + .expect_device_id() + .returning(|| Ok("123ABC".into())); + let device = DeviceInfo::parse( + Some(b"MEMFAULT_DEVICE_ID=X\nMEMFAULT_HARDWARE_VERSION=Y\nblahblahblah\n"), + di_defaults, + ) + .unwrap(); + + let versions = Versions { + version: "1.2.3", + git_commit: "abcdef", + build_id: "123456", + }; + + let enabled_features = ["reboot", "coredump"]; + + let output = Vec::new(); + let mut writer = Cursor::new(output); + dump_settings( + &mut writer, + &configs, + Some(&config_path), + &device, + &versions, + &enabled_features, + ) + .unwrap(); + + let output = String::from_utf8(writer.into_inner()).unwrap(); + assert_snapshot!(output); + } +} diff --git a/memfaultd/src/cli/snapshots/memfaultd__cli__show_settings__tests__test.snap b/memfaultd/src/cli/snapshots/memfaultd__cli__show_settings__tests__test.snap new file mode 100644 index 0000000..cface53 --- /dev/null +++ b/memfaultd/src/cli/snapshots/memfaultd__cli__show_settings__tests__test.snap @@ -0,0 +1,26 @@ +--- +source: memfaultd/src/cli/show_settings.rs +expression: output +--- +Base configuration (/etc/memfaultd.conf): +{ + "project_key": "xyz" +} + +Runtime configuration: +{ + "enable_data_collection": true +} + +Device configuration from memfault-device-info: + MEMFAULT_DEVICE_ID=X + MEMFAULT_HARDWARE_VERSION=Y + +Memfault version: + VERSION=1.2.3 + GIT COMMIT=abcdef + BUILD ID=123456 + +Features enabled: + reboot + coredump diff --git a/memfaultd/src/cli/version.rs b/memfaultd/src/cli/version.rs new file mode 100644 index 0000000..f808448 --- /dev/null +++ b/memfaultd/src/cli/version.rs @@ -0,0 +1,11 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::build_info::{BUILD_ID, GIT_COMMIT, VERSION}; + +pub fn format_version() -> String { + format!( + "VERSION={}\nGIT COMMIT={}\nBUILD ID={}", + VERSION, GIT_COMMIT, BUILD_ID + ) +} diff --git a/memfaultd/src/collectd/collectd_handler.rs b/memfaultd/src/collectd/collectd_handler.rs new file mode 100644 index 0000000..e36a309 --- /dev/null +++ b/memfaultd/src/collectd/collectd_handler.rs @@ -0,0 +1,216 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + io::Read, + sync::{Arc, Mutex}, +}; + +use eyre::{eyre, Result}; +use log::{debug, log_enabled, trace, warn}; +use tiny_http::{Method, Request, Response}; + +use crate::{ + collectd::payload::Payload, + http_server::{HttpHandler, HttpHandlerResult}, + metrics::{KeyedMetricReading, MetricReportManager, BUILTIN_SYSTEM_METRIC_NAMESPACES}, +}; + +/// A server that listens for collectd JSON pushes and stores them in memory. +#[derive(Clone)] +pub struct CollectdHandler { + data_collection_enabled: bool, + builtin_system_metric_collection_enabled: bool, + metrics_store: Arc>, + builtin_namespaces: Vec, +} + +impl CollectdHandler { + pub fn new( + data_collection_enabled: bool, + builtin_system_metric_collection_enabled: bool, + metrics_store: Arc>, + ) -> Self { + CollectdHandler { + data_collection_enabled, + builtin_system_metric_collection_enabled, + metrics_store, + builtin_namespaces: BUILTIN_SYSTEM_METRIC_NAMESPACES + .iter() + .map(|namespace| namespace.to_string() + "/") + .collect(), + } + } + + /// Convert a collectd JSON push (Payload[]) into a list of MetricReading. + fn parse_request(stream: &mut dyn Read) -> Result> { + let payloads: Vec = if log_enabled!(log::Level::Debug) { + let mut buf = vec![]; + stream.read_to_end(&mut buf)?; + let s = String::from_utf8_lossy(&buf); + trace!("Received JSON: {}", s); + match serde_json::from_slice(&buf) { + Ok(payloads) => payloads, + Err(e) => { + debug!("Error parsing JSON: {}\n{}", e, String::from_utf8(buf)?); + return Err(eyre!("Error parsing JSON: {}", e)); + } + } + } else { + serde_json::from_reader(stream)? + }; + Ok(payloads + .into_iter() + .flat_map(Vec::::from) + .collect()) + } +} + +impl HttpHandler for CollectdHandler { + fn handle_request(&self, request: &mut Request) -> HttpHandlerResult { + if request.url() != "/v1/collectd" || *request.method() != Method::Post { + return HttpHandlerResult::NotHandled; + } + if self.data_collection_enabled { + match Self::parse_request(request.as_reader()) { + Ok(readings) => { + let mut metrics_store = self.metrics_store.lock().unwrap(); + for reading in readings { + // If built-in metric collection IS enabled, we need to drop + // collectd metric readings who may have overlapping keys with + // memfaultd's built-in readings. To be safe, any reading whose + // metric key has the same top-level namespace as a built-in system + // metric will be dropped + // + // For example, since CPU metrics can be captured by memfaultd + // this conditional will cause us to drop all collectd + // metric readings whose keys start with "cpu/" when + // built-in system metric collection is enabled + if !self.builtin_system_metric_collection_enabled + || !self + .builtin_namespaces + .iter() + .any(|namespace| reading.name.as_str().starts_with(namespace)) + { + if let Err(e) = metrics_store.add_metric(reading) { + warn!("Invalid metric: {e}"); + } + } + } + } + Err(e) => { + warn!("Error parsing request: {}", e); + } + } + } + HttpHandlerResult::Response(Response::empty(200).boxed()) + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use insta::{assert_json_snapshot, assert_snapshot, with_settings}; + use rstest::{fixture, rstest}; + use tiny_http::{Method, TestRequest}; + + use crate::{ + http_server::{HttpHandler, HttpHandlerResult}, + metrics::MetricReportManager, + }; + + use super::CollectdHandler; + + #[rstest] + fn handle_push(handler: CollectdHandler) { + let r = TestRequest::new().with_method(Method::Post).with_path("/v1/collectd").with_body( + r#"[{"values":[0],"dstypes":["derive"],"dsnames":["value"],"time":1619712000.000,"interval":10.000,"host":"localhost","plugin":"cpu","plugin_instance":"0","type":"cpu","type_instance":"idle"}]"#, + ); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + let metrics = handler + .metrics_store + .lock() + .unwrap() + .take_heartbeat_metrics(); + assert_snapshot!(serde_json::to_string_pretty(&metrics) + .expect("heartbeat_manager should be serializable")); + } + + #[rstest] + fn ignores_data_when_data_collection_is_off() { + let handler = CollectdHandler::new( + false, + false, + Arc::new(Mutex::new(MetricReportManager::new())), + ); + let r = TestRequest::new().with_method(Method::Post).with_path("/v1/collectd").with_body( + r#"[{"values":[0],"dstypes":["derive"],"dsnames":["value"],"time":1619712000.000,"interval":10.000,"host":"localhost","plugin":"cpu","plugin_instance":"0","type":"cpu","type_instance":"idle"}]"#, + ); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + let metrics = handler + .metrics_store + .lock() + .unwrap() + .take_heartbeat_metrics(); + assert_snapshot!(serde_json::to_string_pretty(&metrics) + .expect("heartbeat_manager should be serializable")); + } + + #[rstest] + fn drops_cpu_metrics_when_builtin_system_metrics_are_enabled() { + let handler = + CollectdHandler::new(true, true, Arc::new(Mutex::new(MetricReportManager::new()))); + let r = TestRequest::new().with_method(Method::Post).with_path("/v1/collectd").with_body( + r#"[{"values":[0],"dstypes":["derive"],"dsnames":["value"],"time":1619712000.000,"interval":10.000,"host":"localhost","plugin":"cpu","plugin_instance":"0","type":"cpu","type_instance":"idle"}]"#, + ); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + // cpufreq should NOT be dropped as it's a different top-level namespace from "cpu" + let r = TestRequest::new().with_method(Method::Post).with_path("/v1/collectd").with_body( + r#"[{"values":[0],"dstypes":["derive"],"dsnames":["value"],"time":1619712000.000,"interval":10.000,"host":"localhost","plugin":"cpufreq","plugin_instance":"0","type":"cpu","type_instance":"idle"}]"#, + ); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + let r = TestRequest::new().with_method(Method::Post).with_path("/v1/collectd").with_body( + r#"[{"values":[0],"dstypes":["derive"],"dsnames":["value"],"time":1619712000.000,"interval":10.000,"host":"localhost","plugin":"mockplugin","plugin_instance":"0","type":"mock","type_instance":"test"}]"#, + ); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + let metrics = handler + .metrics_store + .lock() + .unwrap() + .take_heartbeat_metrics(); + + with_settings!({sort_maps => true}, { + assert_json_snapshot!(metrics); + }); + } + + #[fixture] + fn handler() -> CollectdHandler { + CollectdHandler::new( + true, + false, + Arc::new(Mutex::new(MetricReportManager::new())), + ) + } +} diff --git a/memfaultd/src/collectd/fixtures/sample-with-null.json b/memfaultd/src/collectd/fixtures/sample-with-null.json new file mode 100644 index 0000000..e06ed2a --- /dev/null +++ b/memfaultd/src/collectd/fixtures/sample-with-null.json @@ -0,0 +1,158 @@ +[ + { + "values": [973577], + "dstypes": ["gauge"], + "dsnames": ["value"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "uptime", + "plugin_instance": "", + "type": "uptime", + "type_instance": "" + }, + { + "values": [11160961024], + "dstypes": ["gauge"], + "dsnames": ["value"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "memory", + "plugin_instance": "", + "type": "memory", + "type_instance": "inactive" + }, + { + "values": [null, null], + "dstypes": ["derive", "derive"], + "dsnames": ["rx", "tx"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "interface", + "plugin_instance": "lo0", + "type": "if_octets", + "type_instance": "" + }, + { + "values": [null, null], + "dstypes": ["derive", "derive"], + "dsnames": ["rx", "tx"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "interface", + "plugin_instance": "lo0", + "type": "if_packets", + "type_instance": "" + }, + { + "values": [11797676032], + "dstypes": ["gauge"], + "dsnames": ["value"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "memory", + "plugin_instance": "", + "type": "memory", + "type_instance": "active" + }, + { + "values": [null, null], + "dstypes": ["derive", "derive"], + "dsnames": ["rx", "tx"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "interface", + "plugin_instance": "gif0", + "type": "if_packets", + "type_instance": "" + }, + { + "values": [null, null], + "dstypes": ["derive", "derive"], + "dsnames": ["rx", "tx"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "interface", + "plugin_instance": "gif0", + "type": "if_errors", + "type_instance": "" + }, + { + "values": [null, null], + "dstypes": ["derive", "derive"], + "dsnames": ["rx", "tx"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "interface", + "plugin_instance": "stf0", + "type": "if_octets", + "type_instance": "" + }, + { + "values": [3427991552], + "dstypes": ["gauge"], + "dsnames": ["value"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "memory", + "plugin_instance": "", + "type": "memory", + "type_instance": "wired" + }, + { + "values": [null, null], + "dstypes": ["derive", "derive"], + "dsnames": ["rx", "tx"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "interface", + "plugin_instance": "stf0", + "type": "if_errors", + "type_instance": "" + }, + { + "values": [null, null], + "dstypes": ["derive", "derive"], + "dsnames": ["rx", "tx"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "interface", + "plugin_instance": "anpi0", + "type": "if_octets", + "type_instance": "" + }, + { + "values": [null, null], + "dstypes": ["derive", "derive"], + "dsnames": ["rx", "tx"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "interface", + "plugin_instance": "anpi0", + "type": "if_packets", + "type_instance": "" + }, + { + "values": [null, null], + "dstypes": ["derive", "derive"], + "dsnames": ["rx", "tx"], + "time": 1686089042.785, + "interval": 10.0, + "host": "moon.local", + "plugin": "interface", + "plugin_instance": "anpi0", + "type": "if_errors", + "type_instance": "" + } +] diff --git a/memfaultd/src/collectd/fixtures/sample1.json b/memfaultd/src/collectd/fixtures/sample1.json new file mode 100644 index 0000000..1de8735 --- /dev/null +++ b/memfaultd/src/collectd/fixtures/sample1.json @@ -0,0 +1,146 @@ +[ + { + "dsnames": ["read", "write"], + "dstypes": ["derive", "derive"], + "host": "moon.local", + "interval": 10.0, + "plugin": "disk", + "plugin_instance": "1-0", + "time": 1684949147.351, + "type": "disk_octets", + "type_instance": "", + "values": [1887359.3321528, 1241166.17826548] + }, + { + "dsnames": ["read", "write"], + "dstypes": ["derive", "derive"], + "host": "moon.local", + "interval": 10.0, + "plugin": "disk", + "plugin_instance": "1-20", + "time": 1684949147.351, + "type": "disk_time", + "type_instance": "", + "values": [992441.95537229, 0] + }, + { + "dsnames": ["read", "write"], + "dstypes": ["derive", "derive"], + "host": "moon.local", + "interval": 10.0, + "plugin": "disk", + "plugin_instance": "1-0", + "time": 1684949147.351, + "type": "disk_time", + "type_instance": "", + "values": [17146.2298941808, 6375.40234502902] + }, + { + "dsnames": ["read", "write"], + "dstypes": ["derive", "derive"], + "host": "moon.local", + "interval": 10.0, + "plugin": "disk", + "plugin_instance": "1-0", + "time": 1684949147.351, + "type": "disk_ops", + "type_instance": "", + "values": [75.3295819258201, 101.940031849151] + }, + { + "dsnames": ["value"], + "dstypes": ["gauge"], + "host": "moon.local", + "interval": 10.0, + "plugin": "memory", + "plugin_instance": "", + "time": 1684949147.352, + "type": "memory", + "type_instance": "active", + "values": [8541454336] + }, + { + "dsnames": ["value"], + "dstypes": ["gauge"], + "host": "moon.local", + "interval": 10.0, + "plugin": "memory", + "plugin_instance": "", + "time": 1684949147.352, + "type": "memory", + "type_instance": "inactive", + "values": [8286470144] + }, + { + "dsnames": ["value"], + "dstypes": ["gauge"], + "host": "moon.local", + "interval": 10.0, + "plugin": "memory", + "plugin_instance": "", + "time": 1684949147.352, + "type": "memory", + "type_instance": "free", + "values": [502759424] + }, + { + "dsnames": ["value"], + "dstypes": ["gauge"], + "host": "moon.local", + "interval": 10.0, + "plugin": "statsd", + "plugin_instance": "", + "time": 1684949147.352, + "type": "latency", + "type_instance": "app.kombu_publish_duration,exchange=celeryev,routing_key=worker.heartbeat-average", + "values": [0.00399999972432852] + }, + { + "dsnames": ["value"], + "dstypes": ["gauge"], + "host": "moon.local", + "interval": 10.0, + "plugin": "memory", + "plugin_instance": "", + "time": 1684949147.352, + "type": "memory", + "type_instance": "wired", + "values": [3314139136] + }, + { + "dsnames": ["read", "write"], + "dstypes": ["derive", "derive"], + "host": "moon.local", + "interval": 10.0, + "plugin": "disk", + "plugin_instance": "1-20", + "time": 1684949147.351, + "type": "disk_ops", + "type_instance": "", + "values": [17.3067461702536, 0] + }, + { + "dsnames": ["read", "write"], + "dstypes": ["derive", "derive"], + "host": "moon.local", + "interval": 10.0, + "plugin": "disk", + "plugin_instance": "1-20", + "time": 1684949147.351, + "type": "disk_octets", + "type_instance": "", + "values": [70888.4535866987, 0] + }, + { + "dsnames": ["value"], + "dstypes": ["gauge"], + "host": "moon.local", + "interval": 10.0, + "plugin": "uptime", + "plugin_instance": "", + "time": 1684949147.353, + "type": "uptime", + "type_instance": "", + "values": [790156] + } +] diff --git a/memfaultd/src/collectd/fixtures/statsd-counter-first-seen.json b/memfaultd/src/collectd/fixtures/statsd-counter-first-seen.json new file mode 100644 index 0000000..3263295 --- /dev/null +++ b/memfaultd/src/collectd/fixtures/statsd-counter-first-seen.json @@ -0,0 +1,50 @@ +[ + { + "values": [null], + "dstypes": ["derive"], + "dsnames": ["value"], + "time": 1698360252.757, + "interval": 10.0, + "host": "linux-devbox", + "plugin": "statsd", + "plugin_instance": "", + "type": "derive", + "type_instance": "mypythonapp.counter1" + }, + { + "values": [2], + "dstypes": ["gauge"], + "dsnames": ["value"], + "time": 1698360252.757, + "interval": 10.0, + "host": "linux-devbox", + "plugin": "statsd", + "plugin_instance": "", + "type": "count", + "type_instance": "mypythonapp.counter1" + }, + { + "values": [null], + "dstypes": ["derive"], + "dsnames": ["value"], + "time": 1698360252.757, + "interval": 10.0, + "host": "linux-devbox", + "plugin": "statsd", + "plugin_instance": "", + "type": "derive", + "type_instance": "mypythonapp.counter2" + }, + { + "values": [20], + "dstypes": ["gauge"], + "dsnames": ["value"], + "time": 1698360252.757, + "interval": 10.0, + "host": "linux-devbox", + "plugin": "statsd", + "plugin_instance": "", + "type": "count", + "type_instance": "mypythonapp.counter2" + } +] diff --git a/memfaultd/src/collectd/fixtures/statsd-counter.json b/memfaultd/src/collectd/fixtures/statsd-counter.json new file mode 100644 index 0000000..b25b93d --- /dev/null +++ b/memfaultd/src/collectd/fixtures/statsd-counter.json @@ -0,0 +1,50 @@ +[ + { + "values": [1], + "dstypes": ["gauge"], + "dsnames": ["value"], + "time": 1698360892.757, + "interval": 10.0, + "host": "linux-devbox", + "plugin": "statsd", + "plugin_instance": "", + "type": "count", + "type_instance": "mypythonapp.counter1" + }, + { + "values": [0.999995505178062], + "dstypes": ["derive"], + "dsnames": ["value"], + "time": 1698360892.757, + "interval": 10.0, + "host": "linux-devbox", + "plugin": "statsd", + "plugin_instance": "", + "type": "derive", + "type_instance": "mypythonapp.counter2" + }, + { + "values": [10], + "dstypes": ["gauge"], + "dsnames": ["value"], + "time": 1698360892.757, + "interval": 10.0, + "host": "linux-devbox", + "plugin": "statsd", + "plugin_instance": "", + "type": "count", + "type_instance": "mypythonapp.counter2" + }, + { + "values": [0.0999993716964859], + "dstypes": ["derive"], + "dsnames": ["value"], + "time": 1698360892.757, + "interval": 10.0, + "host": "linux-devbox", + "plugin": "statsd", + "plugin_instance": "", + "type": "derive", + "type_instance": "mypythonapp.counter1" + } +] diff --git a/memfaultd/src/collectd/mod.rs b/memfaultd/src/collectd/mod.rs new file mode 100644 index 0000000..9bc7b99 --- /dev/null +++ b/memfaultd/src/collectd/mod.rs @@ -0,0 +1,7 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +mod payload; + +mod collectd_handler; +pub use collectd_handler::CollectdHandler; diff --git a/memfaultd/src/collectd/payload.rs b/memfaultd/src/collectd/payload.rs new file mode 100644 index 0000000..9b86711 --- /dev/null +++ b/memfaultd/src/collectd/payload.rs @@ -0,0 +1,210 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::str::FromStr; + +use chrono::{DateTime, Duration, Utc}; +use eyre::{eyre, Result}; +use itertools::izip; +use log::warn; +use serde::Deserialize; + +use crate::{ + metrics::{KeyedMetricReading, MetricReading, MetricStringKey}, + util::serialization::float_to_datetime, + util::serialization::float_to_duration, +}; + +/// https://collectd.org/wiki/index.php/Data_source +/// and https://git.octo.it/?p=collectd.git;a=blob;f=src/daemon/plugin.h;hb=master#l45 +#[derive(Debug, Deserialize)] +enum DataSourceType { + #[serde(rename = "gauge")] + Gauge, + #[serde(rename = "derive")] + Derive, + #[serde(rename = "counter")] + Counter, + #[serde(rename = "absolute")] + Absolute, + #[serde(rename = "unknown")] + Unknown, +} + +/// https://collectd.org/wiki/index.php/JSON +/// https://collectd.org/wiki/index.php/Value_list +#[derive(Debug, Deserialize)] +pub struct Payload { + dsnames: Vec, + dstypes: Vec, + #[allow(dead_code)] + host: String, + // CollectD encodes time and duration to a float before sending as JSON. + // https://github.com/collectd/collectd/blob/main/src/utils/format_json/format_json.c#L344-L345 + #[allow(dead_code)] + #[serde(with = "float_to_duration")] + interval: Duration, + plugin: String, + plugin_instance: Option, + #[serde(with = "float_to_datetime")] + time: DateTime, + #[serde(rename = "type")] + type_str: String, + type_instance: Option, + values: Vec>, +} + +impl Payload { + fn metric_name(&self, name: &String) -> Result { + let use_simple_reading_name = self.dsnames.len() == 1 && self.dsnames[0] == "value"; + let name_prefix = vec![ + Some(&self.plugin), + self.plugin_instance.as_ref(), + Some(&self.type_str), + self.type_instance.as_ref(), + ] + .into_iter() + .flatten() + .filter(|x| !x.is_empty()) + .map(|x| x.as_str()) + .collect::>() + .join("/"); + + let name = if use_simple_reading_name { + name_prefix + } else { + format!("{}/{}", name_prefix, name) + }; + MetricStringKey::from_str(&name).map_err(|e| eyre!("Invalid metric name: {e}")) + } +} + +impl From for Vec { + fn from(payload: Payload) -> Self { + izip!(&payload.dsnames, &payload.values, &payload.dstypes) + // Remove variables that have no value + .filter_map(|(name, value, dstype)| value.as_ref().map(|v| (name, v, dstype))) + // Remove variables with invalid names + .filter_map(|(name, value, dstype)| match payload.metric_name(name) { + Ok(key) => Some((key, value, dstype)), + _ => { + warn!("Ignoring metric with invalid name: {}", name); + None + } + }) + // Create the KeyedMetricValue + .map(|(key, value, dstype)| KeyedMetricReading { + name: key, + value: match dstype { + // Refer to https://github.com/collectd/collectd/wiki/Data-source + // for a general description of what CollectdD datasources are. + + // Statsd generated counter values. + // See https://github.com/collectd/collectd/blob/7c5ce9f250aafbb6ef89769d7543ea155618b2ad/src/statsd.c#L799-L810 + DataSourceType::Gauge if payload.type_str == "count" => { + MetricReading::Counter { + value: *value, + timestamp: payload.time, + } + } + DataSourceType::Gauge => MetricReading::Histogram { + value: *value, + timestamp: payload.time, + }, + DataSourceType::Derive => MetricReading::Histogram { + value: *value, + timestamp: payload.time, + }, + // A counter is a Derive (rate) that will never be negative. + DataSourceType::Counter => MetricReading::Histogram { + value: *value, + timestamp: payload.time, + }, + DataSourceType::Absolute => MetricReading::Histogram { + value: *value, + timestamp: payload.time, + }, + DataSourceType::Unknown => MetricReading::Histogram { + value: *value, + timestamp: payload.time, + }, + }, + }) + .collect::>() + } +} + +#[cfg(test)] +mod tests { + use chrono::Duration; + use std::fs::read_to_string; + use std::path::PathBuf; + + use rstest::rstest; + + use crate::metrics::KeyedMetricReading; + + use super::DataSourceType; + use super::Payload; + + #[rstest] + #[case("A", Some("B"), "C", Some("D"), "E", "A/B/C/D/E")] + #[case("A", None, "C", Some("D"), "E", "A/C/D/E")] + #[case("A", Some("B"), "C", None, "E", "A/B/C/E")] + #[case("A", None, "C", None, "E", "A/C/E")] + #[case("A", Some(""), "C", Some(""), "E", "A/C/E")] + #[case("A", Some("B"), "C", Some("D"), "value", "A/B/C/D")] + fn convert_collectd_to_metric_name( + #[case] plugin: &str, + #[case] plugin_instance: Option<&str>, + #[case] type_s: &str, + #[case] type_instance: Option<&str>, + #[case] dsname: &str, + #[case] expected: &str, + ) { + let p = Payload { + dsnames: vec![dsname.to_string()], + dstypes: vec![DataSourceType::Gauge], + host: "".to_string(), + interval: Duration::seconds(10), + plugin: plugin.to_string(), + plugin_instance: plugin_instance.map(|x| x.to_owned()), + time: chrono::Utc::now(), + type_str: type_s.to_string(), + type_instance: type_instance.map(|x| x.to_owned()), + values: vec![Some(42.0)], + }; + + let readings = Vec::::from(p); + assert_eq!(readings.len(), 1); + let KeyedMetricReading { name, .. } = &readings[0]; + assert_eq!(name.as_str(), expected) + } + + #[rstest] + // Note: sample1 contains multiple payloads. Some have equal timestamps (and need to be consolidated), some have "simple values". + #[case("sample1")] + #[case("sample-with-null")] + #[case("statsd-counter-first-seen")] + #[case("statsd-counter")] + fn convert_collectd_payload_into_heartbeat_metadata(#[case] name: &str) { + let input_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/collectd/fixtures") + .join(name) + .with_extension("json"); + + // Read multiple payloads from a single file (this is how we receive from CollectD) + let payloads = + serde_json::from_str::>(&read_to_string(&input_path).unwrap()) + .unwrap(); + + // Convert payload into metric-readings + let metadatas = payloads + .into_iter() + .flat_map(Vec::::from) + .collect::>(); + + // Check results + insta::assert_json_snapshot!(format!("{name}"), metadatas); + } +} diff --git a/memfaultd/src/collectd/snapshots/memfaultd__collectd__collectd_handler__tests__drops_cpu_metrics_when_builtin_system_metrics_are_enabled.snap b/memfaultd/src/collectd/snapshots/memfaultd__collectd__collectd_handler__tests__drops_cpu_metrics_when_builtin_system_metrics_are_enabled.snap new file mode 100644 index 0000000..c90ceff --- /dev/null +++ b/memfaultd/src/collectd/snapshots/memfaultd__collectd__collectd_handler__tests__drops_cpu_metrics_when_builtin_system_metrics_are_enabled.snap @@ -0,0 +1,8 @@ +--- +source: memfaultd/src/collectd/collectd_handler.rs +expression: metrics +--- +{ + "cpufreq/0/cpu/idle": 0.0, + "mockplugin/0/mock/test": 0.0 +} diff --git a/memfaultd/src/collectd/snapshots/memfaultd__collectd__collectd_handler__tests__handle_push.snap b/memfaultd/src/collectd/snapshots/memfaultd__collectd__collectd_handler__tests__handle_push.snap new file mode 100644 index 0000000..958fce1 --- /dev/null +++ b/memfaultd/src/collectd/snapshots/memfaultd__collectd__collectd_handler__tests__handle_push.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/collectd/collectd_handler.rs +expression: "serde_json::to_string_pretty(&metrics).expect(\"metric_store should be serializable\")" +--- +{ + "cpu/0/cpu/idle": 0.0 +} diff --git a/memfaultd/src/collectd/snapshots/memfaultd__collectd__collectd_handler__tests__ignores_data_when_data_collection_is_off.snap b/memfaultd/src/collectd/snapshots/memfaultd__collectd__collectd_handler__tests__ignores_data_when_data_collection_is_off.snap new file mode 100644 index 0000000..3f150d8 --- /dev/null +++ b/memfaultd/src/collectd/snapshots/memfaultd__collectd__collectd_handler__tests__ignores_data_when_data_collection_is_off.snap @@ -0,0 +1,5 @@ +--- +source: memfaultd/src/collectd/collectd_handler.rs +expression: "serde_json::to_string_pretty(&metrics).expect(\"metric_store should be serializable\")" +--- +{} diff --git a/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__sample-with-null.snap b/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__sample-with-null.snap new file mode 100644 index 0000000..27cd690 --- /dev/null +++ b/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__sample-with-null.snap @@ -0,0 +1,42 @@ +--- +source: memfaultd/src/collectd/payload.rs +expression: metadatas +--- +[ + { + "name": "uptime/uptime", + "value": { + "Histogram": { + "value": 973577.0, + "timestamp": "2023-06-06T22:04:02.785Z" + } + } + }, + { + "name": "memory/memory/inactive", + "value": { + "Histogram": { + "value": 11160961024.0, + "timestamp": "2023-06-06T22:04:02.785Z" + } + } + }, + { + "name": "memory/memory/active", + "value": { + "Histogram": { + "value": 11797676032.0, + "timestamp": "2023-06-06T22:04:02.785Z" + } + } + }, + { + "name": "memory/memory/wired", + "value": { + "Histogram": { + "value": 3427991552.0, + "timestamp": "2023-06-06T22:04:02.785Z" + } + } + } +] diff --git a/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__sample1.snap b/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__sample1.snap new file mode 100644 index 0000000..1108cb3 --- /dev/null +++ b/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__sample1.snap @@ -0,0 +1,168 @@ +--- +source: memfaultd/src/collectd/payload.rs +expression: metadatas +--- +[ + { + "name": "disk/1-0/disk_octets/read", + "value": { + "Histogram": { + "value": 1887359.3321528, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "disk/1-0/disk_octets/write", + "value": { + "Histogram": { + "value": 1241166.17826548, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "disk/1-20/disk_time/read", + "value": { + "Histogram": { + "value": 992441.95537229, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "disk/1-20/disk_time/write", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "disk/1-0/disk_time/read", + "value": { + "Histogram": { + "value": 17146.2298941808, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "disk/1-0/disk_time/write", + "value": { + "Histogram": { + "value": 6375.40234502902, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "disk/1-0/disk_ops/read", + "value": { + "Histogram": { + "value": 75.3295819258201, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "disk/1-0/disk_ops/write", + "value": { + "Histogram": { + "value": 101.940031849151, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "memory/memory/active", + "value": { + "Histogram": { + "value": 8541454336.0, + "timestamp": "2023-05-24T17:25:47.352Z" + } + } + }, + { + "name": "memory/memory/inactive", + "value": { + "Histogram": { + "value": 8286470144.0, + "timestamp": "2023-05-24T17:25:47.352Z" + } + } + }, + { + "name": "memory/memory/free", + "value": { + "Histogram": { + "value": 502759424.0, + "timestamp": "2023-05-24T17:25:47.352Z" + } + } + }, + { + "name": "statsd/latency/app.kombu_publish_duration,exchange=celeryev,routing_key=worker.heartbeat-average", + "value": { + "Histogram": { + "value": 0.00399999972432852, + "timestamp": "2023-05-24T17:25:47.352Z" + } + } + }, + { + "name": "memory/memory/wired", + "value": { + "Histogram": { + "value": 3314139136.0, + "timestamp": "2023-05-24T17:25:47.352Z" + } + } + }, + { + "name": "disk/1-20/disk_ops/read", + "value": { + "Histogram": { + "value": 17.3067461702536, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "disk/1-20/disk_ops/write", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "disk/1-20/disk_octets/read", + "value": { + "Histogram": { + "value": 70888.4535866987, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "disk/1-20/disk_octets/write", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "2023-05-24T17:25:47.351Z" + } + } + }, + { + "name": "uptime/uptime", + "value": { + "Histogram": { + "value": 790156.0, + "timestamp": "2023-05-24T17:25:47.353Z" + } + } + } +] diff --git a/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__statsd-counter-first-seen.snap b/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__statsd-counter-first-seen.snap new file mode 100644 index 0000000..9b38f4d --- /dev/null +++ b/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__statsd-counter-first-seen.snap @@ -0,0 +1,24 @@ +--- +source: memfaultd/src/collectd/payload.rs +expression: metadatas +--- +[ + { + "name": "statsd/count/mypythonapp.counter1", + "value": { + "Counter": { + "value": 2.0, + "timestamp": "2023-10-26T22:44:12.757Z" + } + } + }, + { + "name": "statsd/count/mypythonapp.counter2", + "value": { + "Counter": { + "value": 20.0, + "timestamp": "2023-10-26T22:44:12.757Z" + } + } + } +] diff --git a/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__statsd-counter.snap b/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__statsd-counter.snap new file mode 100644 index 0000000..32f5b8c --- /dev/null +++ b/memfaultd/src/collectd/snapshots/memfaultd__collectd__payload__tests__statsd-counter.snap @@ -0,0 +1,42 @@ +--- +source: memfaultd/src/collectd/payload.rs +expression: metadatas +--- +[ + { + "name": "statsd/count/mypythonapp.counter1", + "value": { + "Counter": { + "value": 1.0, + "timestamp": "2023-10-26T22:54:52.757Z" + } + } + }, + { + "name": "statsd/derive/mypythonapp.counter2", + "value": { + "Histogram": { + "value": 0.999995505178062, + "timestamp": "2023-10-26T22:54:52.757Z" + } + } + }, + { + "name": "statsd/count/mypythonapp.counter2", + "value": { + "Counter": { + "value": 10.0, + "timestamp": "2023-10-26T22:54:52.757Z" + } + } + }, + { + "name": "statsd/derive/mypythonapp.counter1", + "value": { + "Histogram": { + "value": 0.0999993716964859, + "timestamp": "2023-10-26T22:54:52.757Z" + } + } + } +] diff --git a/memfaultd/src/config/config_file.rs b/memfaultd/src/config/config_file.rs new file mode 100644 index 0000000..a6ceb02 --- /dev/null +++ b/memfaultd/src/config/config_file.rs @@ -0,0 +1,521 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::{eyre, Context}; +use serde::{Deserialize, Serialize}; +use std::net::IpAddr; +use std::time::Duration; +use std::{ + collections::{HashMap, HashSet}, + num::NonZeroU32, +}; +use std::{net::SocketAddr, path::PathBuf}; + +use crate::metrics::{MetricStringKey, SessionName}; +use crate::util::*; +use crate::util::{path::AbsolutePath, serialization::*}; + +#[derive(Serialize, Deserialize, Debug)] +pub struct MemfaultdConfig { + pub persist_dir: AbsolutePath, + pub tmp_dir: Option, + #[serde(rename = "tmp_dir_min_headroom_kib", with = "kib_to_usize")] + pub tmp_dir_min_headroom: usize, + pub tmp_dir_min_inodes: usize, + #[serde(rename = "tmp_dir_max_usage_kib", with = "kib_to_usize")] + pub tmp_dir_max_usage: usize, + #[serde(rename = "upload_interval_seconds", with = "seconds_to_duration")] + pub upload_interval: Duration, + #[serde(rename = "heartbeat_interval_seconds", with = "seconds_to_duration")] + pub heartbeat_interval: Duration, + pub enable_data_collection: bool, + pub enable_dev_mode: bool, + pub software_version: Option, + pub software_type: Option, + pub project_key: String, + pub base_url: String, + pub swupdate: SwUpdateConfig, + pub reboot: RebootConfig, + pub coredump: CoredumpConfig, + #[serde(rename = "fluent-bit")] + pub fluent_bit: FluentBitConfig, + pub logs: LogsConfig, + pub mar: MarConfig, + pub http_server: HttpServerConfig, + pub battery_monitor: Option, + pub connectivity_monitor: Option, + pub sessions: Option>, + pub metrics: MetricReportConfig, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct SwUpdateConfig { + pub input_file: PathBuf, + pub output_file: PathBuf, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct RebootConfig { + pub last_reboot_reason_file: PathBuf, +} + +#[derive(Serialize, Deserialize, Debug, Copy, Clone)] +pub enum CoredumpCompression { + #[serde(rename = "gzip")] + Gzip, + #[serde(rename = "none")] + None, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy)] +#[serde(tag = "type")] +pub enum CoredumpCaptureStrategy { + #[serde(rename = "threads")] + /// Only capture the stacks of the threads that were running at the time of the crash. + Threads { + #[serde(rename = "max_thread_size_kib", with = "kib_to_usize")] + max_thread_size: usize, + }, + #[serde(rename = "kernel_selection")] + /// Keep in the coredump what the kernel selected to be included in the coredump. + /// See https://man7.org/linux/man-pages/man5/core.5.html for more details. + KernelSelection, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct CoredumpConfig { + pub compression: CoredumpCompression, + #[serde(rename = "coredump_max_size_kib", with = "kib_to_usize")] + pub coredump_max_size: usize, + pub rate_limit_count: u32, + #[serde(rename = "rate_limit_duration_seconds", with = "seconds_to_duration")] + pub rate_limit_duration: Duration, + pub capture_strategy: CoredumpCaptureStrategy, + pub log_lines: usize, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct FluentBitConfig { + pub extra_fluentd_attributes: Vec, + pub bind_address: SocketAddr, + pub max_buffered_lines: usize, + pub max_connections: usize, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct HttpServerConfig { + pub bind_address: SocketAddr, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy)] +pub enum LogSource { + #[serde(rename = "fluent-bit")] + FluentBit, + #[cfg(feature = "systemd")] + #[serde(rename = "journald")] + Journald, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct LogsConfig { + #[serde(rename = "rotate_size_kib", with = "kib_to_usize")] + pub rotate_size: usize, + + #[serde(rename = "rotate_after_seconds", with = "seconds_to_duration")] + pub rotate_after: Duration, + + #[serde(with = "number_to_compression")] + pub compression_level: Compression, + + pub max_lines_per_minute: NonZeroU32, + + pub log_to_metrics: Option, + + pub storage: StorageConfig, + + pub source: LogSource, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy)] +pub enum StorageConfig { + #[serde(rename = "disabled")] + Disabled, + #[serde(rename = "persist")] + Persist, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct LogToMetricsConfig { + pub rules: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(tag = "type")] +pub enum LogToMetricRule { + #[serde(rename = "count_matching")] + CountMatching { + /// Regex applied on the MESSAGE field + pattern: String, + metric_name: String, + /// List of key-value that must exactly match before the regexp is applied + #[serde(default)] + filter: HashMap, + }, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct MarConfig { + #[serde(rename = "mar_file_max_size_kib", with = "kib_to_usize")] + pub mar_file_max_size: usize, + #[serde(rename = "mar_entry_max_age_seconds", with = "seconds_to_duration")] + pub mar_entry_max_age: Duration, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct BatteryMonitorConfig { + pub battery_info_command: String, + #[serde(with = "seconds_to_duration")] + pub interval_seconds: Duration, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct ConnectivityMonitorConfig { + #[serde(with = "seconds_to_duration")] + pub interval_seconds: Duration, + pub targets: Vec, + #[serde( + with = "seconds_to_duration", + default = "default_connection_check_timeout" + )] + pub timeout_seconds: Duration, +} +fn default_connection_check_timeout() -> Duration { + Duration::from_secs(10) +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct MetricReportConfig { + pub enable_daily_heartbeats: bool, + pub system_metric_collection: SystemMetricConfig, + pub statsd_server: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ConnectivityMonitorTarget { + #[serde(default = "default_connection_check_protocol")] + pub protocol: ConnectionCheckProtocol, + pub host: IpAddr, + pub port: u16, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "lowercase")] +pub enum ConnectionCheckProtocol { + Tcp, +} + +fn default_connection_check_protocol() -> ConnectionCheckProtocol { + ConnectionCheckProtocol::Tcp +} + +#[derive(Serialize, Clone, Deserialize, Debug)] +pub struct SessionConfig { + pub name: SessionName, + pub captured_metrics: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct StatsDServerConfig { + pub bind_address: SocketAddr, +} + +#[derive(Clone, Serialize, Deserialize, Debug)] +pub struct SystemMetricConfig { + pub enable: bool, + #[serde(with = "seconds_to_duration")] + pub poll_interval_seconds: Duration, + pub processes: Option>, + pub disk_space: Option>, + pub network_interfaces: Option>, +} + +use flate2::Compression; +use serde_json::Value; +use std::fs; +use std::path::Path; + +use crate::config::utils::{software_type_is_valid, software_version_is_valid}; + +pub struct JsonConfigs { + /// Built-in configuration and System configuration + pub base: Value, + /// Runtime configuration + pub runtime: Value, +} + +impl MemfaultdConfig { + pub fn load(config_path: &Path) -> eyre::Result { + let JsonConfigs { + base: mut config_json, + runtime, + } = Self::parse_configs(config_path)?; + Self::merge_into(&mut config_json, runtime); + + // Transform the JSON object into a typed structure. + let config: MemfaultdConfig = serde_json::from_value(config_json)?; + + let mut validation_errors = vec![]; + if let Some(software_version) = &config.software_version { + if let Err(e) = software_version_is_valid(software_version) { + validation_errors.push(format!(" Invalid value for \"software_version\": {}", e)); + } + } + if let Some(software_type) = &config.software_type { + if let Err(e) = software_type_is_valid(software_type) { + validation_errors.push(format!(" Invalid value for \"software_type\": {}", e)); + } + } + + match validation_errors.is_empty() { + true => Ok(config), + false => Err(eyre::eyre!("\n{}", validation_errors.join("\n"))), + } + } + + /// Parse config file from given path and returns (builtin+system config, runtime config). + pub fn parse_configs(config_path: &Path) -> eyre::Result { + // Initialize with the builtin config file. + let mut base: Value = Self::parse(include_str!("../../builtin.conf")) + .wrap_err("Error parsing built-in configuration file")?; + + // Read and parse the user config file. + let user_config = Self::parse(std::fs::read_to_string(config_path)?.as_str()) + .wrap_err(eyre!("Error reading {}", config_path.display()))?; + + // Merge the two JSON objects together + Self::merge_into(&mut base, user_config); + + // Load the runtime config but only if the file exists. (Missing runtime config is not an error.) + let runtime_config_path = Self::runtime_config_path_from_json(&base)?; + let runtime = if runtime_config_path.exists() { + Self::parse(fs::read_to_string(&runtime_config_path)?.as_str()).wrap_err(eyre!( + "Error reading runtime configuration {}", + runtime_config_path.display() + ))? + } else { + Value::Object(serde_json::Map::new()) + }; + + Ok(JsonConfigs { base, runtime }) + } + + /// Set and write boolean in runtime config. + pub fn set_and_write_bool_to_runtime_config(&self, key: &str, value: bool) -> eyre::Result<()> { + let config_string = match fs::read_to_string(self.runtime_config_path()) { + Ok(config_string) => config_string, + Err(e) => { + if e.kind() == std::io::ErrorKind::NotFound { + "{}".to_string() + } else { + return Err(eyre::eyre!("Failed to read runtime config: {}", e)); + } + } + }; + + let mut config_val = Self::parse(&config_string)?; + config_val[key] = Value::Bool(value); + + self.write_value_to_runtime_config(config_val) + } + + /// Write config to runtime config file. + /// + /// This is used to write the config to a file that can be read by the memfaultd process. + fn write_value_to_runtime_config(&self, value: Value) -> eyre::Result<()> { + let runtime_config_path = self.runtime_config_path(); + fs::write(runtime_config_path, value.to_string())?; + + Ok(()) + } + + pub fn runtime_config_path(&self) -> PathBuf { + PathBuf::from(self.persist_dir.clone()).join("runtime.conf") + } + + // Parse a Memfaultd JSON configuration file (with optional C-style comments) and return a serde_json::Value object. + fn parse(config_string: &str) -> eyre::Result { + let json_text = string::remove_comments(config_string); + let json: Value = serde_json::from_str(json_text.as_str())?; + if !json.is_object() { + return Err(eyre::eyre!("Configuration should be a JSON object.")); + } + Ok(json) + } + + /// Merge two JSON objects together. The values from the second one will override values in the first one. + fn merge_into(dest: &mut Value, src: Value) { + assert!(dest.is_object() && src.is_object()); + if let Value::Object(dest_map) = src { + for (key, value) in dest_map { + if let Some(obj) = dest.get_mut(&key) { + if obj.is_object() { + MemfaultdConfig::merge_into(obj, value); + continue; + } + } + dest[&key] = value; + } + } + } + + pub fn generate_tmp_filename(&self, filename: &str) -> PathBuf { + // Fall back to persist dir if tmp_dir is not set. + let tmp_dir = self.tmp_dir.as_ref().unwrap_or(&self.persist_dir); + PathBuf::from(tmp_dir.clone()).join(filename) + } + + pub fn generate_persist_filename(&self, filename: &str) -> PathBuf { + PathBuf::from(self.persist_dir.clone()).join(filename) + } + + /// Generate the path to the runtime config file from a serde_json::Value object. This should include the "persist_dir" field. + fn runtime_config_path_from_json(config: &Value) -> eyre::Result { + let mut persist_dir = PathBuf::from( + config["persist_dir"] + .as_str() + .ok_or(eyre::eyre!("Config['persist_dir'] must be a string."))?, + ); + persist_dir.push("runtime.conf"); + Ok(persist_dir) + } +} + +#[cfg(test)] +impl MemfaultdConfig { + pub fn test_fixture() -> Self { + use std::fs::write; + use tempfile::tempdir; + + let tmp = tempdir().unwrap(); + let config_path = tmp.path().join("memfaultd.conf"); + write(&config_path, "{}").unwrap(); + MemfaultdConfig::load(&config_path).unwrap() + } +} + +#[cfg(test)] +mod test { + use insta::{assert_json_snapshot, with_settings}; + use rstest::rstest; + + use super::*; + + use crate::test_utils::set_snapshot_suffix; + + #[test] + fn test_merge() { + let mut c = + serde_json::from_str(r#"{ "node": { "value": true, "valueB": false } }"#).unwrap(); + let j = serde_json::from_str(r#"{ "node2": "xxx" }"#).unwrap(); + + MemfaultdConfig::merge_into(&mut c, j); + + assert_eq!( + serde_json::to_string(&c).unwrap(), + r#"{"node":{"value":true,"valueB":false},"node2":"xxx"}"# + ); + } + + #[test] + fn test_merge_overwrite() { + let mut c = + serde_json::from_str(r#"{ "node": { "value": true, "valueB": false } }"#).unwrap(); + let j = serde_json::from_str(r#"{ "node": { "value": false }}"#).unwrap(); + + MemfaultdConfig::merge_into(&mut c, j); + + assert_eq!( + serde_json::to_string(&c).unwrap(), + r#"{"node":{"value":false,"valueB":false}}"# + ); + } + + #[test] + fn test_merge_overwrite_nested() { + let mut c = serde_json::from_str( + r#"{ "node": { "value": true, "valueB": false, "valueC": { "a": 1, "b": 2 } } }"#, + ) + .unwrap(); + let j = serde_json::from_str(r#"{ "node": { "valueC": { "b": 42 } }}"#).unwrap(); + + MemfaultdConfig::merge_into(&mut c, j); + + assert_eq!( + serde_json::to_string(&c).unwrap(), + r#"{"node":{"value":true,"valueB":false,"valueC":{"a":1,"b":42}}}"# + ); + } + + #[rstest] + #[case("empty_object")] + #[case("with_partial_logs")] + #[case("without_coredump_compression")] + #[case("with_coredump_capture_strategy_threads")] + #[case("with_log_to_metrics_rules")] + #[case("with_connectivity_monitor")] + #[case("with_sessions")] + #[case("metrics_config")] + fn can_parse_test_files(#[case] name: &str) { + let input_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/config/test-config") + .join(name) + .with_extension("json"); + // Verifies that the file is parsable + let content = MemfaultdConfig::load(&input_path).unwrap(); + // And that the configuration generated is what we expect. + // Use `cargo insta review` to quickly approve changes. + with_settings!({sort_maps => true}, { + assert_json_snapshot!(name, content); + }); + } + + #[rstest] + #[case("with_invalid_path")] + #[case("with_invalid_swt_swv")] + #[case("with_sessions_invalid_metric_name")] + #[case("with_sessions_invalid_session_name")] + fn will_reject_bad_config(#[case] name: &str) { + let input_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/config/test-config") + .join(name) + .with_extension("json"); + let result = MemfaultdConfig::load(&input_path); + assert!(result.is_err()); + } + + #[rstest] + #[case("no_file", None)] + #[case("empty_object", Some("{}"))] + #[case("other_key", Some(r#"{"key2":false}"#))] + fn test_set_and_write_bool_to_runtime_config( + #[case] test_name: &str, + #[case] config_string: Option<&str>, + ) { + let mut config = MemfaultdConfig::test_fixture(); + let temp_data_dir = tempfile::tempdir().unwrap(); + config.persist_dir = AbsolutePath::try_from(temp_data_dir.path().to_path_buf()).unwrap(); + + if let Some(config_string) = config_string { + std::fs::write(config.runtime_config_path(), config_string).unwrap(); + } + + config + .set_and_write_bool_to_runtime_config("key", true) + .unwrap(); + + let disk_config_string = std::fs::read_to_string(config.runtime_config_path()).unwrap(); + + set_snapshot_suffix!("{}", test_name); + insta::assert_json_snapshot!(disk_config_string); + } +} diff --git a/memfaultd/src/config/device_config.rs b/memfaultd/src/config/device_config.rs new file mode 100644 index 0000000..e10c0c5 --- /dev/null +++ b/memfaultd/src/config/device_config.rs @@ -0,0 +1,93 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use serde::{Deserialize, Serialize}; + +use crate::network::{DeviceConfigResponse, DeviceConfigResponseResolution, DeviceConfigRevision}; + +#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord)] +pub enum Resolution { + Off, + Low, + Normal, + High, +} + +impl From for Resolution { + fn from(resolution: DeviceConfigResponseResolution) -> Self { + match resolution { + DeviceConfigResponseResolution::Off => Resolution::Off, + DeviceConfigResponseResolution::Low => Resolution::Low, + DeviceConfigResponseResolution::Normal => Resolution::Normal, + DeviceConfigResponseResolution::High => Resolution::High, + } + } +} + +#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq)] +pub struct Sampling { + pub debugging_resolution: Resolution, + pub logging_resolution: Resolution, + pub monitoring_resolution: Resolution, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +/// DeviceConfig is configuration provided by Memfault backend. +pub struct DeviceConfig { + pub revision: Option, + pub sampling: Sampling, +} + +impl From for DeviceConfig { + fn from(response: DeviceConfigResponse) -> Self { + Self { + revision: Some(response.data.revision), + sampling: Sampling { + debugging_resolution: response + .data + .config + .memfault + .sampling + .debugging_resolution + .into(), + logging_resolution: response + .data + .config + .memfault + .sampling + .logging_resolution + .into(), + monitoring_resolution: response + .data + .config + .memfault + .sampling + .monitoring_resolution + .into(), + }, + } + } +} + +impl Default for DeviceConfig { + fn default() -> Self { + Self { + revision: None, + sampling: Sampling { + debugging_resolution: Resolution::Off, + logging_resolution: Resolution::Off, + monitoring_resolution: Resolution::Off, + }, + } + } +} + +impl Sampling { + pub fn development() -> Self { + Self { + debugging_resolution: Resolution::High, + logging_resolution: Resolution::High, + monitoring_resolution: Resolution::High, + } + } +} diff --git a/memfaultd/src/config/device_info.rs b/memfaultd/src/config/device_info.rs new file mode 100644 index 0000000..f566e69 --- /dev/null +++ b/memfaultd/src/config/device_info.rs @@ -0,0 +1,545 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::fmt::{self, Display}; +use std::fs::read_to_string; +use std::process::Command; + +use eyre::{eyre, Result}; + +use crate::config::utils::{ + device_id_is_valid, hardware_version_is_valid, software_type_is_valid, + software_version_is_valid, +}; +use crate::util::etc_os_release::EtcOsRelease; + +const DEVICE_ID_PATH: &str = "/etc/machine-id"; +const HARDWARE_VERSION_COMMAND: &str = "uname"; +const HARDWARE_VERSION_ARGS: &[&str] = &["-n"]; + +#[cfg_attr(test, mockall::automock)] +/// Trait for providing default values for device info. +/// +/// This is mostly a convenience for testing, as the default implementation +/// reads the software version from /etc/os-release. +pub trait DeviceInfoDefaults { + /// Get the software version from the system. + fn software_version(&self) -> Result>; + + /// Get the device ID from the system. + fn device_id(&self) -> Result; + + /// Get the hardware version from the system. + fn hardware_version(&self) -> Result; + + /// Get the software type from the system. + fn software_type(&self) -> Result>; +} + +/// Default implementation of DeviceInfoDefaults. +pub struct DeviceInfoDefaultsImpl { + os_release: Option, +} + +impl DeviceInfoDefaultsImpl { + fn new(os_release: Option) -> Self { + Self { os_release } + } +} + +impl DeviceInfoDefaults for DeviceInfoDefaultsImpl { + fn software_version(&self) -> Result> { + Ok(self.os_release.as_ref().and_then(|os| os.version_id())) + } + + fn device_id(&self) -> Result { + let device_id = read_to_string(DEVICE_ID_PATH)?; + + Ok(device_id) + } + + fn hardware_version(&self) -> Result { + let output = Command::new(HARDWARE_VERSION_COMMAND) + .args(HARDWARE_VERSION_ARGS) + .output()?; + + Ok(String::from_utf8(output.stdout)?) + } + + fn software_type(&self) -> Result> { + Ok(self.os_release.as_ref().and_then(|os| os.id())) + } +} + +#[derive(Debug, PartialEq, Eq)] +pub enum DeviceInfoValue { + Configured(String), + Default(String), +} + +impl AsRef for DeviceInfoValue { + fn as_ref(&self) -> &str { + match self { + DeviceInfoValue::Configured(s) => s.as_ref(), + DeviceInfoValue::Default(s) => s.as_ref(), + } + } +} + +impl Display for DeviceInfoValue { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.as_ref()) + } +} + +#[derive(Debug)] +pub struct DeviceInfo { + pub device_id: String, + pub hardware_version: String, + pub software_version: Option, + pub software_type: Option, +} + +#[derive(PartialEq, Eq, Debug)] +pub struct DeviceInfoWarning { + line: String, + message: String, +} + +impl std::fmt::Display for DeviceInfoWarning { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Skipped line: '{}' ({})", self.line, self.message) + } +} + +impl DeviceInfo { + pub fn parse( + output: Option<&[u8]>, + defaults: T, + ) -> Result<(DeviceInfo, Vec)> { + let mut warnings = vec![]; + + let mut software_version = defaults + .software_version() + .unwrap_or_else(|_| { + warnings.push(DeviceInfoWarning { + line: "".into(), + message: "Failed to get default software version.".to_string(), + }); + None + }) + .map(DeviceInfoValue::Default); + let mut device_id = defaults.device_id().map_or_else( + |_| { + warnings.push(DeviceInfoWarning { + line: "".into(), + message: format!("Failed to open {}", DEVICE_ID_PATH), + }); + None + }, + |id| Some(id.trim().to_string()), + ); + let mut hardware_version = defaults.hardware_version().map_or_else( + |_| { + warnings.push(DeviceInfoWarning { + line: "".into(), + message: format!( + "Failed to to get hardware version from: '{}'", + HARDWARE_VERSION_COMMAND + ), + }); + None + }, + |hwv| Some(hwv.trim().to_string()), + ); + let mut software_type = defaults + .software_type() + .unwrap_or_else(|_| { + warnings.push(DeviceInfoWarning { + line: "".into(), + message: "Failed to get default software_type.".to_string(), + }); + None + }) + .map(DeviceInfoValue::Default); + + match output { + Some(output) => { + for line in std::str::from_utf8(output)?.lines() { + if let Some((key, value)) = line.split_once('=') { + match key { + "MEMFAULT_DEVICE_ID" => device_id = Some(value.into()), + "MEMFAULT_HARDWARE_VERSION" => hardware_version = Some(value.into()), + "MEMFAULT_SOFTWARE_VERSION" => { + software_version = Some(DeviceInfoValue::Configured(value.into())) + } + "MEMFAULT_SOFTWARE_TYPE" => { + software_type = Some(DeviceInfoValue::Configured(value.into())) + } + _ => warnings.push(DeviceInfoWarning { + line: line.into(), + message: "Unknown variable.".to_string(), + }), + } + } else { + warnings.push(DeviceInfoWarning { + line: line.into(), + message: "Expect '=' separated key/value pairs.".to_string(), + }) + } + } + } + None => { + warnings.push(DeviceInfoWarning { + line: "".into(), + message: "No output from memfault-device-info.".to_string(), + }); + } + } + + let di = DeviceInfo { + device_id: device_id.ok_or(eyre!("No device id supplied"))?, + hardware_version: hardware_version.ok_or(eyre!("No hardware version supplied"))?, + software_version, + software_type, + }; + + // Create vector of keys whose values have invalid characters + let validation_errors: Vec = [ + ( + "MEMFAULT_HARDWARE_VERSION", + hardware_version_is_valid(&di.hardware_version), + ), + ( + "MEMFAULT_SOFTWARE_VERSION", + di.software_version + .as_ref() + .map_or(Ok(()), |swv| software_version_is_valid(swv.as_ref())), + ), + ( + "MEMFAULT_SOFTWARE_TYPE", + di.software_type + .as_ref() + .map_or(Ok(()), |swt| software_type_is_valid(swt.as_ref())), + ), + ("MEMFAULT_DEVICE_ID", device_id_is_valid(&di.device_id)), + ] + .iter() + .filter_map(|(key, result)| match result { + Err(e) => Some(format!(" Invalid {}: {}", key, e)), + _ => None, + }) + .collect(); + + match validation_errors.is_empty() { + true => Ok((di, warnings)), + false => Err(eyre::eyre!("\n{}", validation_errors.join("\n"))), + } + } + + pub fn load() -> eyre::Result<(DeviceInfo, Vec)> { + let user_output = Command::new("memfault-device-info").output().ok(); + let stdout = user_output.as_ref().map(|o| o.stdout.as_slice()); + + let os_release = EtcOsRelease::load().ok(); + let di_defaults = DeviceInfoDefaultsImpl::new(os_release); + Self::parse(stdout, di_defaults) + } +} + +#[cfg(test)] +impl DeviceInfo { + pub fn test_fixture() -> Self { + DeviceInfo { + device_id: "001".to_owned(), + hardware_version: "DVT".to_owned(), + software_version: None, + software_type: None, + } + } + + pub fn test_fixture_with_overrides(software_version: &str, software_type: &str) -> Self { + DeviceInfo { + device_id: "001".to_owned(), + hardware_version: "DVT".to_owned(), + software_version: Some(DeviceInfoValue::Configured(software_version.into())), + software_type: Some(DeviceInfoValue::Configured(software_type.into())), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + #[test] + fn test_empty() { + let mut di_defaults = MockDeviceInfoDefaults::new(); + di_defaults + .expect_software_type() + .returning(|| Err(eyre!(""))); + di_defaults.expect_software_version().returning(|| Ok(None)); + di_defaults + .expect_hardware_version() + .returning(|| Err(eyre!(""))); + di_defaults + .expect_device_id() + .returning(|| Ok("123ABC".into())); + let r = DeviceInfo::parse(Some(b""), di_defaults); + assert!(r.is_err()) + } + + #[test] + fn test_with_warnings() { + let mut di_defaults = MockDeviceInfoDefaults::new(); + di_defaults.expect_software_type().returning(|| Ok(None)); + di_defaults.expect_software_version().returning(|| Ok(None)); + di_defaults + .expect_device_id() + .returning(|| Ok("123ABC".into())); + di_defaults + .expect_hardware_version() + .returning(|| Ok("Hardware".into())); + let r = DeviceInfo::parse( + Some(b"MEMFAULT_DEVICE_ID=X\nMEMFAULT_HARDWARE_VERSION=Y\nblahblahblah\n"), + di_defaults, + ); + assert!(r.is_ok()); + + let (di, warnings) = r.unwrap(); + assert_eq!(di.device_id, "X"); + assert_eq!(di.hardware_version, "Y"); + assert_eq!(warnings.len(), 1); + assert_eq!( + warnings[0], + DeviceInfoWarning { + line: "blahblahblah".into(), + message: "Expect '=' separated key/value pairs.".to_string() + } + ); + } + + #[rstest] + // Override software version + #[case(b"MEMFAULT_DEVICE_ID=123ABC\nMEMFAULT_HARDWARE_VERSION=1.0.0\nMEMFAULT_SOFTWARE_VERSION=1.2.3\n", Some("1.2.3".into()), None)] + // Override software type + #[case(b"MEMFAULT_DEVICE_ID=123ABC\nMEMFAULT_HARDWARE_VERSION=1.0.0\nMEMFAULT_SOFTWARE_TYPE=test\n", None, Some("test".into()))] + // Override both software version and type + #[case(b"MEMFAULT_DEVICE_ID=123ABC\nMEMFAULT_HARDWARE_VERSION=1.0.0\nMEMFAULT_SOFTWARE_VERSION=1.2.3\nMEMFAULT_SOFTWARE_TYPE=test\n", Some("1.2.3".into()), Some("test".into()))] + fn test_with_sw_version_and_type( + #[case] output: &[u8], + #[case] sw_version: Option, + #[case] sw_type: Option, + ) { + let mut di_defaults = MockDeviceInfoDefaults::new(); + di_defaults.expect_software_type().returning(|| Ok(None)); + di_defaults.expect_software_version().returning(|| Ok(None)); + di_defaults + .expect_hardware_version() + .returning(|| Ok("Hardware".into())); + di_defaults + .expect_device_id() + .returning(|| Ok("123ABC".into())); + let r = DeviceInfo::parse(Some(output), di_defaults); + assert!(r.is_ok()); + + let (di, warnings) = r.unwrap(); + assert_eq!(di.device_id, "123ABC"); + assert_eq!(di.hardware_version, "1.0.0"); + assert_eq!( + di.software_version, + sw_version.map(DeviceInfoValue::Configured) + ); + assert_eq!(di.software_type, sw_type.map(DeviceInfoValue::Configured)); + + assert_eq!(warnings.len(), 0); + } + + #[rstest] + #[case::default_with_no_response( + Some("1.2.3".to_string()), + Some(DeviceInfoValue::Default("1.2.3".to_string())), + b"" + )] + #[case::default_with_response( + Some("1.2.3".to_string()), + Some(DeviceInfoValue::Configured("1.2.4".to_string())), + b"MEMFAULT_SOFTWARE_VERSION=1.2.4" + )] + #[case::no_default_with_response( + None, + Some(DeviceInfoValue::Configured("1.2.4".to_string())), + b"MEMFAULT_SOFTWARE_VERSION=1.2.4" + )] + #[case::no_default_no_response(None, None, b"")] + fn test_with_default_swv( + #[case] software_version_default: Option, + #[case] expected: Option, + #[case] output: &[u8], + ) { + // Required device info parameters that will cause a panic if not present + let mut output_required = + b"MEMFAULT_DEVICE_ID=DEVICE\nMEMFAULT_HARDWARE_VERSION=HARDWARE\n".to_vec(); + output_required.extend(output); + + let mut di_defaults = MockDeviceInfoDefaults::new(); + di_defaults + .expect_software_type() + .returning(|| Err(eyre!(""))); + di_defaults + .expect_software_version() + .returning(move || Ok(software_version_default.clone())); + di_defaults + .expect_hardware_version() + .returning(|| Err(eyre!(""))); + di_defaults.expect_device_id().returning(|| Err(eyre!(""))); + + let (di, _warnings) = DeviceInfo::parse(Some(&output_required), di_defaults).unwrap(); + assert_eq!(di.software_version, expected); + } + + #[rstest] + #[case::default_with_no_response(Some("123ABC".to_string()), Some(DeviceInfoValue::Default("123ABC".to_string())), b"")] + #[case::default_with_response(Some("123ABC".to_string()), Some(DeviceInfoValue::Configured("main".to_string())), b"MEMFAULT_SOFTWARE_TYPE=main")] + #[case::no_default_with_response(None, Some(DeviceInfoValue::Configured("main".to_string())), b"MEMFAULT_SOFTWARE_TYPE=main")] + #[case::no_default_no_response(None, None, b"")] + fn test_with_default_sw_type( + #[case] software_type_default: Option, + #[case] expected: Option, + #[case] output: &[u8], + ) { + // Required device info parameters that will cause a panic if not present + let mut output_required = + b"MEMFAULT_DEVICE_ID=DEVICE\nMEMFAULT_HARDWARE_VERSION=HARDWARE\n".to_vec(); + output_required.extend(output); + + let mut di_defaults = MockDeviceInfoDefaults::new(); + di_defaults + .expect_software_version() + .returning(|| Err(eyre!(""))); + di_defaults + .expect_hardware_version() + .returning(|| Err(eyre!(""))); + di_defaults.expect_device_id().returning(|| Err(eyre!(""))); + di_defaults + .expect_software_type() + .returning(move || Ok(software_type_default.clone())); + + let (di, _warnings) = DeviceInfo::parse(Some(&output_required), di_defaults).unwrap(); + assert_eq!(di.software_type, expected); + } + + #[rstest] + #[case::default_with_no_response(Some("123ABC".to_string()), Some("123ABC".to_string()), b"")] + #[case::default_with_whitespace(Some("123ABC\n".to_string()), Some("123ABC".to_string()), b"")] + #[case::default_with_response(Some("123ABC".to_string()), Some("DEVICE".to_string()), b"MEMFAULT_DEVICE_ID=DEVICE")] + #[case::no_default_with_response(None, Some("DEVICE".to_string()), b"MEMFAULT_DEVICE_ID=DEVICE")] + #[case::no_default_no_response(None, None, b"")] + fn test_with_default_device_id( + #[case] device_id_default: Option, + #[case] expected: Option, + #[case] output: &[u8], + ) { + // Required device info parameters that will cause a panic if not present + let mut output_required = b"MEMFAULT_HARDWARE_VERSION=HARDWARE\n".to_vec(); + output_required.extend(output); + + let mut di_defaults = MockDeviceInfoDefaults::new(); + di_defaults + .expect_software_type() + .returning(|| Err(eyre!(""))); + di_defaults + .expect_software_version() + .returning(|| Err(eyre!(""))); + di_defaults + .expect_hardware_version() + .returning(|| Err(eyre!(""))); + di_defaults + .expect_device_id() + .returning(move || device_id_default.clone().ok_or(eyre!(""))); + + let ret = DeviceInfo::parse(Some(&output_required), di_defaults); + if let Some(expected) = expected { + let (di, _warnings) = ret.unwrap(); + assert_eq!(di.device_id, expected); + } else { + assert!(ret.is_err()); + } + } + + #[rstest] + #[case::default_with_no_response(Some("123ABC".to_string()), Some("123ABC".to_string()), b"")] + #[case::default_with_whitespace(Some("123ABC\n".to_string()), Some("123ABC".to_string()), b"")] + #[case::default_with_response(Some("123ABC".to_string()), Some("HARDWARE".to_string()), b"MEMFAULT_HARDWARE_VERSION=HARDWARE")] + #[case::no_default_with_response(None, Some("HARDWARE".to_string()), b"MEMFAULT_HARDWARE_VERSION=HARDWARE")] + #[case::no_default_no_response(None, None, b"")] + fn test_with_default_hardware_version( + #[case] hardware_version_default: Option, + #[case] expected: Option, + #[case] output: &[u8], + ) { + // Required device info parameters that will cause a panic if not present + let mut output_required = b"MEMFAULT_DEVICE_ID=DEVICE\n".to_vec(); + output_required.extend(output); + + let mut di_defaults = MockDeviceInfoDefaults::new(); + di_defaults + .expect_software_type() + .returning(|| Err(eyre!(""))); + di_defaults + .expect_software_version() + .returning(|| Err(eyre!(""))); + di_defaults.expect_device_id().returning(|| Err(eyre!(""))); + di_defaults + .expect_hardware_version() + .returning(move || hardware_version_default.clone().ok_or(eyre!(""))); + + let ret = DeviceInfo::parse(Some(&output_required), di_defaults); + if let Some(expected) = expected { + let (di, _warnings) = ret.unwrap(); + assert_eq!(di.hardware_version, expected); + } else { + assert!(ret.is_err()); + } + } + + #[rstest] + fn test_with_no_device_info() { + let expected_software_type = "SOFTWARE_TYPE".to_string(); + let expected_software_version = "SOFTWARE_VERSION".to_string(); + let expected_hardware_version = "HARDWARE_VERSION".to_string(); + let expected_device_id = "DEVICE_ID".to_string(); + + let mut di_defaults = MockDeviceInfoDefaults::new(); + let default_software_type = expected_software_type.clone(); + di_defaults + .expect_software_type() + .returning(move || Ok(Some(default_software_type.clone()))); + let default_software_version = expected_software_version.clone(); + di_defaults + .expect_software_version() + .returning(move || Ok(Some(default_software_version.clone()))); + let default_hardware_version = expected_hardware_version.clone(); + di_defaults + .expect_hardware_version() + .returning(move || Ok(default_hardware_version.clone())); + let default_device_id = expected_device_id.clone(); + di_defaults + .expect_device_id() + .returning(move || Ok(default_device_id.clone())); + + let r = DeviceInfo::parse(None, di_defaults).unwrap(); + + assert_eq!( + r.0.software_type, + Some(DeviceInfoValue::Default(expected_software_type)) + ); + assert_eq!( + r.0.software_version, + Some(DeviceInfoValue::Default(expected_software_version)) + ); + assert_eq!(r.0.hardware_version, expected_hardware_version); + assert_eq!(r.0.device_id, expected_device_id); + } +} diff --git a/memfaultd/src/config/mod.rs b/memfaultd/src/config/mod.rs new file mode 100644 index 0000000..c5c6e83 --- /dev/null +++ b/memfaultd/src/config/mod.rs @@ -0,0 +1,457 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::eyre; +use std::net::SocketAddr; +use std::time::Duration; +use std::{ + path::{Path, PathBuf}, + sync::{Arc, RwLock}, +}; + +use crate::{ + network::{NetworkClient, NetworkConfig}, + util::{DiskBacked, UnwrapOrDie, UpdateStatus}, +}; + +use crate::util::disk_size::DiskSize; + +#[cfg(test)] +pub use self::config_file::ConnectionCheckProtocol; + +#[cfg(target_os = "linux")] +pub use self::config_file::{CoredumpCaptureStrategy, CoredumpCompression}; + +use self::device_info::DeviceInfoValue; +#[cfg(test)] +pub use self::device_info::MockDeviceInfoDefaults; +pub use self::{ + config_file::{ + ConnectivityMonitorConfig, ConnectivityMonitorTarget, JsonConfigs, LogSource, + LogToMetricRule, MemfaultdConfig, SessionConfig, StorageConfig, SystemMetricConfig, + }, + device_config::{DeviceConfig, Resolution, Sampling}, + device_info::{DeviceInfo, DeviceInfoDefaultsImpl, DeviceInfoWarning}, +}; + +use crate::mar::MarEntryBuilder; +use crate::mar::Metadata; +use eyre::{Context, Result}; + +mod config_file; +mod device_config; +mod device_info; +mod utils; + +const FALLBACK_SOFTWARE_VERSION: &str = "0.0.0-memfault-unknown"; +const FALLBACK_SOFTWARE_TYPE: &str = "memfault-unknown"; + +/// Container of the entire memfaultd configuration. +/// Implement `From` trait to initialize module specific configuration (see `NetworkConfig` for example). +pub struct Config { + pub device_info: DeviceInfo, + pub config_file: MemfaultdConfig, + pub config_file_path: PathBuf, + cached_device_config: Arc>>, +} + +const LOGS_SUBDIRECTORY: &str = "logs"; +const MAR_STAGING_SUBDIRECTORY: &str = "mar"; +const DEVICE_CONFIG_FILE: &str = "device_config.json"; +const COREDUMP_RATE_LIMITER_FILENAME: &str = "coredump_rate_limit"; + +impl Config { + pub const DEFAULT_CONFIG_PATH: &'static str = "/etc/memfaultd.conf"; + + pub fn read_from_system(user_config: Option<&Path>) -> Result { + // Select config file to read + let config_file = user_config.unwrap_or_else(|| Path::new(Self::DEFAULT_CONFIG_PATH)); + + let config = MemfaultdConfig::load(config_file).wrap_err(eyre!( + "Unable to read config file {}", + &config_file.display() + ))?; + + let (device_info, warnings) = + DeviceInfo::load().wrap_err(eyre!("Unable to load device info"))?; + #[allow(clippy::print_stderr)] + warnings.iter().for_each(|w| eprintln!("{}", w)); + + let device_config = DiskBacked::from_path(&Self::device_config_path_from_config(&config)); + + Ok(Self { + device_info, + config_file: config, + config_file_path: config_file.to_owned(), + cached_device_config: Arc::new(RwLock::new(device_config)), + }) + } + + pub fn refresh_device_config(&self, client: &impl NetworkClient) -> Result { + let response = client.fetch_device_config()?; + + // Let the server know that we have applied the new version if it still + // believes we have an older one. + let confirm_version = match response.data.completed { + Some(v) if v == response.data.revision => None, + _ => Some(response.data.revision), + }; + + // Always write the config to our cache. + let new_config: DeviceConfig = response.into(); + let update_status = self + .cached_device_config + .write() + .unwrap_or_die() + .set(new_config)?; + + // After saving, create the device-config confirmation MAR entry + if let Some(revision) = confirm_version { + let mar_staging = self.mar_staging_path(); + MarEntryBuilder::new(&mar_staging)? + .set_metadata(Metadata::new_device_config(revision)) + .save(&NetworkConfig::from(self))?; + } + Ok(update_status) + } + + pub fn tmp_dir(&self) -> PathBuf { + match self.config_file.tmp_dir { + Some(ref tmp_dir) => tmp_dir.clone(), + None => self.config_file.persist_dir.clone(), + } + .into() + } + + pub fn tmp_dir_max_size(&self) -> DiskSize { + DiskSize::new_capacity(self.config_file.tmp_dir_max_usage as u64) + } + + pub fn tmp_dir_min_headroom(&self) -> DiskSize { + DiskSize { + bytes: self.config_file.tmp_dir_min_headroom as u64, + inodes: self.config_file.tmp_dir_min_inodes as u64, + } + } + + pub fn coredump_rate_limiter_file_path(&self) -> PathBuf { + self.tmp_dir().join(COREDUMP_RATE_LIMITER_FILENAME) + } + + pub fn logs_path(&self) -> PathBuf { + self.tmp_dir().join(LOGS_SUBDIRECTORY) + } + + pub fn mar_staging_path(&self) -> PathBuf { + self.tmp_dir().join(MAR_STAGING_SUBDIRECTORY) + } + + fn device_config_path_from_config(config_file: &MemfaultdConfig) -> PathBuf { + config_file.persist_dir.join(DEVICE_CONFIG_FILE) + } + pub fn device_config_path(&self) -> PathBuf { + Self::device_config_path_from_config(&self.config_file) + } + + pub fn sampling(&self) -> Sampling { + if self.config_file.enable_dev_mode { + Sampling::development() + } else { + self.device_config().sampling + } + } + + /// Returns the device_config at the time of the call. If the device_config is updated + /// after this call, the returned value will not be updated. + /// This can block for a small moment if another thread is currently updating the device_config. + fn device_config(&self) -> DeviceConfig { + self.cached_device_config + .read() + // If another thread crashed while holding the mutex we want to crash the program + .unwrap_or_die() + // If we were not able to load from local-cache then return the defaults. + .get() + .clone() + } + + /// Returns the software version for the device. + /// + /// The precedence is as follows: + /// 1. Configured software version in device_info + /// 2. Configured software version in config_file + /// 3. Default software version in device_info + /// 4. Fallback software version + pub fn software_version(&self) -> &str { + match ( + &self.device_info.software_version, + &self.config_file.software_version, + ) { + (Some(DeviceInfoValue::Configured(sw_version)), _) => sw_version.as_ref(), + (None, Some(sw_version)) => sw_version.as_ref(), + (Some(DeviceInfoValue::Default(_)), Some(sw_version)) => sw_version.as_ref(), + (Some(DeviceInfoValue::Default(sw_version)), None) => sw_version.as_ref(), + (None, None) => FALLBACK_SOFTWARE_VERSION, + } + } + + /// Returns the software type for the device. + /// + /// The precedence is as follows: + /// 1. Configured software type in device_info + /// 2. Configured software type in config_file + /// 3. Default software type in device_info + /// 4. Fallback software type + pub fn software_type(&self) -> &str { + match ( + &self.device_info.software_type, + &self.config_file.software_type, + ) { + (Some(DeviceInfoValue::Configured(software_type)), _) => software_type.as_ref(), + (None, Some(software_type)) => software_type.as_ref(), + (Some(DeviceInfoValue::Default(_)), Some(software_type)) => software_type.as_ref(), + (Some(DeviceInfoValue::Default(software_type)), None) => software_type.as_ref(), + (None, None) => FALLBACK_SOFTWARE_TYPE, + } + } + + pub fn mar_entry_max_age(&self) -> Duration { + self.config_file.mar.mar_entry_max_age + } + + pub fn battery_monitor_periodic_update_enabled(&self) -> bool { + self.config_file.battery_monitor.is_some() + } + + pub fn battery_monitor_battery_info_command(&self) -> &str { + match self.config_file.battery_monitor.as_ref() { + Some(battery_config) => battery_config.battery_info_command.as_ref(), + None => "", + } + } + + pub fn battery_monitor_interval(&self) -> Duration { + match self.config_file.battery_monitor.as_ref() { + Some(battery_config) => battery_config.interval_seconds, + None => Duration::from_secs(0), + } + } + + pub fn connectivity_monitor_config(&self) -> Option<&ConnectivityMonitorConfig> { + self.config_file.connectivity_monitor.as_ref() + } + + pub fn session_configs(&self) -> Option<&Vec> { + self.config_file.sessions.as_ref() + } + + pub fn statsd_server_enabled(&self) -> bool { + self.config_file.metrics.statsd_server.is_some() + } + + pub fn statsd_server_address(&self) -> Result { + match &self.config_file.metrics.statsd_server { + Some(statsd_server_config) => Ok(statsd_server_config.bind_address), + None => Err(eyre!("No StatsD server bind_address configured!")), + } + } + + pub fn builtin_system_metric_collection_enabled(&self) -> bool { + self.config_file.metrics.system_metric_collection.enable + } + + pub fn system_metric_poll_interval(&self) -> Duration { + self.config_file + .metrics + .system_metric_collection + .poll_interval_seconds + } + + pub fn system_metric_config(&self) -> SystemMetricConfig { + self.config_file.metrics.system_metric_collection.clone() + } +} + +#[cfg(test)] +impl Config { + pub fn test_fixture() -> Self { + Config { + device_info: DeviceInfo::test_fixture(), + config_file: MemfaultdConfig::test_fixture(), + config_file_path: PathBuf::from("test_fixture.conf"), + cached_device_config: Arc::new(RwLock::new(DiskBacked::from_path(&PathBuf::from( + "/dev/null", + )))), + } + } + + pub fn test_fixture_with_info_overrides(software_version: &str, software_type: &str) -> Self { + Config { + device_info: DeviceInfo::test_fixture_with_overrides(software_version, software_type), + config_file: MemfaultdConfig::test_fixture(), + config_file_path: PathBuf::from("test_fixture.conf"), + cached_device_config: Arc::new(RwLock::new(DiskBacked::from_path(&PathBuf::from( + "/dev/null", + )))), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use std::{fs::create_dir_all, path::PathBuf}; + + use rstest::{fixture, rstest}; + + use crate::{ + config::{device_info::DeviceInfoValue, Config}, + mar::MarEntry, + network::{ + DeviceConfigResponse, DeviceConfigResponseConfig, DeviceConfigResponseData, + DeviceConfigResponseResolution, MockNetworkClient, + }, + util::path::AbsolutePath, + }; + + #[test] + fn tmp_dir_defaults_to_persist_dir() { + let config = Config::test_fixture(); + + assert_eq!(config.tmp_dir(), config.config_file.persist_dir); + } + + #[test] + fn tmp_folder_set() { + let mut config = Config::test_fixture(); + let abs_path = PathBuf::from("/my/abs/path"); + config.config_file.tmp_dir = Some(AbsolutePath::try_from(abs_path.clone()).unwrap()); + + assert_eq!(config.tmp_dir(), abs_path); + } + + #[test] + fn test_info_overrides_file() { + let config = + Config::test_fixture_with_info_overrides("1.0.0-overridden", "overridden-type"); + + assert_eq!(config.software_version(), "1.0.0-overridden"); + assert_eq!(config.software_type(), "overridden-type"); + } + + #[rstest] + fn generate_mar_device_config_confirmation_when_needed(mut fixture: Fixture) { + fixture + .client + .expect_fetch_device_config() + .return_once(|| Ok(DEVICE_CONFIG_SAMPLE)); + fixture + .config + .refresh_device_config(&fixture.client) + .unwrap(); + + assert_eq!(fixture.count_mar_entries(), 1); + } + + #[rstest] + fn do_not_generate_mar_device_config_if_not_needed(mut fixture: Fixture) { + let mut device_config = DEVICE_CONFIG_SAMPLE; + device_config.data.completed = Some(device_config.data.revision); + + fixture + .client + .expect_fetch_device_config() + .return_once(move || Ok(device_config)); + fixture + .config + .refresh_device_config(&fixture.client) + .unwrap(); + + assert_eq!(fixture.count_mar_entries(), 0); + } + + #[rstest] + #[case(Some(DeviceInfoValue::Configured("1.0.0".into())), None, "1.0.0")] + #[case(Some(DeviceInfoValue::Default("1.0.0".into())), None, "1.0.0")] + #[case(Some(DeviceInfoValue::Configured("1.0.0".into())), Some("2.0.0"), "1.0.0")] + #[case(Some(DeviceInfoValue::Default("1.0.0".into())), Some("2.0.0"), "2.0.0")] + #[case(None, Some("2.0.0"), "2.0.0")] + #[case(None, None, FALLBACK_SOFTWARE_VERSION)] + fn software_version_precedence( + #[case] device_info_swv: Option, + #[case] config_swv: Option<&str>, + #[case] expected: &str, + ) { + let mut config = Config::test_fixture(); + config.device_info.software_version = device_info_swv; + config.config_file.software_version = config_swv.map(String::from); + + assert_eq!(config.software_version(), expected); + } + + #[rstest] + #[case(Some(DeviceInfoValue::Configured("test".into())), None, "test")] + #[case(Some(DeviceInfoValue::Default("test".into())), None, "test")] + #[case(Some(DeviceInfoValue::Configured("test".into())), Some("prod"), "test")] + #[case(Some(DeviceInfoValue::Default("test".into())), Some("prod"), "prod")] + #[case(None, Some("prod"), "prod")] + #[case(None, None, FALLBACK_SOFTWARE_TYPE)] + fn software_type_precedence( + #[case] device_info_swv: Option, + #[case] config_swv: Option<&str>, + #[case] expected: &str, + ) { + let mut config = Config::test_fixture(); + config.device_info.software_type = device_info_swv; + config.config_file.software_type = config_swv.map(String::from); + + assert_eq!(config.software_type(), expected); + } + + struct Fixture { + config: Config, + _tmp_dir: tempfile::TempDir, + client: MockNetworkClient, + } + + #[fixture] + fn fixture() -> Fixture { + Fixture::new() + } + + impl Fixture { + fn new() -> Self { + let tmp_dir = tempfile::tempdir().unwrap(); + let mut config = Config::test_fixture(); + config.config_file.persist_dir = tmp_dir.path().to_path_buf().try_into().unwrap(); + create_dir_all(config.mar_staging_path()).unwrap(); + Self { + config, + _tmp_dir: tmp_dir, + client: MockNetworkClient::new(), + } + } + + fn count_mar_entries(self) -> usize { + MarEntry::iterate_from_container(&self.config.mar_staging_path()) + .unwrap() + .count() + } + } + + const DEVICE_CONFIG_SAMPLE: DeviceConfigResponse = DeviceConfigResponse { + data: DeviceConfigResponseData { + completed: None, + revision: 42, + config: DeviceConfigResponseConfig { + memfault: crate::network::DeviceConfigResponseMemfault { + sampling: crate::network::DeviceConfigResponseSampling { + debugging_resolution: DeviceConfigResponseResolution::High, + logging_resolution: DeviceConfigResponseResolution::High, + monitoring_resolution: DeviceConfigResponseResolution::High, + }, + }, + }, + }, + }; +} diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__empty_object.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__empty_object.snap new file mode 100644 index 0000000..e207028 --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__empty_object.snap @@ -0,0 +1,73 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: content +--- +{ + "persist_dir": "/media/memfault", + "tmp_dir": null, + "tmp_dir_min_headroom_kib": 10240, + "tmp_dir_min_inodes": 100, + "tmp_dir_max_usage_kib": 102400, + "upload_interval_seconds": 3600, + "heartbeat_interval_seconds": 3600, + "enable_data_collection": false, + "enable_dev_mode": false, + "software_version": null, + "software_type": null, + "project_key": "", + "base_url": "https://device.memfault.com", + "swupdate": { + "input_file": "/etc/swupdate.cfg", + "output_file": "/tmp/swupdate.cfg" + }, + "reboot": { + "last_reboot_reason_file": "/media/last_reboot_reason" + }, + "coredump": { + "compression": "gzip", + "coredump_max_size_kib": 96000, + "rate_limit_count": 5, + "rate_limit_duration_seconds": 3600, + "capture_strategy": { + "type": "threads", + "max_thread_size_kib": 32 + }, + "log_lines": 100 + }, + "fluent-bit": { + "extra_fluentd_attributes": [], + "bind_address": "127.0.0.1:5170", + "max_buffered_lines": 1000, + "max_connections": 4 + }, + "logs": { + "rotate_size_kib": 10240, + "rotate_after_seconds": 3600, + "compression_level": 1, + "max_lines_per_minute": 500, + "log_to_metrics": null, + "storage": "persist", + "source": "fluent-bit" + }, + "mar": { + "mar_file_max_size_kib": 10240, + "mar_entry_max_age_seconds": 604800 + }, + "http_server": { + "bind_address": "127.0.0.1:8787" + }, + "battery_monitor": null, + "connectivity_monitor": null, + "sessions": null, + "metrics": { + "enable_daily_heartbeats": false, + "system_metric_collection": { + "enable": false, + "poll_interval_seconds": 10, + "processes": null, + "disk_space": null, + "network_interfaces": null + }, + "statsd_server": null + } +} diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__metrics_config.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__metrics_config.snap new file mode 100644 index 0000000..96e7659 --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__metrics_config.snap @@ -0,0 +1,79 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: content +--- +{ + "persist_dir": "/media/memfault", + "tmp_dir": null, + "tmp_dir_min_headroom_kib": 10240, + "tmp_dir_min_inodes": 100, + "tmp_dir_max_usage_kib": 102400, + "upload_interval_seconds": 3600, + "heartbeat_interval_seconds": 3600, + "enable_data_collection": false, + "enable_dev_mode": false, + "software_version": null, + "software_type": null, + "project_key": "", + "base_url": "https://device.memfault.com", + "swupdate": { + "input_file": "/etc/swupdate.cfg", + "output_file": "/tmp/swupdate.cfg" + }, + "reboot": { + "last_reboot_reason_file": "/media/last_reboot_reason" + }, + "coredump": { + "compression": "gzip", + "coredump_max_size_kib": 96000, + "rate_limit_count": 5, + "rate_limit_duration_seconds": 3600, + "capture_strategy": { + "type": "threads", + "max_thread_size_kib": 32 + }, + "log_lines": 100 + }, + "fluent-bit": { + "extra_fluentd_attributes": [], + "bind_address": "127.0.0.1:5170", + "max_buffered_lines": 1000, + "max_connections": 4 + }, + "logs": { + "rotate_size_kib": 10240, + "rotate_after_seconds": 3600, + "compression_level": 1, + "max_lines_per_minute": 500, + "log_to_metrics": null, + "storage": "persist", + "source": "fluent-bit" + }, + "mar": { + "mar_file_max_size_kib": 10240, + "mar_entry_max_age_seconds": 604800 + }, + "http_server": { + "bind_address": "127.0.0.1:8787" + }, + "battery_monitor": null, + "connectivity_monitor": null, + "sessions": null, + "metrics": { + "enable_daily_heartbeats": false, + "system_metric_collection": { + "enable": false, + "poll_interval_seconds": 10, + "processes": [ + "wefaultd" + ], + "disk_space": [ + "/dev/sda1" + ], + "network_interfaces": [ + "eth1" + ] + }, + "statsd_server": null + } +} diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__set_and_write_bool_to_runtime_config@empty_object.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__set_and_write_bool_to_runtime_config@empty_object.snap new file mode 100644 index 0000000..c350b14 --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__set_and_write_bool_to_runtime_config@empty_object.snap @@ -0,0 +1,5 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: disk_config_string +--- +"{\"key\":true}" diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__set_and_write_bool_to_runtime_config@no_file.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__set_and_write_bool_to_runtime_config@no_file.snap new file mode 100644 index 0000000..c350b14 --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__set_and_write_bool_to_runtime_config@no_file.snap @@ -0,0 +1,5 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: disk_config_string +--- +"{\"key\":true}" diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__set_and_write_bool_to_runtime_config@other_key.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__set_and_write_bool_to_runtime_config@other_key.snap new file mode 100644 index 0000000..f7a8775 --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__set_and_write_bool_to_runtime_config@other_key.snap @@ -0,0 +1,5 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: disk_config_string +--- +"{\"key\":true,\"key2\":false}" diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_connectivity_monitor.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_connectivity_monitor.snap new file mode 100644 index 0000000..9fda658 --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_connectivity_monitor.snap @@ -0,0 +1,83 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: content +--- +{ + "persist_dir": "/media/memfault", + "tmp_dir": null, + "tmp_dir_min_headroom_kib": 10240, + "tmp_dir_min_inodes": 100, + "tmp_dir_max_usage_kib": 102400, + "upload_interval_seconds": 3600, + "heartbeat_interval_seconds": 3600, + "enable_data_collection": false, + "enable_dev_mode": false, + "software_version": null, + "software_type": null, + "project_key": "", + "base_url": "https://device.memfault.com", + "swupdate": { + "input_file": "/etc/swupdate.cfg", + "output_file": "/tmp/swupdate.cfg" + }, + "reboot": { + "last_reboot_reason_file": "/media/last_reboot_reason" + }, + "coredump": { + "compression": "gzip", + "coredump_max_size_kib": 96000, + "rate_limit_count": 5, + "rate_limit_duration_seconds": 3600, + "capture_strategy": { + "type": "threads", + "max_thread_size_kib": 32 + }, + "log_lines": 100 + }, + "fluent-bit": { + "extra_fluentd_attributes": [], + "bind_address": "127.0.0.1:5170", + "max_buffered_lines": 1000, + "max_connections": 4 + }, + "logs": { + "rotate_size_kib": 10240, + "rotate_after_seconds": 3600, + "compression_level": 1, + "max_lines_per_minute": 500, + "log_to_metrics": null, + "storage": "persist", + "source": "fluent-bit" + }, + "mar": { + "mar_file_max_size_kib": 10240, + "mar_entry_max_age_seconds": 604800 + }, + "http_server": { + "bind_address": "127.0.0.1:8787" + }, + "battery_monitor": null, + "connectivity_monitor": { + "interval_seconds": 30, + "targets": [ + { + "protocol": "tcp", + "host": "8.8.8.8", + "port": 443 + } + ], + "timeout_seconds": 10 + }, + "sessions": null, + "metrics": { + "enable_daily_heartbeats": false, + "system_metric_collection": { + "enable": false, + "poll_interval_seconds": 10, + "processes": null, + "disk_space": null, + "network_interfaces": null + }, + "statsd_server": null + } +} diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_coredump_capture_strategy_threads.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_coredump_capture_strategy_threads.snap new file mode 100644 index 0000000..e207028 --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_coredump_capture_strategy_threads.snap @@ -0,0 +1,73 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: content +--- +{ + "persist_dir": "/media/memfault", + "tmp_dir": null, + "tmp_dir_min_headroom_kib": 10240, + "tmp_dir_min_inodes": 100, + "tmp_dir_max_usage_kib": 102400, + "upload_interval_seconds": 3600, + "heartbeat_interval_seconds": 3600, + "enable_data_collection": false, + "enable_dev_mode": false, + "software_version": null, + "software_type": null, + "project_key": "", + "base_url": "https://device.memfault.com", + "swupdate": { + "input_file": "/etc/swupdate.cfg", + "output_file": "/tmp/swupdate.cfg" + }, + "reboot": { + "last_reboot_reason_file": "/media/last_reboot_reason" + }, + "coredump": { + "compression": "gzip", + "coredump_max_size_kib": 96000, + "rate_limit_count": 5, + "rate_limit_duration_seconds": 3600, + "capture_strategy": { + "type": "threads", + "max_thread_size_kib": 32 + }, + "log_lines": 100 + }, + "fluent-bit": { + "extra_fluentd_attributes": [], + "bind_address": "127.0.0.1:5170", + "max_buffered_lines": 1000, + "max_connections": 4 + }, + "logs": { + "rotate_size_kib": 10240, + "rotate_after_seconds": 3600, + "compression_level": 1, + "max_lines_per_minute": 500, + "log_to_metrics": null, + "storage": "persist", + "source": "fluent-bit" + }, + "mar": { + "mar_file_max_size_kib": 10240, + "mar_entry_max_age_seconds": 604800 + }, + "http_server": { + "bind_address": "127.0.0.1:8787" + }, + "battery_monitor": null, + "connectivity_monitor": null, + "sessions": null, + "metrics": { + "enable_daily_heartbeats": false, + "system_metric_collection": { + "enable": false, + "poll_interval_seconds": 10, + "processes": null, + "disk_space": null, + "network_interfaces": null + }, + "statsd_server": null + } +} diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_log_to_metrics_rules.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_log_to_metrics_rules.snap new file mode 100644 index 0000000..286b22e --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_log_to_metrics_rules.snap @@ -0,0 +1,90 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: content +--- +{ + "persist_dir": "/media/memfault", + "tmp_dir": null, + "tmp_dir_min_headroom_kib": 10240, + "tmp_dir_min_inodes": 100, + "tmp_dir_max_usage_kib": 102400, + "upload_interval_seconds": 3600, + "heartbeat_interval_seconds": 3600, + "enable_data_collection": false, + "enable_dev_mode": false, + "software_version": null, + "software_type": null, + "project_key": "", + "base_url": "https://device.memfault.com", + "swupdate": { + "input_file": "/etc/swupdate.cfg", + "output_file": "/tmp/swupdate.cfg" + }, + "reboot": { + "last_reboot_reason_file": "/media/last_reboot_reason" + }, + "coredump": { + "compression": "gzip", + "coredump_max_size_kib": 96000, + "rate_limit_count": 5, + "rate_limit_duration_seconds": 3600, + "capture_strategy": { + "type": "threads", + "max_thread_size_kib": 32 + }, + "log_lines": 100 + }, + "fluent-bit": { + "extra_fluentd_attributes": [], + "bind_address": "127.0.0.1:5170", + "max_buffered_lines": 1000, + "max_connections": 4 + }, + "logs": { + "rotate_size_kib": 10240, + "rotate_after_seconds": 3600, + "compression_level": 1, + "max_lines_per_minute": 500, + "log_to_metrics": { + "rules": [ + { + "type": "count_matching", + "pattern": "ssh", + "metric_name": "ssh_logins", + "filter": {} + }, + { + "type": "count_matching", + "pattern": "ssh", + "metric_name": "ssh_logins", + "filter": { + "_SYSTEMD_UNIT": "sshd.service" + } + } + ] + }, + "storage": "persist", + "source": "fluent-bit" + }, + "mar": { + "mar_file_max_size_kib": 10240, + "mar_entry_max_age_seconds": 604800 + }, + "http_server": { + "bind_address": "127.0.0.1:8787" + }, + "battery_monitor": null, + "connectivity_monitor": null, + "sessions": null, + "metrics": { + "enable_daily_heartbeats": false, + "system_metric_collection": { + "enable": false, + "poll_interval_seconds": 10, + "processes": null, + "disk_space": null, + "network_interfaces": null + }, + "statsd_server": null + } +} diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_partial_logs.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_partial_logs.snap new file mode 100644 index 0000000..e207028 --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_partial_logs.snap @@ -0,0 +1,73 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: content +--- +{ + "persist_dir": "/media/memfault", + "tmp_dir": null, + "tmp_dir_min_headroom_kib": 10240, + "tmp_dir_min_inodes": 100, + "tmp_dir_max_usage_kib": 102400, + "upload_interval_seconds": 3600, + "heartbeat_interval_seconds": 3600, + "enable_data_collection": false, + "enable_dev_mode": false, + "software_version": null, + "software_type": null, + "project_key": "", + "base_url": "https://device.memfault.com", + "swupdate": { + "input_file": "/etc/swupdate.cfg", + "output_file": "/tmp/swupdate.cfg" + }, + "reboot": { + "last_reboot_reason_file": "/media/last_reboot_reason" + }, + "coredump": { + "compression": "gzip", + "coredump_max_size_kib": 96000, + "rate_limit_count": 5, + "rate_limit_duration_seconds": 3600, + "capture_strategy": { + "type": "threads", + "max_thread_size_kib": 32 + }, + "log_lines": 100 + }, + "fluent-bit": { + "extra_fluentd_attributes": [], + "bind_address": "127.0.0.1:5170", + "max_buffered_lines": 1000, + "max_connections": 4 + }, + "logs": { + "rotate_size_kib": 10240, + "rotate_after_seconds": 3600, + "compression_level": 1, + "max_lines_per_minute": 500, + "log_to_metrics": null, + "storage": "persist", + "source": "fluent-bit" + }, + "mar": { + "mar_file_max_size_kib": 10240, + "mar_entry_max_age_seconds": 604800 + }, + "http_server": { + "bind_address": "127.0.0.1:8787" + }, + "battery_monitor": null, + "connectivity_monitor": null, + "sessions": null, + "metrics": { + "enable_daily_heartbeats": false, + "system_metric_collection": { + "enable": false, + "poll_interval_seconds": 10, + "processes": null, + "disk_space": null, + "network_interfaces": null + }, + "statsd_server": null + } +} diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_sessions.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_sessions.snap new file mode 100644 index 0000000..898a424 --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__with_sessions.snap @@ -0,0 +1,81 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: content +--- +{ + "persist_dir": "/media/memfault", + "tmp_dir": null, + "tmp_dir_min_headroom_kib": 10240, + "tmp_dir_min_inodes": 100, + "tmp_dir_max_usage_kib": 102400, + "upload_interval_seconds": 3600, + "heartbeat_interval_seconds": 3600, + "enable_data_collection": false, + "enable_dev_mode": false, + "software_version": null, + "software_type": null, + "project_key": "", + "base_url": "https://device.memfault.com", + "swupdate": { + "input_file": "/etc/swupdate.cfg", + "output_file": "/tmp/swupdate.cfg" + }, + "reboot": { + "last_reboot_reason_file": "/media/last_reboot_reason" + }, + "coredump": { + "compression": "gzip", + "coredump_max_size_kib": 96000, + "rate_limit_count": 5, + "rate_limit_duration_seconds": 3600, + "capture_strategy": { + "type": "threads", + "max_thread_size_kib": 32 + }, + "log_lines": 100 + }, + "fluent-bit": { + "extra_fluentd_attributes": [], + "bind_address": "127.0.0.1:5170", + "max_buffered_lines": 1000, + "max_connections": 4 + }, + "logs": { + "rotate_size_kib": 10240, + "rotate_after_seconds": 3600, + "compression_level": 1, + "max_lines_per_minute": 500, + "log_to_metrics": null, + "storage": "persist", + "source": "fluent-bit" + }, + "mar": { + "mar_file_max_size_kib": 10240, + "mar_entry_max_age_seconds": 604800 + }, + "http_server": { + "bind_address": "127.0.0.1:8787" + }, + "battery_monitor": null, + "connectivity_monitor": null, + "sessions": [ + { + "name": "test-session", + "captured_metrics": [ + "cpu.total", + "statsd.wefaultd.blendpower" + ] + } + ], + "metrics": { + "enable_daily_heartbeats": false, + "system_metric_collection": { + "enable": false, + "poll_interval_seconds": 10, + "processes": null, + "disk_space": null, + "network_interfaces": null + }, + "statsd_server": null + } +} diff --git a/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__without_coredump_compression.snap b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__without_coredump_compression.snap new file mode 100644 index 0000000..c22b015 --- /dev/null +++ b/memfaultd/src/config/snapshots/memfaultd__config__config_file__test__without_coredump_compression.snap @@ -0,0 +1,73 @@ +--- +source: memfaultd/src/config/config_file.rs +expression: content +--- +{ + "persist_dir": "/media/memfault", + "tmp_dir": null, + "tmp_dir_min_headroom_kib": 10240, + "tmp_dir_min_inodes": 100, + "tmp_dir_max_usage_kib": 102400, + "upload_interval_seconds": 3600, + "heartbeat_interval_seconds": 3600, + "enable_data_collection": false, + "enable_dev_mode": false, + "software_version": null, + "software_type": null, + "project_key": "", + "base_url": "https://device.memfault.com", + "swupdate": { + "input_file": "/etc/swupdate.cfg", + "output_file": "/tmp/swupdate.cfg" + }, + "reboot": { + "last_reboot_reason_file": "/media/last_reboot_reason" + }, + "coredump": { + "compression": "none", + "coredump_max_size_kib": 96000, + "rate_limit_count": 5, + "rate_limit_duration_seconds": 3600, + "capture_strategy": { + "type": "threads", + "max_thread_size_kib": 32 + }, + "log_lines": 100 + }, + "fluent-bit": { + "extra_fluentd_attributes": [], + "bind_address": "127.0.0.1:5170", + "max_buffered_lines": 1000, + "max_connections": 4 + }, + "logs": { + "rotate_size_kib": 10240, + "rotate_after_seconds": 3600, + "compression_level": 1, + "max_lines_per_minute": 500, + "log_to_metrics": null, + "storage": "persist", + "source": "fluent-bit" + }, + "mar": { + "mar_file_max_size_kib": 10240, + "mar_entry_max_age_seconds": 604800 + }, + "http_server": { + "bind_address": "127.0.0.1:8787" + }, + "battery_monitor": null, + "connectivity_monitor": null, + "sessions": null, + "metrics": { + "enable_daily_heartbeats": false, + "system_metric_collection": { + "enable": false, + "poll_interval_seconds": 10, + "processes": null, + "disk_space": null, + "network_interfaces": null + }, + "statsd_server": null + } +} diff --git a/memfaultd/src/config/test-config/empty_object.json b/memfaultd/src/config/test-config/empty_object.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/memfaultd/src/config/test-config/empty_object.json @@ -0,0 +1 @@ +{} diff --git a/memfaultd/src/config/test-config/metrics_config.json b/memfaultd/src/config/test-config/metrics_config.json new file mode 100644 index 0000000..b3f1ad7 --- /dev/null +++ b/memfaultd/src/config/test-config/metrics_config.json @@ -0,0 +1,12 @@ +{ + "metrics": { + "enable_daily_heartbeats": false, + "system_metric_collection": { + "enable": false, + "poll_interval_seconds": 10, + "processes": ["wefaultd"], + "disk_space": ["/dev/sda1"], + "network_interfaces": ["eth1"] + } + } +} diff --git a/memfaultd/src/config/test-config/with_connectivity_monitor.json b/memfaultd/src/config/test-config/with_connectivity_monitor.json new file mode 100644 index 0000000..620a03d --- /dev/null +++ b/memfaultd/src/config/test-config/with_connectivity_monitor.json @@ -0,0 +1,13 @@ +{ + "connectivity_monitor": { + "interval_seconds": 30, + "timeout_seconds": 10, + "targets": [ + { + "host": "8.8.8.8", + "port": 443, + "protocol": "tcp" + } + ] + } +} diff --git a/memfaultd/src/config/test-config/with_coredump_capture_strategy_threads.json b/memfaultd/src/config/test-config/with_coredump_capture_strategy_threads.json new file mode 100644 index 0000000..6167d96 --- /dev/null +++ b/memfaultd/src/config/test-config/with_coredump_capture_strategy_threads.json @@ -0,0 +1,8 @@ +{ + "coredump": { + "capture_strategy": { + "type": "threads", + "max_thread_size_kib": 32 + } + } +} diff --git a/memfaultd/src/config/test-config/with_invalid_path.json b/memfaultd/src/config/test-config/with_invalid_path.json new file mode 100644 index 0000000..e7fc77c --- /dev/null +++ b/memfaultd/src/config/test-config/with_invalid_path.json @@ -0,0 +1,3 @@ +{ + "tmp_dir": "tmp" +} diff --git a/memfaultd/src/config/test-config/with_invalid_swt_swv.json b/memfaultd/src/config/test-config/with_invalid_swt_swv.json new file mode 100644 index 0000000..41d0b3f --- /dev/null +++ b/memfaultd/src/config/test-config/with_invalid_swt_swv.json @@ -0,0 +1,4 @@ +{ + "software_version": "1.0.0?", + "software_type": "test program" +} diff --git a/memfaultd/src/config/test-config/with_log_to_metrics_rules.json b/memfaultd/src/config/test-config/with_log_to_metrics_rules.json new file mode 100644 index 0000000..f590b8d --- /dev/null +++ b/memfaultd/src/config/test-config/with_log_to_metrics_rules.json @@ -0,0 +1,21 @@ +{ + "logs": { + "log_to_metrics": { + "rules": [ + { + "type": "count_matching", + "pattern": "ssh", + "metric_name": "ssh_logins" + }, + { + "type": "count_matching", + "pattern": "ssh", + "metric_name": "ssh_logins", + "filter": { + "_SYSTEMD_UNIT": "sshd.service" + } + } + ] + } + } +} diff --git a/memfaultd/src/config/test-config/with_partial_logs.json b/memfaultd/src/config/test-config/with_partial_logs.json new file mode 100644 index 0000000..7302b45 --- /dev/null +++ b/memfaultd/src/config/test-config/with_partial_logs.json @@ -0,0 +1,5 @@ +{ + "logs": { + "bind_address": "0.0.0.0:4242" + } +} diff --git a/memfaultd/src/config/test-config/with_sessions.json b/memfaultd/src/config/test-config/with_sessions.json new file mode 100644 index 0000000..8599462 --- /dev/null +++ b/memfaultd/src/config/test-config/with_sessions.json @@ -0,0 +1,8 @@ +{ + "sessions": [ + { + "name": "test-session", + "captured_metrics": ["cpu.total", "statsd.wefaultd.blendpower"] + } + ] +} diff --git a/memfaultd/src/config/test-config/with_sessions_invalid_metric_name.json b/memfaultd/src/config/test-config/with_sessions_invalid_metric_name.json new file mode 100644 index 0000000..849a219 --- /dev/null +++ b/memfaultd/src/config/test-config/with_sessions_invalid_metric_name.json @@ -0,0 +1,11 @@ +{ + "sessions": [ + { + "name": "test-session", + "captured_metrics": [ + "string-with-non-ascii-\u{1F4A9}", + "statsd.wefaultd.blendpower" + ] + } + ] +} diff --git a/memfaultd/src/config/test-config/with_sessions_invalid_session_name.json b/memfaultd/src/config/test-config/with_sessions_invalid_session_name.json new file mode 100644 index 0000000..e6c0a72 --- /dev/null +++ b/memfaultd/src/config/test-config/with_sessions_invalid_session_name.json @@ -0,0 +1,8 @@ +{ + "sessions": [ + { + "name": "123-test-session", + "captured_metrics": ["cpu.total", "statsd.wefaultd.blendpower"] + } + ] +} diff --git a/memfaultd/src/config/test-config/without_coredump_compression.json b/memfaultd/src/config/test-config/without_coredump_compression.json new file mode 100644 index 0000000..03e6be4 --- /dev/null +++ b/memfaultd/src/config/test-config/without_coredump_compression.json @@ -0,0 +1,5 @@ +{ + "coredump": { + "compression": "none" + } +} diff --git a/memfaultd/src/config/utils.rs b/memfaultd/src/config/utils.rs new file mode 100644 index 0000000..0d371b0 --- /dev/null +++ b/memfaultd/src/config/utils.rs @@ -0,0 +1,48 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::util::patterns::{ + alphanum_slug_dots_colon_is_valid, alphanum_slug_dots_colon_spaces_parens_slash_is_valid, + alphanum_slug_is_valid, +}; + +pub fn software_type_is_valid(s: &str) -> eyre::Result<()> { + alphanum_slug_dots_colon_is_valid(s, 128) +} + +pub fn software_version_is_valid(s: &str) -> eyre::Result<()> { + alphanum_slug_dots_colon_spaces_parens_slash_is_valid(s, 128) +} + +pub fn hardware_version_is_valid(s: &str) -> eyre::Result<()> { + alphanum_slug_dots_colon_is_valid(s, 128) +} + +pub fn device_id_is_valid(id: &str) -> eyre::Result<()> { + alphanum_slug_is_valid(id, 128) +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + #[rstest] + // Minimum 1 character + #[case("A", true)] + // Allowed characters + #[case( + "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyz_-", + true + )] + // Disallowed characters + #[case("DEMO.1234", false)] + #[case("DEMO 1234", false)] + // Too short (0 characters) + #[case("", false)] + // Too long (129 characters) + #[case("012345679012345679012345679012345679012345679012345679012345679012345679012345679012345679012345679012345679012345678901234567890", false)] + fn device_id_is_valid_works(#[case] device_id: &str, #[case] expected: bool) { + assert_eq!(device_id_is_valid(device_id).is_ok(), expected); + } +} diff --git a/memfaultd/src/coredump/mod.rs b/memfaultd/src/coredump/mod.rs new file mode 100644 index 0000000..e57d58d --- /dev/null +++ b/memfaultd/src/coredump/mod.rs @@ -0,0 +1,30 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::Result; +use std::path::Path; + +#[cfg_attr(not(target_os = "linux"), allow(unused_variables))] +pub fn coredump_configure_kernel(config_path: &Path) -> Result<()> { + #[cfg(target_os = "linux")] + { + use eyre::Context; + use std::fs::write; + + write( + "/proc/sys/kernel/core_pattern", + format!( + "|/usr/sbin/memfault-core-handler -c {} %P", + config_path.display() + ), + ) + .wrap_err("Unable to write coredump pattern") + } + #[cfg(not(target_os = "linux"))] + { + use log::warn; + + warn!("Skipping coredump setting on non-Linux systems."); + Ok(()) + } +} diff --git a/memfaultd/src/fluent_bit/decode_time.rs b/memfaultd/src/fluent_bit/decode_time.rs new file mode 100644 index 0000000..70da2e9 --- /dev/null +++ b/memfaultd/src/fluent_bit/decode_time.rs @@ -0,0 +1,233 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! This module implements a custom deserializer for FluentBit timestamps. +//! +//! From +//! +//! > EventTime uses msgpack extension format of type 0 to carry nanosecond precision of time. +//! > +//! > Client MAY send EventTime instead of plain integer representation of second since unix epoch. +//! > Server SHOULD accept both formats of integer and EventTime. +//! > Binary representation of EventTime may be fixext or ext(with length 8). + +use core::fmt; +use std::{collections::HashMap, fmt::Formatter}; + +use chrono::{DateTime, TimeZone, Utc}; +use serde::{ + de::{Error, Visitor}, + Deserialize, Deserializer, Serialize, Serializer, +}; +use serde_bytes::ByteBuf; + +/// Deserialize a FluentBit time which can be a u32 timestamp or an EventTime +pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let tv = TimeVisitor {}; + deserializer.deserialize_any(tv) +} + +/// Serialize a FluentBit time as an EventTime +pub fn serialize(time: &DateTime, serializer: S) -> Result +where + S: Serializer, +{ + // From https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#eventtime-ext-format + let mut buf = ByteBuf::with_capacity(8); + buf.extend_from_slice(&i32::to_be_bytes(time.timestamp() as i32)); + buf.extend_from_slice(&i32::to_be_bytes(time.timestamp_subsec_nanos() as i32)); + + FluentdTimeExtType((FLUENTD_TIME_EXT_TYPE, buf)).serialize(serializer) +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename = "_ExtStruct")] +/// This is how Msgpack Ext type is represented by rmp_serde: +/// See: https://docs.rs/rmp-serde/latest/rmp_serde/constant.MSGPACK_EXT_STRUCT_NAME.html +/// And https://docs.racket-lang.org/msgpack/index.html#%28part._.Message.Pack_extension_type%29 +struct FluentdTimeExtType((i8, ByteBuf)); +const FLUENTD_TIME_EXT_TYPE: i8 = 0; + +/// Visit a FluentBit time which can be a u32 timestamp or an EventTime +/// (extended type with nanoseconds precision). +struct TimeVisitor {} + +impl<'de> Visitor<'de> for TimeVisitor { + type Value = DateTime; + + fn expecting(&self, formatter: &mut Formatter) -> fmt::Result { + write!(formatter, "an integer or a Ext/FixExt with length 8") + } + + // Called when the time is provided as an unsigned 32 bit value. + fn visit_u32(self, v: u32) -> Result + where + E: serde::de::Error, + { + Utc.timestamp_opt(v as i64, 0) + .single() + .ok_or_else(|| Error::custom("Invalid timestamp")) + } + + // Called when the time is provided as an EventTime. + fn visit_newtype_struct(self, deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_any(EventTimeVisitor {}) + } + + // Since fluent-bit 2.1, the timestamp is provided in a seq followed by a map of optional metadata. + // See https://github.com/fluent/fluent-bit/issues/6666 + // Before: + // (ExtType(code=0, data=b'd\xc0\xee\x7f7a\xe4\x89'), {'rand_value': 13625873794586244841}) + // After: + // ((ExtType(code=0, data=b'd\xc0\xeeT79\xc5g'), {}), {'rand_value': 1235066654796201019}) + // + // We currently just ignore the map of metadata. + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let a: FluentdTimeExtType = seq.next_element()?.ok_or_else(|| { + Error::custom("Invalid timestamp - expected an extension type with tag 0") + })?; + if a.0 .0 != FLUENTD_TIME_EXT_TYPE { + return Err(Error::custom("Invalid timestamp tag")); + } + + let ts = bytes_to_timestamp(a.0 .1)?; + let _ignored_metadata = seq.next_element::>()?; + Ok(ts) + } +} + +/// Visit a FluentBit EventTime (an extended type with nanosecond precision). +struct EventTimeVisitor {} + +impl<'de> Visitor<'de> for EventTimeVisitor { + type Value = DateTime; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(formatter, " a Ext/FixExt with length 8") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let tag = seq.next_element::()?; + let buf = seq.next_element::()?; + + // Validate the tag value is 0 for a timestamp. + match (tag, buf) { + (Some(FLUENTD_TIME_EXT_TYPE), Some(bytes)) => bytes_to_timestamp(bytes), + (Some(tag), _) => Err(serde::de::Error::custom(format!( + "Invalid tag {} - expected 0.", + tag + ))), + _ => Err(serde::de::Error::custom("Invalid event tag.")), + } + } +} + +/// Convert a byte buffer to a timestamp. +fn bytes_to_timestamp(bytes: ByteBuf) -> Result, E> +where + E: serde::de::Error, +{ + if bytes.len() == 8 { + // We verified that bytes is long enough so bytes[0..4] will + // never fail. It will return a [u8] of length 4. + // We still need `.try_into()` to convert [u8] into [u8; 4] + // because the compiler cannot verify that the length is 4 at + // compile time. #failproof™ + let seconds_bytes: [u8; 4] = bytes[0..4].try_into().unwrap(); + let nanoseconds_bytes: [u8; 4] = bytes[4..].try_into().unwrap(); + Utc.timestamp_opt( + u32::from_be_bytes(seconds_bytes) as i64, + u32::from_be_bytes(nanoseconds_bytes), + ) + .single() + .ok_or_else(|| Error::custom("Invalid timestamp")) + } else { + Err(E::custom("Invalid buffer length.")) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::deserialize; + + // This test make sure we are able to deserialize the three documentated + // variants of time encoding (the third argument specifies the variant to + // use). + #[rstest] + #[case(0, 0, serialize_fixext8)] + #[case(1675709515, 276*1_000_000, serialize_fixext8)] + #[case(1675709515, 276*1_000_000, serialize_varext8)] + #[case(1675709515, 0, serialize_integer)] + fn decode_encoded_time( + #[case] seconds: i32, + #[case] nanoseconds: i32, + #[case] serialize: fn(i32, i32) -> Vec, + ) { + let buf = serialize(seconds, nanoseconds); + let mut deserializer = rmp_serde::Deserializer::new(&buf[..]); + let t = deserialize(&mut deserializer).expect("should be deserializable"); + + assert_eq!(t.timestamp(), seconds as i64); + assert_eq!( + t.timestamp_nanos() - t.timestamp() * 1_000_000_000, + nanoseconds as i64 + ); + } + + #[rstest] + fn decode_ext_buffer_too_small() { + let buf = serialize_fixext8(1675709515, 0); + let mut deserializer = rmp_serde::Deserializer::new(&buf[..(buf.len() - 2)]); + + let e = deserialize(&mut deserializer).err().unwrap(); + assert!(e.to_string().contains("unexpected end of file"),); + } + + #[rstest] + fn decode_ext_invalid_tag() { + let mut buf = serialize_fixext8(1675709515, 0); + buf[1] = 0x42; + let mut deserializer = rmp_serde::Deserializer::new(&buf[..]); + + let e = deserialize(&mut deserializer).err().unwrap(); + assert!(e.to_string().contains("Invalid tag"),); + } + + fn serialize_fixext8(seconds: i32, nanoseconds: i32) -> Vec { + // From https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#eventtime-ext-format + let mut buf = vec![0xd7, 0x00]; + buf.extend_from_slice(&i32::to_be_bytes(seconds)); + buf.extend_from_slice(&i32::to_be_bytes(nanoseconds)); + buf + } + + fn serialize_varext8(seconds: i32, nanoseconds: i32) -> Vec { + // From https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#eventtime-ext-format + let mut buf = vec![0xC7, 0x08, 0x00]; + buf.extend_from_slice(&i32::to_be_bytes(seconds)); + buf.extend_from_slice(&i32::to_be_bytes(nanoseconds)); + buf + } + + fn serialize_integer(seconds: i32, _nanoseconds: i32) -> Vec { + // Fluentd spec says we should support time encoded as an integer + // https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#eventtime-ext-format + // Integers look like this: https://github.com/msgpack/msgpack/blob/master/spec.md#int-format-family + let mut buf = vec![0xCE]; + buf.extend_from_slice(&i32::to_be_bytes(seconds)); + buf + } +} diff --git a/memfaultd/src/fluent_bit/mod.rs b/memfaultd/src/fluent_bit/mod.rs new file mode 100644 index 0000000..d636089 --- /dev/null +++ b/memfaultd/src/fluent_bit/mod.rs @@ -0,0 +1,380 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! fluent-bit +//! +//! Provides FluentBitConnectionHandler to handle to TCP connections from +//! fluent-bit. A threadpool is used to limit the number of active connections at +//! a given time. +//! +//! The start() function returns a multi-producer single-consumer channel in +//! which the messages will be delivered. +//! +//! Messages are deserialized into FluentdMessage instances. +//! +//! We set a limit on the number of messages in the channel. If messages are not +//! consumed, the FluentBitReceiver will start to apply backpressure on +//! fluent-bitbit server. +//! +use std::net::TcpStream; +use std::sync::mpsc::{Receiver, SyncSender}; +use std::{collections::HashMap, net::SocketAddr}; + +use chrono::{DateTime, Utc}; +use eyre::Result; +use log::warn; +use rmp_serde::Deserializer; +use serde::{Deserialize, Serialize}; + +use crate::{config::Config, logs::log_entry::LogEntry}; +use crate::{ + logs::log_entry::LogValue, + util::tcp_server::{TcpConnectionHandler, TcpNullConnectionHandler, ThreadedTcpServer}, +}; + +mod decode_time; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum FluentdValue { + String(String), + Float(f64), +} + +impl From for LogValue { + fn from(value: FluentdValue) -> Self { + match value { + FluentdValue::String(s) => LogValue::String(s), + FluentdValue::Float(f) => LogValue::Float(f), + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct FluentdMessage( + #[serde(with = "decode_time")] pub DateTime, + pub HashMap, +); + +impl From for LogEntry { + fn from(value: FluentdMessage) -> Self { + let data = value.1.into_iter().map(|(k, v)| (k, v.into())).collect(); + LogEntry { ts: value.0, data } + } +} + +#[derive(Clone)] +pub struct FluentBitConnectionHandler { + sender: SyncSender, +} + +impl FluentBitConnectionHandler { + /// Starts the fluent-bit server with a handler delivers parsed messages to a receiver channel. + pub fn start(config: FluentBitConfig) -> Result<(ThreadedTcpServer, Receiver)> { + let (sender, receiver) = std::sync::mpsc::sync_channel(config.max_buffered_lines); + let server = ThreadedTcpServer::start( + config.bind_address, + config.max_connections, + FluentBitConnectionHandler { sender }, + )?; + Ok((server, receiver)) + } + + /// Starts the fluent-bit server with a handler that drops all data. + /// This is used in case data collection is disabled. We want to keep servicing fluent-bit in + /// this scenario, to avoid it retrying and buffering up data. + pub fn start_null(config: FluentBitConfig) -> Result { + ThreadedTcpServer::start( + config.bind_address, + config.max_connections, + TcpNullConnectionHandler {}, + ) + } +} + +impl TcpConnectionHandler for FluentBitConnectionHandler { + fn handle_connection(&self, stream: TcpStream) -> Result<()> { + let mut de = Deserializer::new(stream); + + loop { + match FluentdMessage::deserialize(&mut de) { + Ok(msg) => { + if self.sender.send(msg).is_err() { + // An error indicates that the channel has been closed, we should + // kill this thread. + break; + } + } + Err(e) => { + match e { + rmp_serde::decode::Error::InvalidMarkerRead(e) + if e.kind() == std::io::ErrorKind::UnexpectedEof => + { + // silently ignore end of stream + } + _ => warn!("FluentD decoding error: {:?}", e), + } + // After any deserialization error, we want to kill the connection. + break; + } + } + } + Ok(()) + } +} + +pub struct FluentBitConfig { + bind_address: SocketAddr, + max_buffered_lines: usize, + max_connections: usize, +} + +impl From<&Config> for FluentBitConfig { + fn from(config: &Config) -> Self { + Self { + bind_address: config.config_file.fluent_bit.bind_address, + max_buffered_lines: config.config_file.fluent_bit.max_buffered_lines, + max_connections: config.config_file.fluent_bit.max_connections, + } + } +} + +#[cfg(test)] +mod tests { + use std::net::TcpListener; + use std::{ + io::Write, net::Shutdown, sync::mpsc::sync_channel, thread, thread::JoinHandle, + time::Duration, + }; + + use rstest::{fixture, rstest}; + + use crate::test_utils::setup_logger; + + use super::*; + + #[rstest] + #[cfg_attr(not(target_os = "linux"), allow(unused_variables, unused_mut))] + fn deserialize_bogus_message(_setup_logger: (), mut connection: FluentBitFixture) { + // this test can flake on macOS. Run it only on Linux to avoid flaking Mac CI + #[cfg(target_os = "linux")] + { + connection.client.write_all("bogus".as_bytes()).unwrap(); + connection.client.shutdown(Shutdown::Both).unwrap(); + + // Make sure there is nothing received + let received = connection.receiver.recv(); + assert!(received.is_err()); + + // The handler should return without an error + assert!(connection.thread.join().is_ok()); + } + } + + #[rstest] + fn deserialize_one_message( + _setup_logger: (), + mut connection: FluentBitFixture, + message: FluentBitMessageFixture, + ) { + connection.client.write_all(&message.bytes).unwrap(); + connection.client.shutdown(Shutdown::Both).unwrap(); + + // Make sure message is received + let received = connection.receiver.recv().unwrap(); + assert_eq!(received.0, message.msg.0); + assert_eq!( + serde_json::to_string(&received.1).unwrap(), + serde_json::to_string(&message.msg.1).unwrap() + ); + + // The handler should return without an error + assert!(connection.thread.join().is_ok()); + } + + #[rstest] + fn deserialize_one_message_received_in_two_parts( + _setup_logger: (), + mut connection: FluentBitFixture, + message: FluentBitMessageFixture, + ) { + let buf1 = &message.bytes[0..10]; + let buf2 = &message.bytes[10..]; + + connection.client.write_all(buf1).unwrap(); + connection.client.flush().unwrap(); + // Make sure the other thread has time to do something + thread::sleep(Duration::from_millis(5)); + connection.client.write_all(buf2).unwrap(); + connection.client.shutdown(Shutdown::Both).unwrap(); + + // Make sure message is received + let received = connection.receiver.recv().unwrap(); + assert_eq!(received.0, message.msg.0); + assert_eq!( + serde_json::to_string(&received.1).unwrap(), + serde_json::to_string(&message.msg.1).unwrap() + ); + + // The handler should return without an error + assert!(connection.thread.join().is_ok()); + } + + #[rstest] + fn deserialize_two_concatenated_messages( + _setup_logger: (), + mut connection: FluentBitFixture, + message: FluentBitMessageFixture, + #[from(message)] message2: FluentBitMessageFixture, + ) { + let mut buf = message.bytes.clone(); + buf.extend(message2.bytes); + connection.client.write_all(&buf).unwrap(); + connection.client.shutdown(Shutdown::Both).unwrap(); + + // Make sure two messages are received + let received1 = connection.receiver.recv().unwrap(); + let received2 = connection.receiver.recv().unwrap(); + assert_eq!(received1.0, message.msg.0); + assert_eq!( + serde_json::to_string(&received1.1).unwrap(), + serde_json::to_string(&message.msg.1).unwrap() + ); + + assert_eq!(received2.0, message2.msg.0); + assert_eq!( + serde_json::to_string(&received2.1).unwrap(), + serde_json::to_string(&message2.msg.1).unwrap() + ); + + // The handler should return without an error + assert!(connection.thread.join().is_ok()); + } + + #[rstest] + /// Test the new data format with metadata associated to the timestamp (fluent-bit 2.1+) + fn deserialize_timestamp_with_metadata(_setup_logger: (), mut connection: FluentBitFixture) { + let buf = [ + 0xDD, 0x00, 0x00, 0x00, 0x02, 0xDD, 0x00, 0x00, 0x00, 0x02, 0xD7, 0x00, 0x64, 0x1B, + 0xD5, 0xF9, 0x1E, 0x4A, 0xFD, 0x58, 0xDF, 0x00, 0x00, 0x00, 0x00, 0xDF, 0x00, 0x00, + 0x00, 0x15, 0xAA, 0x5F, 0x54, 0x52, 0x41, 0x4E, 0x53, 0x50, 0x4F, 0x52, 0x54, 0xA6, + 0x73, 0x74, 0x64, 0x6F, 0x75, 0x74, 0xAA, 0x5F, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4D, + 0x5F, 0x49, 0x44, 0xD9, 0x20, 0x65, 0x62, 0x33, 0x37, 0x39, 0x30, 0x37, 0x62, 0x63, + 0x33, 0x34, 0x61, 0x34, 0x31, 0x64, 0x63, 0x62, 0x34, 0x61, 0x37, 0x65, 0x36, 0x63, + 0x37, 0x33, 0x61, 0x30, 0x32, 0x61, 0x66, 0x30, 0x32, 0xA8, 0x50, 0x52, 0x49, 0x4F, + 0x52, 0x49, 0x54, 0x59, 0xA1, 0x36, 0xAF, 0x53, 0x59, 0x53, 0x4C, 0x4F, 0x47, 0x5F, + 0x46, 0x41, 0x43, 0x49, 0x4C, 0x49, 0x54, 0x59, 0xA1, 0x33, 0xB1, 0x53, 0x59, 0x53, + 0x4C, 0x4F, 0x47, 0x5F, 0x49, 0x44, 0x45, 0x4E, 0x54, 0x49, 0x46, 0x49, 0x45, 0x52, + 0xA4, 0x67, 0x65, 0x74, 0x68, 0xA4, 0x5F, 0x50, 0x49, 0x44, 0xA3, 0x37, 0x37, 0x38, + 0xA4, 0x5F, 0x55, 0x49, 0x44, 0xA4, 0x31, 0x30, 0x30, 0x35, 0xA4, 0x5F, 0x47, 0x49, + 0x44, 0xA4, 0x31, 0x30, 0x30, 0x35, 0xA5, 0x5F, 0x43, 0x4F, 0x4D, 0x4D, 0xA4, 0x67, + 0x65, 0x74, 0x68, 0xA4, 0x5F, 0x45, 0x58, 0x45, 0xB3, 0x2F, 0x75, 0x73, 0x72, 0x2F, + 0x6C, 0x6F, 0x63, 0x61, 0x6C, 0x2F, 0x62, 0x69, 0x6E, 0x2F, 0x67, 0x65, 0x74, 0x68, + 0xA8, 0x5F, 0x43, 0x4D, 0x44, 0x4C, 0x49, 0x4E, 0x45, 0xD9, 0x9D, 0x2F, 0x75, 0x73, + 0x72, 0x2F, 0x6C, 0x6F, 0x63, 0x61, 0x6C, 0x2F, 0x62, 0x69, 0x6E, 0x2F, 0x67, 0x65, + 0x74, 0x68, 0x20, 0x2D, 0x2D, 0x6D, 0x61, 0x69, 0x6E, 0x6E, 0x65, 0x74, 0x20, 0x2D, + 0x2D, 0x64, 0x61, 0x74, 0x61, 0x64, 0x69, 0x72, 0x20, 0x2F, 0x64, 0x61, 0x74, 0x61, + 0x2D, 0x65, 0x74, 0x68, 0x2F, 0x67, 0x65, 0x74, 0x68, 0x20, 0x2D, 0x2D, 0x61, 0x75, + 0x74, 0x68, 0x72, 0x70, 0x63, 0x2E, 0x6A, 0x77, 0x74, 0x73, 0x65, 0x63, 0x72, 0x65, + 0x74, 0x20, 0x2F, 0x64, 0x61, 0x74, 0x61, 0x2D, 0x65, 0x74, 0x68, 0x2F, 0x6A, 0x77, + 0x74, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x2F, 0x6A, 0x77, 0x74, 0x2E, 0x68, 0x65, + 0x78, 0x20, 0x2D, 0x2D, 0x6D, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x20, 0x2D, 0x2D, + 0x6D, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2E, 0x61, 0x64, 0x64, 0x72, 0x20, 0x31, + 0x32, 0x37, 0x2E, 0x30, 0x2E, 0x30, 0x2E, 0x31, 0x20, 0x2D, 0x2D, 0x6D, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x73, 0x2E, 0x70, 0x6F, 0x72, 0x74, 0x20, 0x36, 0x30, 0x36, 0x31, + 0xAE, 0x5F, 0x43, 0x41, 0x50, 0x5F, 0x45, 0x46, 0x46, 0x45, 0x43, 0x54, 0x49, 0x56, + 0x45, 0xA1, 0x30, 0xB0, 0x5F, 0x53, 0x45, 0x4C, 0x49, 0x4E, 0x55, 0x58, 0x5F, 0x43, + 0x4F, 0x4E, 0x54, 0x45, 0x58, 0x54, 0xAB, 0x75, 0x6E, 0x63, 0x6F, 0x6E, 0x66, 0x69, + 0x6E, 0x65, 0x64, 0x0A, 0xAF, 0x5F, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4D, 0x44, 0x5F, + 0x43, 0x47, 0x52, 0x4F, 0x55, 0x50, 0xD9, 0x22, 0x2F, 0x73, 0x79, 0x73, 0x74, 0x65, + 0x6D, 0x2E, 0x73, 0x6C, 0x69, 0x63, 0x65, 0x2F, 0x67, 0x65, 0x74, 0x68, 0x2D, 0x6D, + 0x61, 0x69, 0x6E, 0x6E, 0x65, 0x74, 0x2E, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0xAD, 0x5F, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4D, 0x44, 0x5F, 0x55, 0x4E, 0x49, 0x54, + 0xB4, 0x67, 0x65, 0x74, 0x68, 0x2D, 0x6D, 0x61, 0x69, 0x6E, 0x6E, 0x65, 0x74, 0x2E, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0xAE, 0x5F, 0x53, 0x59, 0x53, 0x54, 0x45, + 0x4D, 0x44, 0x5F, 0x53, 0x4C, 0x49, 0x43, 0x45, 0xAC, 0x73, 0x79, 0x73, 0x74, 0x65, + 0x6D, 0x2E, 0x73, 0x6C, 0x69, 0x63, 0x65, 0xB6, 0x5F, 0x53, 0x59, 0x53, 0x54, 0x45, + 0x4D, 0x44, 0x5F, 0x49, 0x4E, 0x56, 0x4F, 0x43, 0x41, 0x54, 0x49, 0x4F, 0x4E, 0x5F, + 0x49, 0x44, 0xD9, 0x20, 0x35, 0x33, 0x30, 0x33, 0x35, 0x36, 0x66, 0x36, 0x33, 0x63, + 0x64, 0x31, 0x34, 0x64, 0x64, 0x61, 0x62, 0x30, 0x66, 0x65, 0x64, 0x31, 0x65, 0x38, + 0x30, 0x64, 0x36, 0x61, 0x30, 0x35, 0x65, 0x63, 0xA8, 0x5F, 0x42, 0x4F, 0x4F, 0x54, + 0x5F, 0x49, 0x44, 0xD9, 0x20, 0x62, 0x36, 0x36, 0x36, 0x31, 0x61, 0x66, 0x39, 0x36, + 0x33, 0x64, 0x38, 0x34, 0x61, 0x62, 0x37, 0x39, 0x62, 0x33, 0x38, 0x32, 0x33, 0x63, + 0x62, 0x65, 0x66, 0x35, 0x39, 0x66, 0x37, 0x33, 0x64, 0xAB, 0x5F, 0x4D, 0x41, 0x43, + 0x48, 0x49, 0x4E, 0x45, 0x5F, 0x49, 0x44, 0xD9, 0x20, 0x35, 0x61, 0x32, 0x63, 0x39, + 0x38, 0x66, 0x38, 0x35, 0x31, 0x64, 0x32, 0x34, 0x33, 0x64, 0x33, 0x38, 0x36, 0x61, + 0x62, 0x62, 0x35, 0x64, 0x37, 0x62, 0x39, 0x31, 0x32, 0x64, 0x64, 0x31, 0x66, 0xA9, + 0x5F, 0x48, 0x4F, 0x53, 0x54, 0x4E, 0x41, 0x4D, 0x45, 0xA7, 0x66, 0x72, 0x61, 0x63, + 0x74, 0x61, 0x6C, 0xA7, 0x4D, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0xD9, 0xC2, 0x49, + 0x4E, 0x46, 0x4F, 0x20, 0x5B, 0x30, 0x33, 0x2D, 0x32, 0x33, 0x7C, 0x30, 0x34, 0x3A, + 0x33, 0x30, 0x3A, 0x34, 0x39, 0x2E, 0x35, 0x30, 0x38, 0x5D, 0x20, 0x49, 0x6D, 0x70, + 0x6F, 0x72, 0x74, 0x65, 0x64, 0x20, 0x6E, 0x65, 0x77, 0x20, 0x70, 0x6F, 0x74, 0x65, + 0x6E, 0x74, 0x69, 0x61, 0x6C, 0x20, 0x63, 0x68, 0x61, 0x69, 0x6E, 0x20, 0x73, 0x65, + 0x67, 0x6D, 0x65, 0x6E, 0x74, 0x20, 0x20, 0x20, 0x20, 0x20, 0x62, 0x6C, 0x6F, 0x63, + 0x6B, 0x73, 0x3D, 0x31, 0x20, 0x20, 0x20, 0x20, 0x74, 0x78, 0x73, 0x3D, 0x36, 0x30, + 0x30, 0x20, 0x20, 0x20, 0x20, 0x20, 0x6D, 0x67, 0x61, 0x73, 0x3D, 0x32, 0x34, 0x2E, + 0x36, 0x39, 0x37, 0x20, 0x20, 0x65, 0x6C, 0x61, 0x70, 0x73, 0x65, 0x64, 0x3D, 0x33, + 0x31, 0x36, 0x2E, 0x35, 0x39, 0x34, 0x6D, 0x73, 0x20, 0x20, 0x20, 0x20, 0x6D, 0x67, + 0x61, 0x73, 0x70, 0x73, 0x3D, 0x37, 0x38, 0x2E, 0x30, 0x30, 0x37, 0x20, 0x20, 0x6E, + 0x75, 0x6D, 0x62, 0x65, 0x72, 0x3D, 0x31, 0x36, 0x2C, 0x38, 0x38, 0x37, 0x2C, 0x38, + 0x38, 0x36, 0x20, 0x68, 0x61, 0x73, 0x68, 0x3D, 0x30, 0x37, 0x63, 0x33, 0x65, 0x36, + 0x2E, 0x2E, 0x63, 0x34, 0x36, 0x37, 0x33, 0x39, 0x20, 0x64, 0x69, 0x72, 0x74, 0x79, + 0x3D, 0x31, 0x30, 0x32, 0x32, 0x2E, 0x34, 0x30, 0x4D, 0x69, 0x42, + ]; + + connection.client.write_all(&buf).unwrap(); + + // Make sure one messages is received + let _received1 = connection + .receiver + .recv_timeout(Duration::from_millis(10)) + .unwrap(); + + // The handler should return without an error + connection.client.shutdown(Shutdown::Both).unwrap(); + assert!(connection.thread.join().is_ok()); + } + + struct FluentBitFixture { + client: TcpStream, + thread: JoinHandle>, + receiver: Receiver, + } + + #[fixture] + fn connection() -> FluentBitFixture { + let (sender, receiver) = sync_channel(1); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let local_address = listener.local_addr().unwrap(); + + let client = TcpStream::connect(local_address).unwrap(); + let (server, _) = listener.accept().unwrap(); + + let handler = FluentBitConnectionHandler { sender }; + let thread = thread::spawn(move || handler.handle_connection(server)); + + FluentBitFixture { + client, + thread, + receiver, + } + } + + struct FluentBitMessageFixture { + msg: FluentdMessage, + bytes: Vec, + } + + #[fixture] + fn message() -> FluentBitMessageFixture { + let msg = FluentdMessage( + Utc::now(), + HashMap::from([( + "MESSAGE".to_owned(), + FluentdValue::String("something happened on the way to the moon".into()), + )]), + ); + let bytes = rmp_serde::to_vec(&msg).unwrap(); + FluentBitMessageFixture { msg, bytes } + } +} diff --git a/memfaultd/src/http_server/handler.rs b/memfaultd/src/http_server/handler.rs new file mode 100644 index 0000000..43e07b8 --- /dev/null +++ b/memfaultd/src/http_server/handler.rs @@ -0,0 +1,94 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::Result; +use tiny_http::{Request, ResponseBox}; + +/// Return type of the HttpHandler +pub enum HttpHandlerResult { + /// Request was processed and response is proposed. + Response(ResponseBox), + /// An error occurred while processing the request (will return 500). + Error(String), + /// Request not handled by this handler. Continue to next handler. + NotHandled, +} + +/// This little helper makes it possible to use the ? operator in handlers when +/// you have already checked method and path and know that they should handle +/// the request, possibly failing while doing so. +/// +/// ``` +/// # use eyre::Result; +/// use tiny_http::{Request, Response, ResponseBox}; +/// use memfaultd::http_server::{HttpHandler, HttpHandlerResult}; +/// +/// struct CounterHandler { +/// counter: u32, +/// }; +/// +/// impl CounterHandler { +/// fn handle_read(&self) -> Result { +/// Ok(Response::from_string("42").boxed()) +/// } +/// } +/// +/// impl HttpHandler for CounterHandler { +/// fn handle_request(&self, r: &mut Request) -> HttpHandlerResult { +/// if r.url() == "/count" { +/// self.handle_read().into() +/// } +/// else { +/// HttpHandlerResult::NotHandled +/// } +/// } +/// } +/// ``` +impl From> for HttpHandlerResult { + fn from(r: Result) -> Self { + match r { + Ok(response) => HttpHandlerResult::Response(response), + Err(e) => HttpHandlerResult::Error(e.to_string()), + } + } +} + +/// An HttpHandler can handle a request and send a response. +pub trait HttpHandler: Send { + /// Handle a request and prepares the response. + /// + /// ``` + /// # use eyre::Result; + /// use tiny_http::{ Request, Response, ResponseBox }; + /// use memfaultd::http_server::{HttpHandler, HttpHandlerResult}; + /// + /// struct CounterHandler { + /// counter: u32, + /// }; + /// + /// impl HttpHandler for CounterHandler { + /// fn handle_request(&self, r: &mut Request) -> HttpHandlerResult { + /// HttpHandlerResult::Response(Response::empty(200).boxed()) + /// } + /// } + /// + /// ``` + fn handle_request(&self, request: &mut Request) -> HttpHandlerResult; +} + +#[cfg(test)] +mod tests { + use tiny_http::ResponseBox; + + use super::HttpHandlerResult; + + impl HttpHandlerResult { + pub fn expect(self, m: &'static str) -> ResponseBox { + match self { + HttpHandlerResult::Response(response) => response, + HttpHandlerResult::Error(e) => panic!("{}: HttpHandlerResult::Error({})", m, e), + HttpHandlerResult::NotHandled => panic!("{}: HttpHandlerResult::Nothandled", m), + } + } + } +} diff --git a/memfaultd/src/http_server/mod.rs b/memfaultd/src/http_server/mod.rs new file mode 100644 index 0000000..d8a46d2 --- /dev/null +++ b/memfaultd/src/http_server/mod.rs @@ -0,0 +1,19 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Memfaultd HTTP server +//! Used on device for communication with other programs and `memfaultctl`. +//! +//! Typically binds to 127.0.0.1 and only available locally. +//! +mod handler; +mod request_bodies; +mod server; +mod utils; + +pub use handler::{HttpHandler, HttpHandlerResult}; +pub use server::HttpServer; + +pub use utils::ConvenientHeader; + +pub use request_bodies::SessionRequest; diff --git a/memfaultd/src/http_server/request_bodies.rs b/memfaultd/src/http_server/request_bodies.rs new file mode 100644 index 0000000..155f782 --- /dev/null +++ b/memfaultd/src/http_server/request_bodies.rs @@ -0,0 +1,28 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use serde::{Deserialize, Serialize}; + +use crate::metrics::{KeyedMetricReading, SessionName}; + +#[derive(Serialize, Deserialize)] +pub struct SessionRequest { + pub session_name: SessionName, + pub readings: Vec, +} + +impl SessionRequest { + pub fn new(session_name: SessionName, readings: Vec) -> Self { + Self { + session_name, + readings, + } + } + + pub fn new_without_readings(session_name: SessionName) -> Self { + Self { + session_name, + readings: vec![], + } + } +} diff --git a/memfaultd/src/http_server/server.rs b/memfaultd/src/http_server/server.rs new file mode 100644 index 0000000..5441b6e --- /dev/null +++ b/memfaultd/src/http_server/server.rs @@ -0,0 +1,73 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{net::SocketAddr, thread::spawn}; + +use eyre::{eyre, Result}; +use log::{debug, trace, warn}; +use tiny_http::{Request, Response, Server}; + +use crate::http_server::{HttpHandler, HttpHandlerResult}; + +/// A server that listens for collectd JSON pushes and stores them in memory. +pub struct HttpServer { + handlers: Option>>, +} + +impl HttpServer { + pub fn new(handlers: Vec>) -> Self { + HttpServer { + handlers: Some(handlers), + } + } + + pub fn start(&mut self, listening_address: SocketAddr) -> Result<()> { + let server = Server::http(listening_address).map_err(|e| { + eyre!("Error starting server: could not bind to {listening_address}: {e}") + })?; + + if let Some(handlers) = self.handlers.take() { + spawn(move || { + debug!("HTTP Server started on {listening_address}"); + + for request in server.incoming_requests() { + Self::handle_request(&handlers, request); + } + }); + Ok(()) + } else { + Err(eyre!("HTTP Server already started")) + } + } + + fn handle_request(handlers: &[Box], mut request: Request) { + trace!( + "HTTP request {:?} {:?}\n{:?}", + request.method(), + request.url(), + request.headers() + ); + + let method = request.method().to_owned(); + let url = request.url().to_owned(); + for handler in handlers.iter() { + match handler.handle_request(&mut request) { + HttpHandlerResult::Response(response) => { + if let Err(e) = request.respond(response) { + warn!("HTTP: Error sending response {} {}: {:?}", method, url, e); + } + return; + } + HttpHandlerResult::Error(e) => { + warn!("HTTP: Error processing request {} {}: {}", method, url, e); + let _r = request + .respond(Response::empty(500).with_data(e.as_bytes(), Some(e.len()))); + return; + } + HttpHandlerResult::NotHandled => { /* continue */ } + }; + } + debug!("HTTP[404] {} {}", method, url); + let _r = request.respond(Response::empty(404)); + } +} diff --git a/memfaultd/src/http_server/snapshots/memfaultd__http_server__response__tests__response_error.snap b/memfaultd/src/http_server/snapshots/memfaultd__http_server__response__tests__response_error.snap new file mode 100644 index 0000000..d2fa9eb --- /dev/null +++ b/memfaultd/src/http_server/snapshots/memfaultd__http_server__response__tests__response_error.snap @@ -0,0 +1,5 @@ +--- +source: memfaultd/src/http_server/response.rs +expression: "serde_json::to_string(&HttpResponse::Error(\"oops\".into())).unwrap()" +--- +{"status":"error","message":"oops"} diff --git a/memfaultd/src/http_server/snapshots/memfaultd__http_server__response__tests__response_not_found.snap b/memfaultd/src/http_server/snapshots/memfaultd__http_server__response__tests__response_not_found.snap new file mode 100644 index 0000000..abd36b3 --- /dev/null +++ b/memfaultd/src/http_server/snapshots/memfaultd__http_server__response__tests__response_not_found.snap @@ -0,0 +1,5 @@ +--- +source: memfaultd/src/http_server/response.rs +expression: "serde_json::to_string(&HttpResponse::NotFound).unwrap()" +--- +{"status":"not_found"} diff --git a/memfaultd/src/http_server/snapshots/memfaultd__http_server__response__tests__response_ok.snap b/memfaultd/src/http_server/snapshots/memfaultd__http_server__response__tests__response_ok.snap new file mode 100644 index 0000000..2b20e60 --- /dev/null +++ b/memfaultd/src/http_server/snapshots/memfaultd__http_server__response__tests__response_ok.snap @@ -0,0 +1,5 @@ +--- +source: memfaultd/src/http_server/response.rs +expression: "serde_json::to_string(&HttpResponse::Ok).unwrap()" +--- +{"status":"ok"} diff --git a/memfaultd/src/http_server/utils.rs b/memfaultd/src/http_server/utils.rs new file mode 100644 index 0000000..deffebb --- /dev/null +++ b/memfaultd/src/http_server/utils.rs @@ -0,0 +1,15 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::{eyre, Result}; +use tiny_http::Header; + +/// Wraps Header.from_bytes into something that returns a Result<> compatible with eyre::Result. +pub trait ConvenientHeader { + fn from_strings(name: &str, value: &str) -> Result
; +} +impl ConvenientHeader for Header { + fn from_strings(name: &str, value: &str) -> Result
{ + Header::from_bytes(name, value).map_err(|_e| eyre!("Invalid header ({}: {})", name, value)) + } +} diff --git a/memfaultd/src/lib.rs b/memfaultd/src/lib.rs new file mode 100644 index 0000000..f4d8bfb --- /dev/null +++ b/memfaultd/src/lib.rs @@ -0,0 +1,33 @@ +#![deny(clippy::print_stdout, clippy::print_stderr)] +// +// Copyright (c) Memfault, Inc. +// See License.txt for details + +pub mod cli; +#[cfg(feature = "collectd")] +mod collectd; +mod config; +#[cfg(feature = "coredump")] +mod coredump; +#[cfg(feature = "logging")] +mod fluent_bit; + +pub mod http_server; +#[cfg(feature = "logging")] +mod logs; +pub mod mar; +mod memfaultd; +pub mod metrics; +mod network; +mod reboot; +mod retriable_error; +mod service_manager; +#[cfg(feature = "swupdate")] +mod swupdate; +#[cfg(test)] +mod test_utils; +pub mod util; + +pub mod build_info { + include!(concat!(env!("OUT_DIR"), "/build_info.rs")); +} diff --git a/memfaultd/src/logs/completed_log.rs b/memfaultd/src/logs/completed_log.rs new file mode 100644 index 0000000..e3569eb --- /dev/null +++ b/memfaultd/src/logs/completed_log.rs @@ -0,0 +1,15 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::mar::CompressionAlgorithm; +use std::path::PathBuf; +use uuid::Uuid; + +/// CompletedLog represents a log that has been rotated and is ready to be moved into the MAR +/// staging area. +pub struct CompletedLog { + pub path: PathBuf, + pub cid: Uuid, + pub next_cid: Uuid, + pub compression: CompressionAlgorithm, +} diff --git a/memfaultd/src/logs/fluent_bit_adapter.rs b/memfaultd/src/logs/fluent_bit_adapter.rs new file mode 100644 index 0000000..c2249c1 --- /dev/null +++ b/memfaultd/src/logs/fluent_bit_adapter.rs @@ -0,0 +1,104 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! An adapter to connect FluentBit to our LogCollector. +//! +use std::sync::mpsc::Receiver; + +use log::warn; + +use crate::fluent_bit::FluentdMessage; +use crate::logs::log_entry::LogEntry; + +/// An iterator that can be used as a source of logs for LogCollector. +/// Will filter fluent-bit messages to keep only log messages and convert them +/// to a serde_json::Value ready to be written to disk. +pub struct FluentBitAdapter { + receiver: Receiver, + extra_fields: Vec, +} + +impl FluentBitAdapter { + pub fn new(receiver: Receiver, extra_fluent_bit_fields: &[String]) -> Self { + Self { + receiver, + extra_fields: extra_fluent_bit_fields.to_owned(), + } + } + + /// Convert a FluentdMessage into a serde_json::Value that we can log. + /// Returns None when this message should be filtered out. + fn convert_message(msg: FluentdMessage, extra_fields: &[String]) -> Option { + if !msg.1.contains_key("MESSAGE") { + // We are only interested in log messages. They will have a 'MESSAGE' key. + // Metrics do not have the MESSAGE key. + return None; + } + + let mut log_entry = LogEntry::from(msg); + log_entry.filter_fields(extra_fields); + + Some(log_entry) + } +} + +impl Iterator for FluentBitAdapter { + type Item = LogEntry; + /// Convert a FluentdMessage to a LogRecord for LogCollector. + /// Messages can be filtered out by returning None here. + fn next(&mut self) -> Option { + loop { + let msg_r = self.receiver.recv(); + match msg_r { + Ok(msg) => { + let value = FluentBitAdapter::convert_message(msg, &self.extra_fields); + match value { + v @ Some(_) => return v, + None => continue, + } + } + Err(e) => { + warn!("fluent-bit stopped receiving messages with error: {:?}", e); + return None; + } + } + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::mpsc::channel; + + use chrono::{DateTime, NaiveDateTime, Utc}; + use insta::{assert_json_snapshot, with_settings}; + + use super::*; + use crate::fluent_bit::{FluentdMessage, FluentdValue}; + + #[test] + fn test_fluent_bit_adapter() { + let (tx, rx) = channel(); + let mut adapter = FluentBitAdapter::new(rx, &[]); + + let mut map = HashMap::new(); + map.insert( + "MESSAGE".to_string(), + FluentdValue::String("test".to_string()), + ); + let msg = FluentdMessage(time(), map); + tx.send(msg).unwrap(); + + let log_entry = adapter.next().unwrap(); + + with_settings!({sort_maps => true}, { + assert_json_snapshot!(log_entry); + }); + } + + fn time() -> DateTime { + let naive = NaiveDateTime::from_timestamp_millis(1334250000000).unwrap(); + DateTime::::from_utc(naive, Utc) + } +} diff --git a/memfaultd/src/logs/headroom.rs b/memfaultd/src/logs/headroom.rs new file mode 100644 index 0000000..6302849 --- /dev/null +++ b/memfaultd/src/logs/headroom.rs @@ -0,0 +1,465 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::{ + logs::log_file::{LogFile, LogFileControl}, + util::disk_size::DiskSize, +}; +use eyre::Result; +use serde_json::Value; + +#[derive(Debug)] +enum Headroom { + Ok, + Shortage { + num_dropped_logs: usize, + has_rotated: bool, + }, +} + +pub trait HeadroomCheck { + fn check( + &mut self, + log_timestamp: Option<&Value>, + log_file_control: &mut impl LogFileControl, + ) -> Result; +} + +pub struct HeadroomLimiter { + state: Headroom, + /// Minimum amount of free space that must be kept available in the mount point in which + /// log_tmp_path resides. If there is not sufficient head room, logs will be dropped. + min_headroom: DiskSize, + get_available_space: Box Result + Send>, +} + +impl HeadroomLimiter { + pub fn new Result + Send + 'static>( + min_headroom: DiskSize, + get_available_space: S, + ) -> Self { + Self { + state: Headroom::Ok, + min_headroom, + get_available_space: Box::new(get_available_space), + } + } +} + +impl HeadroomCheck for HeadroomLimiter { + /// Checks whether there is enough headroom to continue writing logs. + /// If there is not enough headroom, this will flush the current log file and rotate at most + /// once when needed, until there is enough headroom again. When there's enough space again, it + /// will emit a log message mentioning the number of dropped logs. + /// Returns Ok(true) if there is enough headroom, Ok(false) if there is not enough headroom. + /// It only returns an error if there is an error writing the "Dropped N logs" message. + fn check( + &mut self, + log_timestamp: Option<&Value>, + log_file_control: &mut impl LogFileControl, + ) -> Result { + let available = (self.get_available_space)()?; + let has_headroom = available.exceeds(&self.min_headroom); + + self.state = match (has_headroom, &self.state) { + // Enter insufficient headroom state: + (false, Headroom::Ok) => { + // Best-effort warning log & flush. If this fails, just keep going. + let current_log = log_file_control.current_log()?; + let _ = current_log.write_log( + log_timestamp.cloned(), + match ( + available.bytes >= self.min_headroom.bytes, + available.inodes >= self.min_headroom.inodes, + ) { + (false, false) => "Low on disk space and inodes. Starting to drop logs...", + (false, true) => "Low on disk space. Starting to drop logs...", + (true, false) => "Low on inodes. Starting to drop logs...", + _ => unreachable!(), + }, + ); + let _ = current_log.flush(); + Headroom::Shortage { + has_rotated: log_file_control.rotate_if_needed().unwrap_or(false), + num_dropped_logs: 1, + } + } + // Already in insufficient headroom state: + ( + false, + Headroom::Shortage { + has_rotated, + num_dropped_logs, + }, + ) => { + // Rotate logs once only: + let num_dropped_logs = *num_dropped_logs + 1; + let has_rotated = + *has_rotated || log_file_control.rotate_if_needed().unwrap_or(false); + Headroom::Shortage { + num_dropped_logs, + has_rotated, + } + } + // Exit insufficient headroom state: + ( + true, + Headroom::Shortage { + num_dropped_logs, .. + }, + ) => { + let current_log = log_file_control.current_log()?; + current_log.write_log( + log_timestamp.cloned(), + format!( + "Recovered from low disk space. Dropped {} logs.", + num_dropped_logs + ), + )?; + Headroom::Ok + } + // Already in headroom OK state and staying in this state: + (true, Headroom::Ok) => Headroom::Ok, + }; + Ok(has_headroom) + } +} + +#[cfg(test)] +mod tests { + use crate::util::disk_size::DiskSize; + use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }; + + use super::*; + use eyre::eyre; + use rstest::{fixture, rstest}; + + #[rstest] + fn returns_true_if_headroom_ok_and_stays_ok(mut fixture: Fixture) { + let log_timestamp = Value::from(12345); + let mut log_file_control = FakeLogFileControl::default(); + fixture.set_available_space(MIN_HEADROOM); + + // Enough headroom: check() returns true and no calls to log_file_control are made: + assert!(fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap()); + assert_eq!(0, log_file_control.logs_written.len()); + assert_eq!(0, log_file_control.flush_count); + assert_eq!(0, log_file_control.rotation_count); + } + + #[rstest] + fn log_upon_enter_and_exit_headroom_space_shortage(mut fixture: Fixture) { + let log_timestamp = Value::from(12345); + let mut log_file_control = FakeLogFileControl::default(); + + // Enter headroom shortage: check() returns false: + fixture.set_available_space(MIN_HEADROOM - 1); + assert!(!fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap()); + + // Check that the warning log was written: + assert_eq!(1, log_file_control.logs_written.len()); + assert!(log_file_control.logs_written[0] + .contains("Low on disk space. Starting to drop logs...")); + // Check that the log was flushed: + assert_eq!(1, log_file_control.flush_count); + + // Still not enough headroom: check() returns false: + assert!(!fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap()); + + // Recover from headroom shortage: check() returns true again: + fixture.set_available_space(MIN_HEADROOM); + assert!(fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap()); + + // Check that the "recovered" log was written: + assert_eq!(2, log_file_control.logs_written.len()); + assert!(log_file_control.logs_written[1] + .contains("Recovered from low disk space. Dropped 2 logs.")); + } + + #[rstest] + fn log_upon_enter_and_exit_headroom_node_shortage(mut fixture: Fixture) { + let log_timestamp = Value::from(12345); + let mut log_file_control = FakeLogFileControl::default(); + + // Enter headroom shortage: check() returns false: + fixture.set_available_inodes(MIN_INODES - 1); + assert!(!fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap()); + + // Check that the warning log was written: + assert_eq!(1, log_file_control.logs_written.len()); + assert!( + log_file_control.logs_written[0].contains("Low on inodes. Starting to drop logs...") + ); + // Check that the log was flushed: + assert_eq!(1, log_file_control.flush_count); + + // Still not enough headroom: check() returns false: + assert!(!fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap()); + + // Recover from headroom shortage: check() returns true again: + fixture.set_available_inodes(MIN_INODES); + assert!(fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap()); + + // Check that the "recovered" log was written: + assert_eq!(2, log_file_control.logs_written.len()); + assert!(log_file_control.logs_written[1] + .contains("Recovered from low disk space. Dropped 2 logs.")); + } + + #[rstest] + fn rotate_once_only_entering_headroom_shortage(mut fixture: Fixture) { + let log_timestamp = Value::from(12345); + let mut log_file_control = FakeLogFileControl { + // Make log_file_control.rotate_if_needed() return Ok(true): + rotate_return: Some(true), + ..Default::default() + }; + + // Enter headroom shortage: + fixture.set_available_space(MIN_HEADROOM - 1); + fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap(); + assert_eq!(log_file_control.rotation_count, 1); + + // Check again. Rotation should not be attempted again: + fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap(); + assert_eq!(log_file_control.rotation_count, 1); + } + + #[rstest] + fn rotate_once_only_during_headroom_shortage(mut fixture: Fixture) { + let log_timestamp = Value::from(12345); + let mut log_file_control = FakeLogFileControl::default(); + + // Enter headroom shortage: + fixture.set_available_space(MIN_HEADROOM - 1); + fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap(); + assert_eq!(log_file_control.rotation_count, 0); + + // Make log_file_control.rotate_if_needed() return Ok(true): + log_file_control.rotate_return = Some(true); + + // Check again. Rotation should be attempted again: + fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap(); + assert_eq!(log_file_control.rotation_count, 1); + + // Check again. Rotation should not be attempted again: + fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap(); + assert_eq!(log_file_control.rotation_count, 1); + } + + #[rstest] + fn retry_rotate_after_failure(mut fixture: Fixture) { + let log_timestamp = Value::from(12345); + let mut log_file_control = FakeLogFileControl { + // Make log_file_control.rotate_if_needed() return Err(...): + rotate_return: None, + ..Default::default() + }; + + // Enter headroom shortage: + fixture.set_available_space(MIN_HEADROOM - 1); + fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap(); + assert_eq!(log_file_control.rotation_count, 0); + + // Check again. Rotation should be attempted again: + // Make log_file_control.rotate_if_needed() return Ok(true): + log_file_control.rotate_return = Some(true); + fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap(); + assert_eq!(log_file_control.rotation_count, 1); + } + + #[rstest] + fn write_error_of_initial_warning_message_is_ignored(mut fixture: Fixture) { + let log_timestamp = Value::from(12345); + let mut log_file_control = FakeLogFileControl::default(); + + fixture.set_available_space(MIN_HEADROOM - 1); + log_file_control.write_should_fail = true; + assert!(fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .is_ok()); + } + + #[rstest] + fn write_error_of_recovery_log_message_is_bubbled_up(mut fixture: Fixture) { + let log_timestamp = Value::from(12345); + let mut log_file_control = FakeLogFileControl::default(); + + fixture.set_available_space(MIN_HEADROOM - 1); + fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .unwrap(); + fixture.set_available_space(MIN_HEADROOM); + log_file_control.write_should_fail = true; + assert!(fixture + .limiter + .check(Some(&log_timestamp), &mut log_file_control) + .is_err()); + } + + struct FakeLogFileControl { + logs_written: Vec, + write_should_fail: bool, + flush_count: usize, + flush_should_fail: bool, + /// This controls the result of rotate_if_needed(). + /// Some(...) is mapped to Ok(...) and None is mapped to Err(...). + rotate_return: Option, + /// Number of times actually rotated (rotate_if_needed() calls while rotate_return was Some(true)): + rotation_count: usize, + } + + impl Default for FakeLogFileControl { + fn default() -> Self { + FakeLogFileControl { + logs_written: Vec::new(), + flush_count: 0, + flush_should_fail: false, + write_should_fail: false, + rotate_return: Some(false), + rotation_count: 0, + } + } + } + + impl LogFile for FakeLogFileControl { + fn write_json_line(&mut self, json: Value) -> Result<()> { + if self.write_should_fail { + Err(eyre!("Write failed")) + } else { + self.logs_written.push(serde_json::to_string(&json)?); + Ok(()) + } + } + + fn flush(&mut self) -> Result<()> { + self.flush_count += 1; + if self.flush_should_fail { + Err(eyre!("Flush failed")) + } else { + Ok(()) + } + } + } + + impl LogFileControl for FakeLogFileControl { + fn rotate_if_needed(&mut self) -> Result { + match self.rotate_return { + Some(rv) => { + if rv { + self.rotation_count += 1; + } + Ok(rv) + } + None => Err(eyre!("Rotate failed")), + } + } + + fn rotate_unless_empty(&mut self) -> Result<()> { + unimplemented!(); + } + + fn current_log(&mut self) -> Result<&mut FakeLogFileControl> { + Ok(self) + } + + fn close(self) -> Result<()> { + Ok(()) + } + } + + struct Fixture { + available_space: Arc, + available_inodes: Arc, + limiter: HeadroomLimiter, + } + + impl Fixture { + fn set_available_space(&mut self, available_space: u64) { + self.available_space + .store(available_space, Ordering::Relaxed) + } + fn set_available_inodes(&mut self, available_inodes: u64) { + self.available_inodes + .store(available_inodes, Ordering::Relaxed) + } + } + + const MIN_HEADROOM: u64 = 1024; + const MIN_INODES: u64 = 10; + const INITIAL_AVAILABLE_SPACE: u64 = 1024 * 1024; + const INITIAL_AVAILABLE_INODES: u64 = 100; + + #[fixture] + fn fixture() -> Fixture { + let available_space = Arc::new(AtomicU64::new(INITIAL_AVAILABLE_SPACE)); + let available_inodes = Arc::new(AtomicU64::new(INITIAL_AVAILABLE_INODES)); + + let space = available_space.clone(); + let inodes = available_inodes.clone(); + + Fixture { + limiter: HeadroomLimiter::new( + DiskSize { + bytes: MIN_HEADROOM, + inodes: MIN_INODES, + }, + move || { + Ok(DiskSize { + bytes: space.load(Ordering::Relaxed), + inodes: inodes.load(Ordering::Relaxed), + }) + }, + ), + available_inodes, + available_space, + } + } +} diff --git a/memfaultd/src/logs/journald_parser.rs b/memfaultd/src/logs/journald_parser.rs new file mode 100644 index 0000000..2332a58 --- /dev/null +++ b/memfaultd/src/logs/journald_parser.rs @@ -0,0 +1,393 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use chrono::{DateTime, NaiveDateTime, Utc}; +use eyre::{eyre, Result}; +use libc::free; +use nix::poll::{poll, PollFd}; +use serde::Serialize; +use std::ffi::CString; +use std::fs::read_to_string; +use std::{collections::HashMap, path::PathBuf}; +use std::{ffi::c_char, mem::MaybeUninit}; + +use log::{debug, warn}; +use memfaultc_sys::systemd::{ + sd_journal, sd_journal_add_match, sd_journal_enumerate_data, sd_journal_get_cursor, + sd_journal_get_fd, sd_journal_get_realtime_usec, sd_journal_next, sd_journal_open, + sd_journal_process, sd_journal_seek_cursor, +}; + +use super::log_entry::{LogEntry, LogValue}; +use crate::util::system::read_system_boot_id; +/// A trait for reading journal entries from the systemd journal. +/// +/// This trait is used to abstract the raw journal entry reading logic from the rest of the codebase. +/// This allows for easier testing and mocking of the journal reading logic. +#[cfg_attr(test, mockall::automock)] +pub trait JournalRaw { + /// Check if the next journal entry is available. + /// + /// Returns `Ok(true)` if the next journal entry is available, `Ok(false)` if there are no more entries, + /// and an error if there was an issue reading the journal. + fn next_entry_available(&mut self) -> Result; + + /// Get the raw field data of the current journal entry. + /// + /// Returns the raw string representing the key-value pairs of the journal entry, or `None` if there are no more entries. + /// Returns an error if there was an issue reading the journal. + fn get_entry_field_data(&mut self) -> Result>; + + /// Waits for the next journal entry to be available. + /// + /// This method should block until the next journal entry is available. + fn wait_for_entry(&mut self) -> Result<()>; +} + +/// Raw journal entry data. +/// +/// This struct represents the raw data of a journal entry. It can be converted into a `JournalEntry` struct, +/// which contains the parsed key-value pairs of the journal entry. +#[derive(Debug)] +pub struct JournalEntryRaw { + pub ts: DateTime, + pub fields: Vec, +} + +impl JournalEntryRaw { + pub fn new(fields: Vec, ts: DateTime) -> Self { + Self { ts, fields } + } +} + +/// An implementation of the `JournalRaw` trait that reads journal entries from the systemd journal. +/// +/// This struct is used to read journal entries from the systemd journal. It relies on ffi calls into +/// libsystemd. +pub struct JournalRawImpl { + journal: *mut sd_journal, + wait_fd: PollFd, + cursor_file: PathBuf, +} + +impl JournalRawImpl { + /// Timeout journal polling after 1 minute. + const POLL_TIMEOUT_MS: i32 = 1000 * 60; + const JOURNAL_CURSOR_FILE: &str = "JOURNALD_CURSOR"; + + pub fn new(tmp_path: PathBuf) -> Self { + let mut journal = std::ptr::null_mut(); + let cursor_file = tmp_path.join(Self::JOURNAL_CURSOR_FILE); + let cursor_string = read_to_string(&cursor_file).ok(); + + unsafe { + sd_journal_open(&mut journal, 0); + } + + let fd = unsafe { sd_journal_get_fd(journal) }; + let wait_fd = PollFd::new(fd, nix::poll::PollFlags::POLLIN); + + if let Some(cursor) = cursor_string { + let cursor = cursor.trim(); + let ret = unsafe { sd_journal_seek_cursor(journal, cursor.as_ptr() as *const c_char) }; + if ret < 0 { + warn!("Failed to seek journal to cursor: {}", ret); + } + } else if let Err(e) = Self::seek_to_current_boot_start(journal) { + warn!("Couldn't seek journal to start of current boot: {}", e); + } + + Self { + journal, + wait_fd, + cursor_file, + } + } + + /// Seeks the journal to the start of the current boot's logs + /// + /// Returns OK if the journal is now set up to return the first log + /// from the current boot + /// Returns an error if the function could not confirm the next + /// entry will be the start of the current boot + fn seek_to_current_boot_start(journal: *mut sd_journal) -> Result<()> { + let boot_id = read_system_boot_id()?; + let boot_id_match = CString::new(format!("_BOOT_ID={}", boot_id.as_simple()))?; + + let ret = + unsafe { sd_journal_add_match(journal, boot_id_match.as_ptr() as *const c_char, 0) }; + match ret { + ret if ret < 0 => Err(eyre!( + "Failed to add match on current boot ID to journal: {}", + ret + )), + 0 => Ok(()), + _ => Ok(()), + } + } + + /// Save the current journal cursor to a file. + /// + /// This method saves the current journal cursor to a file so that the journal can be resumed from the + /// same point after a restart. + fn save_cursor(&self) { + let mut cursor: MaybeUninit<*const c_char> = MaybeUninit::uninit(); + let ret = unsafe { sd_journal_get_cursor(self.journal, cursor.as_mut_ptr()) }; + if ret < 0 { + warn!("Failed to get journal cursor: {}", ret); + } else { + let cursor = unsafe { cursor.assume_init() }; + let cursor_cstr = unsafe { std::ffi::CStr::from_ptr(cursor) }; + let cursor_str = cursor_cstr.to_str().unwrap_or_default(); + + let write_result = std::fs::write(&self.cursor_file, cursor_str); + unsafe { + free(cursor as *mut libc::c_void); + } + match write_result { + Ok(_) => (), + Err(e) => warn!( + "Failed to write journal cursor to {:?}: {}", + self.cursor_file, e + ), + } + } + } + + pub fn get_timestamp(&self) -> Result> { + let mut timestamp = 0u64; + let ret = unsafe { sd_journal_get_realtime_usec(self.journal, &mut timestamp) }; + if ret < 0 { + return Err(eyre!("Failed to get journal entry timestamp: {}", ret)); + } + + let datetime = NaiveDateTime::from_timestamp_micros(timestamp as i64) + .ok_or_else(|| eyre!("Failed to convert journal timestamp to DateTime"))?; + + Ok(DateTime::::from_utc(datetime, Utc)) + } +} + +impl Drop for JournalRawImpl { + fn drop(&mut self) { + self.save_cursor(); + unsafe { + libc::free(self.journal as *mut libc::c_void); + } + } +} + +impl JournalRaw for JournalRawImpl { + fn next_entry_available(&mut self) -> Result { + let ret = unsafe { sd_journal_next(self.journal) }; + match ret { + ret if ret < 0 => Err(eyre!("Failed to get next journal entry: {}", ret)), + 0 => Ok(false), + _ => Ok(true), + } + } + + fn get_entry_field_data(&mut self) -> Result> { + let mut data = MaybeUninit::uninit(); + let mut data_len = MaybeUninit::uninit(); + let mut fields = Vec::new(); + + let timestamp = self.get_timestamp().map_or_else( + |e| { + debug!( + "Failed to get journal entry timestamp, falling back to ingestion time: {}", + e + ); + Utc::now() + }, + |t| t, + ); + + let mut enum_ret = unsafe { + sd_journal_enumerate_data(self.journal, data.as_mut_ptr(), data_len.as_mut_ptr()) + }; + + while enum_ret > 0 { + let bytes = + unsafe { std::slice::from_raw_parts(data.assume_init(), data_len.assume_init()) }; + + let kv_string = String::from_utf8_lossy(bytes).to_string(); + fields.push(kv_string); + + enum_ret = unsafe { + sd_journal_enumerate_data(self.journal, data.as_mut_ptr(), data_len.as_mut_ptr()) + }; + } + + if enum_ret < 0 { + Err(eyre!("Failed to read journal entry data: {}", enum_ret)) + } else { + Ok(Some(JournalEntryRaw::new(fields, timestamp))) + } + } + + fn wait_for_entry(&mut self) -> Result<()> { + let mut fds = [self.wait_fd]; + let ret = poll(&mut fds, Self::POLL_TIMEOUT_MS)?; + if ret < 0 { + return Err(eyre!("Failed to poll for journal entry: {}", ret)); + } + + // This call clears the queue status of the poll fd + let ret = unsafe { sd_journal_process(self.journal) }; + if ret < 0 { + return Err(eyre!("Failed to process journal entry: {}", ret)); + } + + Ok(()) + } +} + +/// A fully parsed journal entry. +/// +/// This struct represents a fully parsed journal entry, with all key-value pairs parsed into a HashMap. +#[derive(Serialize, Debug)] +pub struct JournalEntry { + pub ts: DateTime, + pub fields: HashMap, +} + +impl From for JournalEntry { + fn from(raw: JournalEntryRaw) -> Self { + let fields = raw + .fields + .into_iter() + .fold(HashMap::new(), |mut acc, field| { + let kv: Vec<&str> = field.splitn(2, '=').collect(); + if kv.len() == 2 { + acc.insert(kv[0].to_string(), kv[1].to_string()); + } + acc + }); + + Self { ts: raw.ts, fields } + } +} + +impl From for LogEntry { + fn from(entry: JournalEntry) -> Self { + let ts = entry.ts; + let data = entry + .fields + .into_iter() + .fold(HashMap::new(), |mut acc, (k, v)| { + acc.insert(k, LogValue::String(v)); + acc + }); + + LogEntry { ts, data } + } +} + +/// A wrapper around the `JournalRaw` trait that provides an iterator over journal entries. +/// +/// This struct provides an iterator over journal entries, abstracting the raw journal reading logic. +pub struct Journal { + journal: J, +} + +impl Journal { + pub fn new(journal: J) -> Self { + Self { journal } + } + + /// Get the next journal entry. + /// + /// This function will return a `JournalEntry` if the next entry is available, or `None` if there are no more entries. + /// Returns an error if there was an issue reading the journal. + fn next_entry(&mut self) -> Result> { + match self.journal.next_entry_available()? { + false => Ok(None), + true => { + let raw = self.journal.get_entry_field_data()?; + Ok(raw.map(|raw| raw.into())) + } + } + } + + /// Get an iterator over all available journal entries. + pub fn iter(&mut self) -> impl Iterator + '_ { + std::iter::from_fn(move || match self.next_entry() { + Ok(entry) => entry, + Err(e) => { + warn!("Failed to get next journal entry: {}", e); + None + } + }) + } + + pub fn wait_for_entry(&mut self) -> Result<()> { + self.journal.wait_for_entry() + } +} + +#[cfg(test)] +mod test { + use super::*; + + use insta::{assert_json_snapshot, with_settings}; + use mockall::Sequence; + + #[test] + fn test_from_raw_journal_entry() { + let raw_entry = raw_journal_entry(); + let entry = JournalEntry::from(raw_entry); + + with_settings!({sort_maps => true}, { + assert_json_snapshot!(entry); + }); + } + + #[test] + fn test_journal_happy_path() { + let mut raw_journal = MockJournalRaw::new(); + let mut seq = Sequence::new(); + + raw_journal + .expect_next_entry_available() + .times(1) + .in_sequence(&mut seq) + .returning(|| Ok(true)); + + raw_journal + .expect_get_entry_field_data() + .times(1) + .in_sequence(&mut seq) + .returning(|| Ok(Some(raw_journal_entry()))); + + raw_journal + .expect_next_entry_available() + .times(1) + .in_sequence(&mut seq) + .returning(|| Ok(false)); + + let mut journal = Journal::new(raw_journal); + let mut journal_iter = journal.iter(); + let entry = journal_iter.next(); + assert!(entry.is_some()); + with_settings!({sort_maps => true}, { + assert_json_snapshot!(entry.unwrap()); + }); + let entry = journal_iter.next(); + assert!(entry.is_none()); + } + + fn timestamp() -> DateTime { + let timestamp = NaiveDateTime::from_timestamp_millis(1713462571).unwrap(); + DateTime::::from_utc(timestamp, Utc) + } + + fn raw_journal_entry() -> JournalEntryRaw { + let fields = [ + "_SYSTEMD_UNIT=user@1000.service", + "MESSAGE=audit: type=1400 audit(1713462571.968:7508): apparmor=\"DENIED\" operation=\"open\" class=\"file\" profile=\"snap.firefox.firefox\" name=\"/etc/fstab\" pid=10122 comm=\"firefox\" requested_mask=\"r\" denied_mask=\"r\" fsuid=1000 ouid=0", + ]; + + JournalEntryRaw::new(fields.iter().map(|s| s.to_string()).collect(), timestamp()) + } +} diff --git a/memfaultd/src/logs/journald_provider.rs b/memfaultd/src/logs/journald_provider.rs new file mode 100644 index 0000000..55b8713 --- /dev/null +++ b/memfaultd/src/logs/journald_provider.rs @@ -0,0 +1,173 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::logs::journald_parser::{Journal, JournalRaw, JournalRawImpl}; + +use eyre::{eyre, Result}; +use log::{error, warn}; + +use std::thread::spawn; +use std::{ + path::PathBuf, + sync::mpsc::{sync_channel, Receiver, SyncSender}, +}; + +use super::log_entry::LogEntry; + +const ENTRY_CHANNEL_SIZE: usize = 1024; + +/// A log provider that reads log entries from Journald and sends them to a receiver. +pub struct JournaldLogProvider { + journal: Journal, + entry_sender: SyncSender, +} + +impl JournaldLogProvider { + pub fn new(journal: J, entry_sender: SyncSender) -> Self { + Self { + journal: Journal::new(journal), + entry_sender, + } + } + + fn run_once(&mut self) -> Result<()> { + for entry in self.journal.iter() { + let mut log_entry = LogEntry::from(entry); + // TODO: Add support for filtering additional fields + log_entry.filter_fields(&[]); + + if let Err(e) = self.entry_sender.send(log_entry) { + return Err(eyre!("Journald channel dropped: {}", e)); + } + } + Ok(()) + } + + pub fn start(&mut self) -> Result<()> { + loop { + // Block until another entry is available + self.journal.wait_for_entry()?; + + self.run_once()?; + } + } +} + +/// A receiver for log entries from Journald. +pub struct JournaldLogReceiver { + entry_receiver: Receiver, +} + +impl JournaldLogReceiver { + pub fn new(entry_receiver: Receiver) -> Self { + Self { entry_receiver } + } +} + +impl Iterator for JournaldLogReceiver { + type Item = LogEntry; + + fn next(&mut self) -> Option { + match self.entry_receiver.recv() { + Ok(v) => Some(v), + Err(e) => { + warn!("Failed to receive entry: {}", e); + None + } + } + } +} + +/// Start a Journald log provider and return a receiver for the log entries. +/// +/// This function will start a new thread that reads log entries from Journald and sends them to the +/// returned receiver. It takes in the temporary storage path to use as the location of storing the +/// cursor file. +pub fn start_journald_provider(tmp_path: PathBuf) -> JournaldLogReceiver { + let (entry_sender, entry_receiver) = sync_channel(ENTRY_CHANNEL_SIZE); + + spawn(move || { + let journal_raw = JournalRawImpl::new(tmp_path); + let mut provider = JournaldLogProvider::new(journal_raw, entry_sender); + if let Err(e) = provider.start() { + error!("Journald provider failed: {}", e); + } + }); + + JournaldLogReceiver::new(entry_receiver) +} + +#[cfg(test)] +mod test { + use chrono::{DateTime, NaiveDateTime, Utc}; + use insta::{assert_json_snapshot, with_settings}; + use mockall::Sequence; + + use super::*; + + use crate::logs::journald_parser::{JournalEntryRaw, MockJournalRaw}; + + #[test] + fn test_happy_path() { + let mut journal_raw = MockJournalRaw::new(); + let mut seq = Sequence::new(); + + let (sender, receiver) = sync_channel(ENTRY_CHANNEL_SIZE); + + journal_raw + .expect_next_entry_available() + .times(1) + .in_sequence(&mut seq) + .returning(|| Ok(true)); + journal_raw + .expect_get_entry_field_data() + .returning(|| Ok(Some(raw_journal_entry()))); + journal_raw + .expect_next_entry_available() + .times(1) + .in_sequence(&mut seq) + .returning(|| Ok(false)); + + let mut provider = JournaldLogProvider::new(journal_raw, sender); + + assert!(provider.run_once().is_ok()); + let entry = receiver.try_recv().unwrap(); + with_settings!({sort_maps => true}, { + assert_json_snapshot!(entry); + }); + } + + #[test] + fn test_channel_dropped() { + let mut journal_raw = MockJournalRaw::new(); + let mut seq = Sequence::new(); + + let (sender, receiver) = sync_channel(1); + drop(receiver); + + journal_raw + .expect_next_entry_available() + .times(1) + .in_sequence(&mut seq) + .returning(|| Ok(true)); + journal_raw + .expect_get_entry_field_data() + .returning(|| Ok(Some(raw_journal_entry()))); + + let mut provider = JournaldLogProvider::new(journal_raw, sender); + + assert!(provider.run_once().is_err()); + } + + fn raw_journal_entry() -> JournalEntryRaw { + let fields = [ + "_SYSTEMD_UNIT=user@1000.service", + "MESSAGE=audit: type=1400 audit(1713462571.968:7508): apparmor=\"DENIED\" operation=\"open\" class=\"file\" profile=\"snap.firefox.firefox\" name=\"/etc/fstab\" pid=10122 comm=\"firefox\" requested_mask=\"r\" denied_mask=\"r\" fsuid=1000 ouid=0", + ]; + + let timestamp = NaiveDateTime::from_timestamp_millis(1337).unwrap(); + let timestamp = DateTime::::from_utc(timestamp, Utc); + + JournalEntryRaw::new(fields.iter().map(|s| s.to_string()).collect(), timestamp) + } +} diff --git a/memfaultd/src/logs/log_collector.rs b/memfaultd/src/logs/log_collector.rs new file mode 100644 index 0000000..b896353 --- /dev/null +++ b/memfaultd/src/logs/log_collector.rs @@ -0,0 +1,755 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Collect logs into log files and save them as MAR entries. +//! +use std::io::Cursor; +use std::num::NonZeroU32; +use std::sync::Arc; +use std::time::Duration; +use std::{fs, thread}; +use std::{path::PathBuf, sync::Mutex}; + +use eyre::{eyre, Context, Result}; +use flate2::Compression; +use log::{error, trace, warn}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tiny_http::{Header, Method, Request, Response, ResponseBox, StatusCode}; + +use crate::util::rate_limiter::RateLimiter; +use crate::{config::Config, metrics::MetricReportManager}; +use crate::{config::LogToMetricRule, logs::completed_log::CompletedLog}; +use crate::{config::StorageConfig, http_server::ConvenientHeader}; +use crate::{ + http_server::HttpHandler, + logs::log_file::{LogFile, LogFileControl, LogFileControlImpl}, +}; +use crate::{http_server::HttpHandlerResult, logs::recovery::recover_old_logs}; +use crate::{logs::headroom::HeadroomCheck, util::circular_queue::CircularQueue}; + +pub const CRASH_LOGS_URL: &str = "/api/v1/crash-logs"; + +use super::log_entry::LogEntry; +#[cfg(feature = "log-to-metrics")] +use super::log_to_metrics::LogToMetrics; + +pub struct LogCollector { + inner: Arc>>>, +} + +impl LogCollector { + /// This value is used to clamp the number of lines captured in a coredump. + /// + /// This is done to prevent the coredump from becoming too large. The value was chosen + /// arbitrarily to be large enough to capture a reasonable amount of logs, but small enough + /// to prevent the coredump from becoming too large. The current default is 100 lines. + const MAX_IN_MEMORY_LINES: usize = 500; + + /// Create a new log collector and open a new log file for writing. + /// The on_log_completion callback will be called when a log file is completed. + /// This callback must move (or delete) the log file! + pub fn open Result<()> + Send + 'static>( + log_config: LogCollectorConfig, + mut on_log_completion: R, + headroom_limiter: H, + #[cfg_attr(not(feature = "log-to-metrics"), allow(unused_variables))] + heartbeat_manager: Arc>, + ) -> Result { + fs::create_dir_all(&log_config.log_tmp_path).wrap_err_with(|| { + format!( + "Unable to create directory to store in-progress logs: {}", + log_config.log_tmp_path.display() + ) + })?; + + // Collect any leftover logfiles in the tmp folder + let next_cid = recover_old_logs(&log_config.log_tmp_path, &mut on_log_completion)?; + + let in_memory_lines = if log_config.in_memory_lines > Self::MAX_IN_MEMORY_LINES { + warn!( + "Too many lines captured in coredump ({}), clamping to {}", + log_config.in_memory_lines, + Self::MAX_IN_MEMORY_LINES + ); + Self::MAX_IN_MEMORY_LINES + } else { + log_config.in_memory_lines + }; + Ok(Self { + inner: Arc::new(Mutex::new(Some(Inner { + log_file_control: LogFileControlImpl::open( + log_config.log_tmp_path, + next_cid, + log_config.log_max_size, + log_config.log_max_duration, + log_config.log_compression_level, + on_log_completion, + )?, + rate_limiter: RateLimiter::new(log_config.max_lines_per_minute), + headroom_limiter, + #[cfg(feature = "log-to-metrics")] + log_to_metrics: LogToMetrics::new( + log_config.log_to_metrics_rules, + heartbeat_manager, + ), + log_queue: CircularQueue::new(in_memory_lines), + storage_config: log_config.storage_config, + }))), + }) + } + + /// Spawn a thread to read log records from receiver. + pub fn spawn_collect_from + Send + 'static>(&self, source: T) { + // Clone the atomic reference counting "pointer" (not the inner struct itself) + let c = self.inner.clone(); + + thread::spawn(move || { + for line in source { + match c.lock() { + Ok(mut inner_opt) => { + match &mut *inner_opt { + Some(inner) => { + let log_val = match serde_json::to_value(line) { + Ok(val) => val, + Err(e) => { + warn!("Error converting log to json: {}", e); + continue; + } + }; + if let Err(e) = inner.process_log_record(log_val) { + warn!("Error writing log: {:?}", e); + } + } + // log_collector has shutdown. exit the thread cleanly. + None => return, + } + } + Err(e) => { + // This should never happen but we are unable to recover from this so bail out. + error!("Log collector got into an unrecoverable state: {}", e); + std::process::exit(-1); + } + } + } + trace!("Log collection thread shutting down - Channel closed"); + }); + } + + /// Get a handler for the /api/v1/crash-logs endpoint + pub fn crash_log_handler(&self) -> CrashLogHandler { + CrashLogHandler::new(self.inner.clone()) + } + + /// Force the log_collector to close the current log and generate a MAR entry. + pub fn flush_logs(&mut self) -> Result<()> { + self.with_mut_inner(|inner| inner.log_file_control.rotate_unless_empty().map(|_| ())) + } + + /// Rotate the logs if needed + pub fn rotate_if_needed(&mut self) -> Result { + self.with_mut_inner(|inner| inner.rotate_if_needed()) + } + + /// Try to get the inner log_collector or return an error + fn with_mut_inner) -> Result>(&mut self, fun: F) -> Result { + let mut inner_opt = self + .inner + .lock() + // This should never happen so we choose to panic in this case. + .expect("Fatal: log_collector mutex is poisoned."); + + match &mut *inner_opt { + Some(inner) => fun(inner), + None => Err(eyre!("Log collector has already shutdown.")), + } + } + + /// Close and dispose of the inner log collector. + /// This is not public because it does not consume self (to be compatible with drop()). + fn close_internal(&mut self) -> Result<()> { + match self.inner.lock() { + Ok(mut inner_opt) => { + match (*inner_opt).take() { + Some(inner) => inner.log_file_control.close(), + None => { + // Already closed. + Ok(()) + } + } + } + Err(_) => { + // Should never happen. + panic!("Log collector is poisoned.") + } + } + } +} + +impl Drop for LogCollector { + fn drop(&mut self) { + if let Err(e) = self.close_internal() { + warn!("Error closing log collector: {}", e); + } + } +} + +/// The log collector keeps one Inner struct behind a Arc> so it can be +/// shared by multiple threads. +struct Inner { + // We use an Option here because we have no typed-guarantee that every + // log message will include a `ts` key. + rate_limiter: RateLimiter>, + log_file_control: LogFileControlImpl, + headroom_limiter: H, + #[cfg(feature = "log-to-metrics")] + log_to_metrics: LogToMetrics, + log_queue: CircularQueue, + storage_config: StorageConfig, +} + +impl Inner { + // Process one log record - To call this, the caller must have acquired a + // mutex on the Inner object. + // Be careful to not try to acquire other mutexes here to avoid a + // dead-lock. Everything we need should be in Inner. + fn process_log_record(&mut self, log: Value) -> Result<()> { + let log_timestamp = log.get("ts"); + + #[cfg(feature = "log-to-metrics")] + if let Err(e) = self.log_to_metrics.process(&log) { + warn!("Error processing log to metrics: {:?}", e); + } + + if !self + .headroom_limiter + .check(log_timestamp, &mut self.log_file_control)? + { + return Ok(()); + } + self.log_queue.push(log.clone()); + + // Return early and do not write a log message to file if not persisting + if !self.should_persist() { + return Ok(()); + } + + // Rotate before writing (in case log file is now too old) + self.log_file_control.rotate_if_needed()?; + + let logfile = self.log_file_control.current_log()?; + self.rate_limiter + .run_within_limits(log_timestamp.cloned(), |rate_limited_calls| { + // Print a message if some previous calls were rate limited. + if let Some(limited) = rate_limited_calls { + logfile.write_log( + limited.latest_call, + format!("Memfaultd rate limited {} messages.", limited.count), + )?; + } + logfile.write_json_line(log)?; + Ok(()) + })?; + + // Rotate after writing (in case log file is now too large) + self.log_file_control.rotate_if_needed()?; + Ok(()) + } + + fn should_persist(&self) -> bool { + matches!(self.storage_config, StorageConfig::Persist) + } + + fn rotate_if_needed(&mut self) -> Result { + self.log_file_control.rotate_if_needed() + } + + pub fn get_log_queue(&mut self) -> Result> { + let logs = self.log_queue.iter().map(|v| v.to_string()).collect(); + + Ok(logs) + } +} + +pub struct LogCollectorConfig { + /// Folder where to store logfiles while they are being written + pub log_tmp_path: PathBuf, + + /// Files will be rotated when they reach this size (so they may be slightly larger) + log_max_size: usize, + + /// MAR entry will be rotated when they get this old. + log_max_duration: Duration, + + /// Compression level to use for compressing the logs. + log_compression_level: Compression, + + /// Maximum number of lines written per second continuously + max_lines_per_minute: NonZeroU32, + + /// Rules to convert logs to metrics + #[cfg_attr(not(feature = "log-to-metrics"), allow(dead_code))] + log_to_metrics_rules: Vec, + + /// Maximum number of lines to keep in memory + in_memory_lines: usize, + + /// Whether or not to persist log lines + storage_config: StorageConfig, +} + +impl From<&Config> for LogCollectorConfig { + fn from(config: &Config) -> Self { + Self { + log_tmp_path: config.logs_path(), + log_max_size: config.config_file.logs.rotate_size, + log_max_duration: config.config_file.logs.rotate_after, + log_compression_level: config.config_file.logs.compression_level, + max_lines_per_minute: config.config_file.logs.max_lines_per_minute, + log_to_metrics_rules: config + .config_file + .logs + .log_to_metrics + .as_ref() + .map(|c| c.rules.clone()) + .unwrap_or_default(), + in_memory_lines: config.config_file.coredump.log_lines, + storage_config: config.config_file.logs.storage, + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +/// A list of crash logs. +/// +/// This structure is passed to the client when they request the crash logs. +pub struct CrashLogs { + pub logs: Vec, +} + +/// A handler for the /api/v1/crash-logs endpoint. +pub struct CrashLogHandler { + inner: Arc>>>, +} + +impl CrashLogHandler { + fn new(inner: Arc>>>) -> Self { + Self { inner } + } + + /// Handle a GET request to /api/v1/crash-logs + /// + /// Will take a snapshot of the current circular queue and return it as a JSON array. + fn handle_get_crash_logs(&self) -> Result { + let logs = self + .inner + .lock() + .expect("Log collector mutex poisoned") + .as_mut() + .ok_or_else(|| eyre!("Log collector has already shutdown."))? + .get_log_queue()?; + let crash_logs = CrashLogs { logs }; + + let serialized_logs = serde_json::to_string(&crash_logs)?; + let logs_len = serialized_logs.as_bytes().len(); + Ok(Response::new( + StatusCode(200), + vec![Header::from_strings("Content-Type", "application/json")?], + Cursor::new(serialized_logs), + Some(logs_len), + None, + ) + .boxed()) + } +} + +impl HttpHandler for CrashLogHandler { + fn handle_request(&self, request: &mut Request) -> HttpHandlerResult { + if request.url() == CRASH_LOGS_URL { + match *request.method() { + Method::Get => self.handle_get_crash_logs().into(), + _ => HttpHandlerResult::Response(Response::empty(405).boxed()), + } + } else { + HttpHandlerResult::NotHandled + } + } +} + +#[cfg(test)] +mod tests { + use std::cmp::min; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::mpsc::{channel, Receiver}; + use std::sync::Arc; + use std::{fs::remove_file, sync::Mutex}; + use std::{io::Write, path::PathBuf, time::Duration}; + use std::{mem::replace, num::NonZeroU32}; + + use crate::logs::log_file::{LogFile, LogFileControl}; + use crate::test_utils::setup_logger; + use crate::{logs::completed_log::CompletedLog, metrics::MetricReportManager}; + use crate::{logs::headroom::HeadroomCheck, util::circular_queue::CircularQueue}; + use eyre::Context; + use flate2::Compression; + use rstest::{fixture, rstest}; + use serde_json::{json, Value}; + use tempfile::{tempdir, TempDir}; + use tiny_http::{Method, TestRequest}; + use uuid::Uuid; + + use super::*; + + const IN_MEMORY_LINES: usize = 100; + + #[rstest] + fn write_logs_to_disk(mut fixture: LogFixture) { + fixture.write_log(json!({"ts": 0, "MESSAGE": "xxx"})); + assert_eq!(fixture.count_log_files(), 1); + assert_eq!(fixture.on_log_completion_calls(), 0); + } + + #[rstest] + #[case(50)] + #[case(100)] + #[case(150)] + fn circular_log_queue(#[case] mut log_count: usize, mut fixture: LogFixture) { + for i in 0..log_count { + fixture.write_log(json!({"ts": i, "MESSAGE": "xxx"})); + } + + let log_queue = fixture.get_log_queue(); + + // Assert that the last value in the queue has the correct timestamp + let last_val = log_queue.back().unwrap(); + let ts = last_val.get("ts").unwrap().as_u64().unwrap(); + assert_eq!(ts, log_count as u64 - 1); + + // Clamp the log_count to the maximum size of the queue + log_count = min(log_count, IN_MEMORY_LINES); + assert_eq!(log_queue.len(), log_count); + } + + #[rstest] + fn clamp_coredump_log_count(fixture: LogFixture) { + let config = LogCollectorConfig { + log_tmp_path: fixture.logs_dir.path().to_owned(), + log_max_size: 1024, + log_max_duration: Duration::from_secs(3600), + log_compression_level: Compression::default(), + max_lines_per_minute: NonZeroU32::new(1_000).unwrap(), + log_to_metrics_rules: vec![], + in_memory_lines: 1000, + storage_config: StorageConfig::Persist, + }; + + let mut collector = LogCollector::open( + config, + |CompletedLog { path, .. }| { + remove_file(&path) + .with_context(|| format!("rm {path:?}")) + .unwrap(); + Ok(()) + }, + StubHeadroomLimiter, + Arc::new(Mutex::new(MetricReportManager::new())), + ) + .unwrap(); + + let log_queue = collector + .with_mut_inner(|inner| Ok(replace(&mut inner.log_queue, CircularQueue::new(1000)))) + .unwrap(); + + // The log queue should be clamped to the maximum size + assert_eq!( + log_queue.capacity(), + LogCollector::::MAX_IN_MEMORY_LINES + ); + } + + #[rstest] + fn do_not_create_newfile_on_close(mut fixture: LogFixture) { + fixture.write_log(json!({"ts": 0, "MESSAGE": "xxx"})); + fixture.collector.close_internal().expect("error closing"); + // 0 because the fixture "on_log_completion" moves the file out + assert_eq!(fixture.count_log_files(), 0); + assert_eq!(fixture.on_log_completion_calls(), 1); + } + + #[rstest] + #[case(StorageConfig::Persist, 25)] + #[case(StorageConfig::Disabled, 0)] + fn log_persistence( + #[case] storage_config: StorageConfig, + #[case] expected_size: usize, + mut fixture: LogFixture, + _setup_logger: (), + ) { + fixture.set_log_config(storage_config); + + fixture.write_log(json!({"ts": 0, "MESSAGE": "xxx"})); + fixture.flush_log_writes().unwrap(); + + assert_eq!(fixture.count_log_files(), 1); + assert_eq!(fixture.read_log_len(), expected_size); + } + + #[rstest] + fn forced_rotation_with_nonempty_log(mut fixture: LogFixture) { + fixture.write_log(json!({"ts": 0, "MESSAGE": "xxx"})); + + fixture.collector.flush_logs().unwrap(); + + assert_eq!(fixture.count_log_files(), 0); + assert_eq!(fixture.on_log_completion_calls(), 1); + } + + #[rstest] + fn delete_log_after_failed_on_completion_callback(mut fixture: LogFixture) { + fixture + .on_completion_should_fail + .store(true, Ordering::Relaxed); + fixture.write_log(test_line()); + + fixture.collector.flush_logs().unwrap(); + + assert_eq!(fixture.on_log_completion_calls(), 1); + + // The old log should have been deleted, to avoid accumulating logs that fail to be moved. + // No new file will be created without a subsequent write + assert_eq!(fixture.count_log_files(), 0); + } + + #[rstest] + fn forced_rotation_with_empty_log(mut fixture: LogFixture) { + fixture.collector.flush_logs().unwrap(); + + assert_eq!(fixture.count_log_files(), 0); + assert_eq!(fixture.on_log_completion_calls(), 0); + } + + #[rstest] + fn forced_rotation_with_write_after_rotate(mut fixture: LogFixture) { + fixture.write_log(test_line()); + fixture.collector.flush_logs().unwrap(); + + fixture.write_log(test_line()); + assert_eq!(fixture.count_log_files(), 1); + assert_eq!(fixture.on_log_completion_calls(), 1); + } + + #[rstest] + fn recover_old_logfiles() { + let (tmp_logs, _old_file_path) = existing_tmplogs_with_log(&(Uuid::new_v4().to_string())); + let fixture = collector_with_logs_dir(tmp_logs); + + // We should have generated a MAR entry for the pre-existing logfile. + assert_eq!(fixture.on_log_completion_calls(), 1); + } + + #[rstest] + fn delete_files_that_are_not_uuids() { + let (tmp_logs, old_file_path) = existing_tmplogs_with_log("testfile"); + let fixture = collector_with_logs_dir(tmp_logs); + + // And we should have removed the bogus file + assert!(!old_file_path.exists()); + + // We should NOT have generated a MAR entry for the pre-existing bogus file. + assert_eq!(fixture.on_log_completion_calls(), 0); + } + + #[rstest] + fn http_handler_log_get(mut fixture: LogFixture) { + let logs = vec![ + json!({"ts": 0, "MESSAGE": "xxx"}), + json!({"ts": 1, "MESSAGE": "yyy"}), + json!({"ts": 2, "MESSAGE": "zzz"}), + ]; + let log_strings = logs.iter().map(|l| l.to_string()).collect::>(); + + for log in &logs { + fixture.write_log(log.clone()); + } + + let inner = fixture.collector.inner.clone(); + let handler = CrashLogHandler::new(inner); + + let log_response = handler.handle_get_crash_logs().unwrap(); + let mut log_response_string = String::new(); + log_response + .into_reader() + .read_to_string(&mut log_response_string) + .unwrap(); + + let crash_logs: CrashLogs = serde_json::from_str(&log_response_string).unwrap(); + assert_eq!(crash_logs.logs, log_strings); + } + + #[rstest] + #[case(Method::Post)] + #[case(Method::Put)] + #[case(Method::Delete)] + #[case(Method::Patch)] + fn http_handler_unsupported_method(fixture: LogFixture, #[case] method: Method) { + let inner = fixture.collector.inner.clone(); + let handler = CrashLogHandler::new(inner); + + let request = TestRequest::new() + .with_path(CRASH_LOGS_URL) + .with_method(method); + let response = handler + .handle_request(&mut request.into()) + .expect("Error handling request"); + assert_eq!(response.status_code().0, 405); + } + + #[rstest] + fn unhandled_url(fixture: LogFixture) { + let inner = fixture.collector.inner.clone(); + let handler = CrashLogHandler::new(inner); + + let request = TestRequest::new().with_path("/api/v1/other"); + let response = handler.handle_request(&mut request.into()); + assert!(matches!(response, HttpHandlerResult::NotHandled)); + } + + fn existing_tmplogs_with_log(filename: &str) -> (TempDir, PathBuf) { + let tmp_logs = tempdir().unwrap(); + let file_path = tmp_logs + .path() + .to_path_buf() + .join(filename) + .with_extension("log.zlib"); + + let mut file = std::fs::File::create(&file_path).unwrap(); + file.write_all(b"some content in the log").unwrap(); + drop(file); + (tmp_logs, file_path) + } + + struct LogFixture { + collector: LogCollector, + // TempDir needs to be after the collector, otherwise we fail to delete + // the file in LogCollector::Drop because the tempdir is gone + logs_dir: TempDir, + on_log_completion_receiver: Receiver<(PathBuf, Uuid)>, + on_completion_should_fail: Arc, + } + impl LogFixture { + fn count_log_files(&self) -> usize { + std::fs::read_dir(&self.logs_dir).unwrap().count() + } + + fn write_log(&mut self, line: Value) { + self.collector + .with_mut_inner(|inner| inner.process_log_record(line)) + .unwrap(); + } + + fn read_log_len(&mut self) -> usize { + self.collector + .with_mut_inner(|inner| { + let log = inner.log_file_control.current_log()?; + Ok(log.bytes_written()) + }) + .unwrap() + } + + fn flush_log_writes(&mut self) -> Result<()> { + self.collector + .with_mut_inner(|inner| inner.log_file_control.current_log()?.flush()) + } + + fn on_log_completion_calls(&self) -> usize { + self.on_log_completion_receiver.try_iter().count() + } + + fn get_log_queue(&mut self) -> CircularQueue { + self.collector + .with_mut_inner(|inner| Ok(replace(&mut inner.log_queue, CircularQueue::new(100)))) + .unwrap() + } + + fn set_log_config(&mut self, storage_config: StorageConfig) { + self.collector + .with_mut_inner(|inner| { + inner.storage_config = storage_config; + Ok(()) + }) + .unwrap() + } + } + + #[fixture] + fn fixture() -> LogFixture { + collector_with_logs_dir(tempdir().unwrap()) + } + + struct StubHeadroomLimiter; + + impl HeadroomCheck for StubHeadroomLimiter { + fn check( + &mut self, + _log_timestamp: Option<&Value>, + _log_file_control: &mut impl LogFileControl, + ) -> eyre::Result { + Ok(true) + } + } + + fn collector_with_logs_dir(logs_dir: TempDir) -> LogFixture { + let config = LogCollectorConfig { + log_tmp_path: logs_dir.path().to_owned(), + log_max_size: 1024, + log_max_duration: Duration::from_secs(3600), + log_compression_level: Compression::default(), + max_lines_per_minute: NonZeroU32::new(1_000).unwrap(), + log_to_metrics_rules: vec![], + in_memory_lines: IN_MEMORY_LINES, + storage_config: StorageConfig::Persist, + }; + + let (on_log_completion_sender, on_log_completion_receiver) = channel(); + + let on_completion_should_fail = Arc::new(AtomicBool::new(false)); + + let heartbeat_manager = Arc::new(Mutex::new(MetricReportManager::new())); + + let collector = { + let on_completion_should_fail = on_completion_should_fail.clone(); + let on_log_completion = move |CompletedLog { path, cid, .. }| { + on_log_completion_sender.send((path.clone(), cid)).unwrap(); + if on_completion_should_fail.load(Ordering::Relaxed) { + // Don't move / unlink the log file. The LogCollector should clean up now. + Err(eyre::eyre!("on_log_completion failure!")) + } else { + // Unlink the log file. The real implementation moves it into the MAR staging area. + remove_file(&path) + .with_context(|| format!("rm {path:?}")) + .unwrap(); + Ok(()) + } + }; + + LogCollector::open( + config, + on_log_completion, + StubHeadroomLimiter, + heartbeat_manager, + ) + .unwrap() + }; + + LogFixture { + logs_dir, + collector, + on_log_completion_receiver, + on_completion_should_fail, + } + } + + fn test_line() -> Value { + json!({"ts": 0, "MESSAGE": "xxx"}) + } +} diff --git a/memfaultd/src/logs/log_entry.rs b/memfaultd/src/logs/log_entry.rs new file mode 100644 index 0000000..70f66c1 --- /dev/null +++ b/memfaultd/src/logs/log_entry.rs @@ -0,0 +1,87 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::collections::HashMap; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::util::serialization::datetime_to_rfc3339; + +// journal fields that should always be captured by memfaultd: +// https://man7.org/linux/man-pages/man7/systemd.journal-fields.7.html +const ALWAYS_INCLUDE_KEYS: &[&str] = &["MESSAGE", "_PID", "_SYSTEMD_UNIT", "PRIORITY"]; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(untagged)] +pub enum LogValue { + String(String), + Float(f64), +} + +/// Represents a structured log that could come from a variety of sources. +#[derive(Debug, Serialize, Deserialize)] +pub struct LogEntry { + #[serde(with = "datetime_to_rfc3339")] + pub ts: DateTime, + pub data: HashMap, +} + +impl LogEntry { + /// Filter log fields to only include defaults and those specified in `extra_fields`. + /// + /// This function modifies the log entry in place by removing fields that are not in the + /// `ALWAYS_INCLUDE_KEYS` list or in `extra_fields`. This is useful for reducing the size of + /// log entries sent to Memfault, as there are fields that are not useful or displayed. + pub fn filter_fields(&mut self, extra_fields: &[String]) { + self.data + .retain(|k, _| ALWAYS_INCLUDE_KEYS.contains(&k.as_str()) || extra_fields.contains(k)); + } +} + +#[cfg(test)] +mod tests { + use chrono::{DateTime, TimeZone, Utc}; + use insta::{assert_json_snapshot, with_settings}; + use rstest::{fixture, rstest}; + + use super::*; + + #[rstest] + #[case("empty", "{}", "")] + #[case("only_message", r#"{"MESSAGE":"TEST" }"#, "")] + #[case("extra_key", r#"{"MESSAGE":"TEST", "SOME_EXTRA_KEY":"XX" }"#, "")] + #[case( + "multi_key_match", + r#"{"MESSAGE":"TEST", "SOME_EXTRA_KEY":"XX", "_PID": "44", "_SYSTEMD_UNIT": "some.service", "PRIORITY": "6" }"#, + "" + )] + #[case( + "extra_attribute_filter", + r#"{"MESSAGE":"TEST", "SOME_EXTRA_KEY":"XX" }"#, + "SOME_EXTRA_KEY" + )] + fn test_filtering( + time: DateTime, + #[case] test_name: String, + #[case] input: String, + #[case] extras: String, + ) { + let mut entry = LogEntry { + ts: time, + data: serde_json::from_str(&input).unwrap(), + }; + + let extra_attributes = extras.split(',').map(String::from).collect::>(); + entry.filter_fields(&extra_attributes); + + with_settings!({sort_maps => true}, { + assert_json_snapshot!(test_name, entry); + }); + } + + #[fixture] + fn time() -> DateTime { + Utc.timestamp_millis_opt(1334250000000).unwrap() + } +} diff --git a/memfaultd/src/logs/log_file.rs b/memfaultd/src/logs/log_file.rs new file mode 100644 index 0000000..23ce6d9 --- /dev/null +++ b/memfaultd/src/logs/log_file.rs @@ -0,0 +1,267 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Contains LogFile and LogFileControl traits and their real implementations. +//! +use crate::logs::completed_log::CompletedLog; +use crate::mar::CompressionAlgorithm; +use eyre::{eyre, Result, WrapErr}; +use flate2::write::ZlibEncoder; +use flate2::Compression; +use log::{trace, warn}; +use serde_json::{json, Value}; +use std::fs::{remove_file, File}; +use std::io::{BufWriter, Write}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; +use uuid::Uuid; + +pub trait LogFile { + fn write_json_line(&mut self, json: Value) -> Result<()>; + fn write_log>(&mut self, ts: Option, msg: S) -> Result<()> { + self.write_json_line(json!({ + "ts": ts, + "data": { "MESSAGE": msg.as_ref() } + })) + } + fn flush(&mut self) -> Result<()>; +} + +/// In memory representation of one logfile while it is being written to. +pub struct LogFileImpl { + cid: Uuid, + path: PathBuf, + writer: BufWriter>, + bytes_written: usize, + since: Instant, +} + +impl LogFileImpl { + fn open(log_tmp_path: &Path, cid: Uuid, compression_level: Compression) -> Result { + let filename = cid.to_string() + ".log.zlib"; + let path = log_tmp_path.join(filename); + let file = File::create(&path)?; + let writer = BufWriter::new(ZlibEncoder::new(file, compression_level)); + + trace!("Now writing logs to: {}", path.display()); + Ok(LogFileImpl { + cid, + path, + writer, + bytes_written: 0, + since: Instant::now(), + }) + } + + #[cfg(test)] + pub fn bytes_written(&self) -> usize { + self.bytes_written + } +} + +impl LogFile for LogFileImpl { + fn write_json_line(&mut self, json: Value) -> Result<()> { + let bytes = serde_json::to_vec(&json)?; + let mut written = self.writer.write(&bytes)?; + written += self.writer.write("\n".as_bytes())?; + self.bytes_written += written; + Ok(()) + } + + fn flush(&mut self) -> Result<()> { + self.writer.flush().wrap_err("Flush error") + } +} + +pub trait LogFileControl { + fn rotate_if_needed(&mut self) -> Result; + fn rotate_unless_empty(&mut self) -> Result<()>; + fn current_log(&mut self) -> Result<&mut L>; + fn close(self) -> Result<()>; +} + +/// Controls the creation and rotation of logfiles. +pub struct LogFileControlImpl { + current_log: Option, + tmp_path: PathBuf, + max_size: usize, + max_duration: Duration, + compression_level: Compression, + on_log_completion: Box<(dyn FnMut(CompletedLog) -> Result<()> + Send)>, + next_cid: Uuid, +} + +impl LogFileControlImpl { + pub fn open Result<()> + Send + 'static>( + tmp_path: PathBuf, + next_cid: Uuid, + max_size: usize, + max_duration: Duration, + compression_level: Compression, + on_log_completion: R, + ) -> Result { + Ok(LogFileControlImpl { + current_log: None, + tmp_path, + max_size, + max_duration, + compression_level, + on_log_completion: Box::new(on_log_completion), + next_cid, + }) + } + + /// Close current logfile, create a MAR entry and starts a new one. + fn rotate_log(&mut self) -> Result<()> { + let current_log = self.current_log.take(); + + if let Some(current_log) = current_log { + self.next_cid = Uuid::new_v4(); + + Self::dispatch_on_log_completion( + &mut self.on_log_completion, + current_log, + self.next_cid, + ); + } + + Ok(()) + } + + fn dispatch_on_log_completion( + on_log_completion: &mut Box<(dyn FnMut(CompletedLog) -> Result<()> + Send)>, + mut log: LogFileImpl, + next_cid: Uuid, + ) { + // Drop the old log, finishing the compression, closing the buffered writer and the file. + log.writer.flush().unwrap_or_else(|e| { + warn!("Failed to flush logs: {}", e); + }); + + let LogFileImpl { path, cid, .. } = log; + + // The callback is responsible for moving the file to its final location (or deleting it): + (on_log_completion)(CompletedLog { + path: path.clone(), + cid, + next_cid, + compression: CompressionAlgorithm::Zlib, + }) + .unwrap_or_else(|e| { + warn!( + "Dropping log due to failed on_log_completion callback: {}", + e + ); + remove_file(&path).unwrap_or_else(|e| { + warn!("Failed to remove log file: {}", e); + }); + }); + } +} + +impl LogFileControl for LogFileControlImpl { + fn rotate_if_needed(&mut self) -> Result { + if let Some(current_log) = &mut self.current_log { + if current_log.bytes_written >= self.max_size + || current_log.since.elapsed() > self.max_duration + { + self.rotate_log()?; + Ok(true) + } else { + Ok(false) + } + } else { + Ok(false) + } + } + + fn rotate_unless_empty(&mut self) -> Result<()> { + if let Some(current_log) = &self.current_log { + if current_log.bytes_written > 0 { + self.rotate_log()?; + } + } + Ok(()) + } + + fn current_log(&mut self) -> Result<&mut LogFileImpl> { + if self.current_log.is_none() { + self.current_log = Some( + LogFileImpl::open(&self.tmp_path, self.next_cid, self.compression_level) + .map_err(|e| eyre!("Failed to open log file: {e}"))?, + ); + } + + // NOTE: The error case should not be possible here as it is always set above. + // still this is better than panicking. + self.current_log + .as_mut() + .ok_or_else(|| eyre!("No current log")) + } + + fn close(mut self) -> Result<()> { + if let Some(current_log) = self.current_log { + if current_log.bytes_written > 0 { + Self::dispatch_on_log_completion( + &mut self.on_log_completion, + current_log, + Uuid::new_v4(), + ); + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::io::Read; + + use super::*; + use flate2::bufread::ZlibDecoder; + use rand::distributions::{Alphanumeric, DistString}; + use rstest::rstest; + use tempfile::tempdir; + + // We saw this bug when we tried switching to the rust-based backend for flate2 (miniz-oxide) + // With miniz 0.7.1 and flate2 1.0.28, this test does not pass. + #[rstest] + fn test_write_without_corruption() { + let tmp = tempdir().expect("tmpdir"); + + // Generate a logfile with lots of bogus data + let mut log = LogFileImpl::open(tmp.path(), Uuid::new_v4(), Compression::fast()) + .expect("open log error"); + let mut count_lines = 0; + while log.bytes_written < 1024 * 1024 { + let message = format!( + "bogus {} bogum {} bodoum", + Alphanumeric.sample_string(&mut rand::thread_rng(), 16), + Alphanumeric.sample_string(&mut rand::thread_rng(), 20), + ); + log.write_json_line(json!({ "data": { "message": message, "prio": 42, "unit": "systemd"}, "ts": "2023-22-22T22:22:22Z"})).expect("error writing json line"); + count_lines += 1; + } + + let logfile = log.path.clone(); + drop(log); + + // Decompress without error + let bytes = std::fs::read(&logfile).expect("Unable to read {filename}"); + let mut z = ZlibDecoder::new(&bytes[..]); + let mut loglines = String::new(); + z.read_to_string(&mut loglines).expect("read error"); + + // Check we have all the lines + assert_eq!(count_lines, loglines.lines().count()); + + // Check all lines are valid json + let mut count_invalid_lines = 0; + for line in loglines.lines() { + if serde_json::from_str::(line).is_err() { + count_invalid_lines += 1; + } + } + assert_eq!(count_invalid_lines, 0); + } +} diff --git a/memfaultd/src/logs/log_to_metrics.rs b/memfaultd/src/logs/log_to_metrics.rs new file mode 100644 index 0000000..708e8cc --- /dev/null +++ b/memfaultd/src/logs/log_to_metrics.rs @@ -0,0 +1,204 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; + +use eyre::Result; +use log::{debug, warn}; +use regex::Regex; +use serde_json::{Map, Value}; + +use crate::{config::LogToMetricRule, metrics::MetricReportManager}; + +const SEARCH_FIELD: &str = "MESSAGE"; + +pub struct LogToMetrics { + rules: Vec, + heartbeat_manager: Arc>, + regex_cache: HashMap, +} + +impl LogToMetrics { + pub fn new( + rules: Vec, + heartbeat_manager: Arc>, + ) -> Self { + Self { + rules, + heartbeat_manager, + regex_cache: HashMap::new(), + } + } + + pub fn process(&mut self, structured_log: &Value) -> Result<()> { + if let Some(data) = structured_log["data"].as_object() { + if !self.rules.is_empty() { + debug!("LogToMetrics: Processing log: {:?}", data); + for rule in &self.rules { + match rule { + LogToMetricRule::CountMatching { + pattern, + metric_name, + filter, + } => Self::apply_count_matching( + data, + pattern, + &mut self.regex_cache, + metric_name, + filter, + self.heartbeat_manager.clone(), + ), + } + } + } + } + Ok(()) + } + + fn get_metric_name_with_captures(metric_name: &str, captures: regex::Captures) -> String { + let mut metric_name_with_captures = metric_name.to_string(); + for (i, capture) in captures.iter().enumerate() { + if let Some(capture) = capture { + metric_name_with_captures = + metric_name_with_captures.replace(&format!("${}", i), capture.as_str()); + } + } + metric_name_with_captures + } + + fn apply_count_matching( + data: &Map, + pattern: &str, + regex_cache: &mut HashMap, + metric_name: &str, + filter: &HashMap, + heartbeat_manager: Arc>, + ) { + // Use filter to quickly disqualify a log entry + for (key, value) in filter { + if let Some(log_value) = data.get(key) { + if log_value != value { + return; + } + } else { + return; + } + } + + let regex = regex_cache + .entry(pattern.to_string()) + .or_insert_with(|| Regex::new(pattern).unwrap()); + if let Some(search_value) = data[SEARCH_FIELD].as_str() { + let captures = regex.captures(search_value); + debug!( + "LogToMetrics Pattern '{}'=> MATCH={} Captures={:?}", + &pattern, + captures.is_some(), + captures + ); + + if let Some(captures) = captures { + let metric_name_with_captures = + Self::get_metric_name_with_captures(metric_name, captures); + + if let Err(e) = heartbeat_manager + .lock() + .unwrap() + .increment_counter(&metric_name_with_captures) + { + warn!("Failed to increment metric: {}", e) + } + } + } + } +} + +#[cfg(test)] +mod tests { + use crate::{metrics::MetricValue, test_utils::setup_logger}; + + use super::*; + use rstest::rstest; + use serde_json::json; + + #[rstest] + #[case(vec![LogToMetricRule::CountMatching { + pattern: "foo".to_string(), + metric_name: "foo".to_string(), + filter: HashMap::default() + }], vec![json!({"MESSAGE": "foo"})], "foo", 1.0)] + #[case(vec![LogToMetricRule::CountMatching { + pattern: "session opened for user (\\w*)\\(uid=".to_string(), + metric_name: "ssh_sessions_$1_count".to_string(), + filter: HashMap::default() + }], vec![json!({"MESSAGE": "pam_unix(sshd:session): session opened for user thomas(uid=1000) by (uid=0)"})], "ssh_sessions_thomas_count", 1.0)] + #[case(vec![LogToMetricRule::CountMatching { + pattern: "(.*): Scheduled restart job, restart counter is at".to_string(), + metric_name: "$1_restarts".to_string(), + filter: HashMap::default() + }], vec![json!({"MESSAGE": /* systemd[1]: */"docker.service: Scheduled restart job, restart counter is at 1."})], "docker.service_restarts", 1.0)] + #[case(vec![LogToMetricRule::CountMatching { + pattern: "(.*): Scheduled restart job, restart counter is at".to_string(), + metric_name: "$1_restarts".to_string(), + filter: HashMap::default() + }], + vec![ + json!({"MESSAGE": /* systemd[1]: */"docker.service: Scheduled restart job, restart counter is at 1."}), + json!({"MESSAGE": /* systemd[1]: */"sshd.service: Scheduled restart job, restart counter is at 1."}), + json!({"MESSAGE": /* systemd[1]: */"docker.service: Scheduled restart job, restart counter is at 2."}), + ], "docker.service_restarts", 2.0) + ] + #[case(vec![LogToMetricRule::CountMatching { + pattern: "(.*): Scheduled restart job, restart counter is at".to_string(), + metric_name: "$1_restarts".to_string(), + filter: HashMap::from([("UNIT".to_owned(), "systemd".to_owned())]) + }], vec![json!({"MESSAGE": /* systemd[1]: */"docker.service: Scheduled restart job, restart counter is at 1.", "UNIT": "systemd"})], "docker.service_restarts", 1.0)] + #[case(vec![LogToMetricRule::CountMatching { + pattern: "(.*): Scheduled restart job, restart counter is at".to_string(), + metric_name: "$1_restarts".to_string(), + filter: HashMap::from([("_SYSTEMD_UNIT".to_owned(), "ssh.service".to_owned())]) + }], vec![json!({"MESSAGE": /* systemd[1]: */"docker.service: Scheduled restart job, restart counter is at 1.", "_SYSTEMD_UNIT": ""})], "docker.service_restarts", 0.0)] + #[case(vec![LogToMetricRule::CountMatching { + pattern: "Out of memory: Killed process \\d+ \\((.*)\\)".to_string(), + metric_name: "oomkill_$1".to_string(), + filter: HashMap::default() + }], vec![json!({"MESSAGE": "Out of memory: Killed process 423 (wefaultd) total-vm:553448kB, anon-rss:284496kB, file-rss:0kB, shmem-rss:0kB, UID:0 pgtables:624kB oom_score_adj:0"})], "oomkill_wefaultd", 1.0)] + + fn test_log_to_metrics( + #[case] rules: Vec, + #[case] logs: Vec, + #[case] metric_name: &str, + #[case] expected_value: f64, + _setup_logger: (), + ) { + let metric_report_manager = Arc::new(Mutex::new(MetricReportManager::new())); + let mut log_to_metrics = LogToMetrics::new(rules, metric_report_manager.clone()); + + for log in logs { + log_to_metrics + .process(&json!({ "data": log })) + .expect("process error"); + } + let metrics = metric_report_manager + .lock() + .unwrap() + .take_heartbeat_metrics(); + + if expected_value == 0.0 { + assert!(!metrics.iter().any(|m| m.0.as_str() == metric_name)); + } else { + let m = metrics + .iter() + .find(|m| m.0.as_str() == metric_name) + .unwrap(); + + match m.1 { + MetricValue::Number(v) => assert_eq!(*v, expected_value), + _ => panic!("This test only expects number metric values!"), + } + } + } +} diff --git a/memfaultd/src/logs/mod.rs b/memfaultd/src/logs/mod.rs new file mode 100644 index 0000000..059788b --- /dev/null +++ b/memfaultd/src/logs/mod.rs @@ -0,0 +1,21 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +pub mod completed_log; +pub use completed_log::CompletedLog; +pub mod fluent_bit_adapter; +pub use fluent_bit_adapter::FluentBitAdapter; +pub mod log_collector; +pub use log_collector::{LogCollector, LogCollectorConfig}; +pub mod headroom; +pub use headroom::HeadroomLimiter; +#[cfg(feature = "systemd")] +mod journald_parser; +#[cfg(feature = "systemd")] +pub mod journald_provider; +pub mod log_entry; +mod log_file; +mod recovery; + +#[cfg(feature = "log-to-metrics")] +mod log_to_metrics; diff --git a/memfaultd/src/logs/recovery.rs b/memfaultd/src/logs/recovery.rs new file mode 100644 index 0000000..b8d95c0 --- /dev/null +++ b/memfaultd/src/logs/recovery.rs @@ -0,0 +1,282 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Recovery of old log files after restarting the memfaultd service. +//! +use std::fs; +use std::fs::remove_file; +use std::iter::{once, zip}; +use std::mem::take; +use std::path::{Path, PathBuf}; + +use crate::logs::completed_log::CompletedLog; +use eyre::Result; +use log::{debug, warn}; +use uuid::Uuid; + +use crate::mar::CompressionAlgorithm; +use crate::util::fs::get_files_sorted_by_mtime; +use crate::util::path::file_prefix; + +struct FileInfo { + path: PathBuf, + uuid: Option, + size: Option, +} + +#[derive(Debug, PartialEq)] +struct LogFileToRecover { + path: PathBuf, + cid: Uuid, + next_cid: Uuid, +} + +#[derive(Debug, PartialEq)] +struct Recovery { + to_delete: Vec, + to_recover: Vec, + next_cid: Uuid, +} + +fn should_recover(file_info: &FileInfo) -> bool { + match file_info { + FileInfo { uuid: None, .. } => false, + FileInfo { size: None, .. } => false, + FileInfo { + size: Some(size), .. + } if *size == 0 => false, + // Only keep files that are not empty and have a valid uuid: + _ => true, + } +} + +/// The "functional core" of the recovery logic. It is pure for unit-testing sake. +/// Note: the file_infos must be sorted by mtime, newest last. +fn get_recovery(file_infos: Vec, gen_uuid: fn() -> Uuid) -> Recovery { + // If the last file was empty we'll delete it, but we want to reuse the CID because it's + // possible the previous file is already uploaded and references that CID: + let last_cid = file_infos + .iter() + .filter_map(|info| match info { + FileInfo { + uuid: Some(uuid), .. + } => match should_recover(info) { + true => Some(gen_uuid()), + false => Some(*uuid), + }, + _ => None, + }) + .last() + .unwrap_or_else(gen_uuid); + + let (mut to_recover_infos, to_delete_infos): (Vec, Vec) = + file_infos.into_iter().partition(should_recover); + + #[allow(clippy::needless_collect)] + let next_cids: Vec = to_recover_infos + .iter() + .skip(1) + .map(|i| i.uuid.unwrap()) + .chain(once(last_cid)) + .collect(); + + Recovery { + to_delete: to_delete_infos.into_iter().map(|info| info.path).collect(), + to_recover: zip(to_recover_infos.iter_mut(), next_cids) + .map(|(info, next_cid)| LogFileToRecover { + path: take(&mut info.path), + cid: info.uuid.unwrap(), + next_cid, + }) + .collect(), + next_cid: last_cid, + } +} + +pub fn recover_old_logs Result<()> + Send + 'static>( + tmp_logs: &Path, + on_log_recovery: &mut R, +) -> Result { + // Make a list of all the info of the files we want to collect, parsing the CID from the + // filename and getting the size on disk: + let file_infos = get_files_sorted_by_mtime(tmp_logs)? + .into_iter() + .map(|path| { + let uuid = file_prefix(&path) + .and_then(|prefix| Uuid::parse_str(&prefix.to_string_lossy()).ok()); + let size = fs::metadata(&path).ok().map(|metadata| metadata.len()); + FileInfo { path, uuid, size } + }) + .collect(); + + let Recovery { + next_cid, + to_delete, + to_recover, + } = get_recovery(file_infos, Uuid::new_v4); + + // Delete any unwanted files from disk: + for path in to_delete { + if let Err(e) = remove_file(&path) { + warn!( + "Unable to delete bogus log file: {} - {}.", + path.display(), + e + ); + } + } + + for LogFileToRecover { + path, + cid, + next_cid, + } in to_recover + { + // Write the MAR entry which will move the logfile + debug!("Recovering logfile: {:?}", path.display()); + + if let Err(e) = (on_log_recovery)(CompletedLog { + path, + cid, + next_cid, + compression: CompressionAlgorithm::Zlib, + }) { + warn!("Unable to recover log file: {}", e); + } + } + + Ok(next_cid) +} + +#[cfg(test)] +mod tests { + use super::*; + + const UUID_A: Uuid = Uuid::from_u128(1); + const UUID_B: Uuid = Uuid::from_u128(2); + const UUID_NEW: Uuid = Uuid::from_u128(3); + + const PATH_A: &str = "/tmp_log/11111111-1111-1111-1111-111111111111"; + const PATH_B: &str = "/tmp_log/22222222-2222-2222-2222-222222222222"; + + #[test] + fn empty_logging_directory() { + let file_infos: Vec = vec![]; + let expected = Recovery { + to_delete: vec![], + to_recover: vec![], + next_cid: UUID_NEW, + }; + assert_eq!(get_recovery(file_infos, gen_uuid), expected); + } + + #[test] + fn delete_improperly_named_files() { + // File w/o proper UUID name should get deleted, even if it's not empty: + let file_infos = vec![FileInfo { + path: PathBuf::from("/tmp_log/foo"), + uuid: None, + size: Some(1), + }]; + let expected = Recovery { + to_delete: vec![PathBuf::from("/tmp_log/foo")], + to_recover: vec![], + next_cid: UUID_NEW, + }; + assert_eq!(get_recovery(file_infos, gen_uuid), expected); + } + + #[test] + fn use_empty_trailing_uuid_named_file_as_next_cid() { + // Any trailing but empty UUID-named file should be used as next_cid: + let file_infos = vec![ + FileInfo { + path: PATH_A.into(), + uuid: Some(UUID_A), + size: Some(1), + }, + FileInfo { + path: PATH_B.into(), + uuid: Some(UUID_B), + size: Some(0), + }, + ]; + let expected = Recovery { + to_delete: vec![PATH_B.into()], + to_recover: vec![LogFileToRecover { + path: PATH_A.into(), + cid: UUID_A, + next_cid: UUID_B, + }], + next_cid: UUID_B, + }; + assert_eq!(get_recovery(file_infos, gen_uuid), expected); + } + + #[test] + fn dont_use_non_trailing_empty_uuid_named_file_as_next_cid() { + // An empty UUID-named file that is not trailing (has newer, non-empty files following) + // should not be used as next_cid: + let file_infos = vec![ + FileInfo { + path: PATH_A.into(), + uuid: Some(UUID_A), + size: Some(0), + }, + FileInfo { + path: PATH_B.into(), + uuid: Some(UUID_B), + size: Some(1), + }, + ]; + let expected = Recovery { + to_delete: vec![PATH_A.into()], + to_recover: vec![LogFileToRecover { + path: PATH_B.into(), + cid: UUID_B, + next_cid: UUID_NEW, + }], + next_cid: UUID_NEW, + }; + assert_eq!(get_recovery(file_infos, gen_uuid), expected); + } + + #[test] + fn chain_cids() { + // Chain CIDs, setting next_cid to the CID of the next file or a newly generated one if it's the + // last file (and it's not empty): + let file_infos = vec![ + FileInfo { + path: PATH_A.into(), + uuid: Some(UUID_A), + size: Some(1), + }, + FileInfo { + path: PATH_B.into(), + uuid: Some(UUID_B), + size: Some(1), + }, + ]; + let expected = Recovery { + to_delete: vec![], + to_recover: vec![ + LogFileToRecover { + path: PATH_A.into(), + cid: UUID_A, + next_cid: UUID_B, + }, + LogFileToRecover { + path: PATH_B.into(), + cid: UUID_B, + next_cid: UUID_NEW, + }, + ], + next_cid: UUID_NEW, + }; + assert_eq!(get_recovery(file_infos, gen_uuid), expected); + } + + fn gen_uuid() -> Uuid { + UUID_NEW + } +} diff --git a/memfaultd/src/logs/snapshots/memfaultd__logs__fluent_bit_adapter__tests__fluent_bit_adapter.snap b/memfaultd/src/logs/snapshots/memfaultd__logs__fluent_bit_adapter__tests__fluent_bit_adapter.snap new file mode 100644 index 0000000..2f0a909 --- /dev/null +++ b/memfaultd/src/logs/snapshots/memfaultd__logs__fluent_bit_adapter__tests__fluent_bit_adapter.snap @@ -0,0 +1,10 @@ +--- +source: memfaultd/src/logs/fluent_bit_adapter.rs +expression: log_entry +--- +{ + "ts": "2012-04-12T17:00:00+00:00", + "data": { + "MESSAGE": "test" + } +} diff --git a/memfaultd/src/logs/snapshots/memfaultd__logs__journald_parser__test__from_raw_journal_entry.snap b/memfaultd/src/logs/snapshots/memfaultd__logs__journald_parser__test__from_raw_journal_entry.snap new file mode 100644 index 0000000..7624f97 --- /dev/null +++ b/memfaultd/src/logs/snapshots/memfaultd__logs__journald_parser__test__from_raw_journal_entry.snap @@ -0,0 +1,11 @@ +--- +source: memfaultd/src/logs/journald_parser.rs +expression: entry +--- +{ + "ts": "1970-01-20T19:57:42.571Z", + "fields": { + "MESSAGE": "audit: type=1400 audit(1713462571.968:7508): apparmor=\"DENIED\" operation=\"open\" class=\"file\" profile=\"snap.firefox.firefox\" name=\"/etc/fstab\" pid=10122 comm=\"firefox\" requested_mask=\"r\" denied_mask=\"r\" fsuid=1000 ouid=0", + "_SYSTEMD_UNIT": "user@1000.service" + } +} diff --git a/memfaultd/src/logs/snapshots/memfaultd__logs__journald_parser__test__journal_happy_path.snap b/memfaultd/src/logs/snapshots/memfaultd__logs__journald_parser__test__journal_happy_path.snap new file mode 100644 index 0000000..ecc66d2 --- /dev/null +++ b/memfaultd/src/logs/snapshots/memfaultd__logs__journald_parser__test__journal_happy_path.snap @@ -0,0 +1,11 @@ +--- +source: memfaultd/src/logs/journald_parser.rs +expression: entry.unwrap() +--- +{ + "ts": "1970-01-20T19:57:42.571Z", + "fields": { + "MESSAGE": "audit: type=1400 audit(1713462571.968:7508): apparmor=\"DENIED\" operation=\"open\" class=\"file\" profile=\"snap.firefox.firefox\" name=\"/etc/fstab\" pid=10122 comm=\"firefox\" requested_mask=\"r\" denied_mask=\"r\" fsuid=1000 ouid=0", + "_SYSTEMD_UNIT": "user@1000.service" + } +} diff --git a/memfaultd/src/logs/snapshots/memfaultd__logs__journald_provider__test__happy_path.snap b/memfaultd/src/logs/snapshots/memfaultd__logs__journald_provider__test__happy_path.snap new file mode 100644 index 0000000..a5e13f5 --- /dev/null +++ b/memfaultd/src/logs/snapshots/memfaultd__logs__journald_provider__test__happy_path.snap @@ -0,0 +1,11 @@ +--- +source: memfaultd/src/logs/journald_provider.rs +expression: entry +--- +{ + "ts": "1970-01-01T00:00:01.337+00:00", + "data": { + "MESSAGE": "audit: type=1400 audit(1713462571.968:7508): apparmor=\"DENIED\" operation=\"open\" class=\"file\" profile=\"snap.firefox.firefox\" name=\"/etc/fstab\" pid=10122 comm=\"firefox\" requested_mask=\"r\" denied_mask=\"r\" fsuid=1000 ouid=0", + "_SYSTEMD_UNIT": "user@1000.service" + } +} diff --git a/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__empty.snap b/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__empty.snap new file mode 100644 index 0000000..603ec3f --- /dev/null +++ b/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__empty.snap @@ -0,0 +1,8 @@ +--- +source: memfaultd/src/logs/log_entry.rs +expression: entry +--- +{ + "ts": "2012-04-12T17:00:00+00:00", + "data": {} +} diff --git a/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__extra_attribute_filter.snap b/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__extra_attribute_filter.snap new file mode 100644 index 0000000..6bb2e0d --- /dev/null +++ b/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__extra_attribute_filter.snap @@ -0,0 +1,11 @@ +--- +source: memfaultd/src/logs/log_entry.rs +expression: entry +--- +{ + "ts": "2012-04-12T17:00:00+00:00", + "data": { + "MESSAGE": "TEST", + "SOME_EXTRA_KEY": "XX" + } +} diff --git a/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__extra_key.snap b/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__extra_key.snap new file mode 100644 index 0000000..57de0a8 --- /dev/null +++ b/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__extra_key.snap @@ -0,0 +1,10 @@ +--- +source: memfaultd/src/logs/log_entry.rs +expression: entry +--- +{ + "ts": "2012-04-12T17:00:00+00:00", + "data": { + "MESSAGE": "TEST" + } +} diff --git a/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__multi_key_match.snap b/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__multi_key_match.snap new file mode 100644 index 0000000..694bbee --- /dev/null +++ b/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__multi_key_match.snap @@ -0,0 +1,13 @@ +--- +source: memfaultd/src/logs/log_entry.rs +expression: entry +--- +{ + "ts": "2012-04-12T17:00:00+00:00", + "data": { + "MESSAGE": "TEST", + "PRIORITY": "6", + "_PID": "44", + "_SYSTEMD_UNIT": "some.service" + } +} diff --git a/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__only_message.snap b/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__only_message.snap new file mode 100644 index 0000000..57de0a8 --- /dev/null +++ b/memfaultd/src/logs/snapshots/memfaultd__logs__log_entry__tests__only_message.snap @@ -0,0 +1,10 @@ +--- +source: memfaultd/src/logs/log_entry.rs +expression: entry +--- +{ + "ts": "2012-04-12T17:00:00+00:00", + "data": { + "MESSAGE": "TEST" + } +} diff --git a/memfaultd/src/mar/chunks.rs b/memfaultd/src/mar/chunks.rs new file mode 100644 index 0000000..43f714a --- /dev/null +++ b/memfaultd/src/mar/chunks.rs @@ -0,0 +1,14 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +mod chunk; +mod chunk_header; +mod chunk_message; +mod chunk_wrapper; +mod crc_padded_stream; + +pub use chunk::Chunk; +pub use chunk_message::ChunkMessage; +pub use chunk_message::ChunkMessageType; +pub use chunk_wrapper::ChunkWrapper; +pub use crc_padded_stream::CRCPaddedStream; diff --git a/memfaultd/src/mar/chunks/chunk.rs b/memfaultd/src/mar/chunks/chunk.rs new file mode 100644 index 0000000..014837e --- /dev/null +++ b/memfaultd/src/mar/chunks/chunk.rs @@ -0,0 +1,81 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{Chain, Cursor, Read}; + +use crate::util::io::StreamLen; + +use super::{chunk_header::ChunkHeader, CRCPaddedStream}; + +/// One chunk of a message. +/// This implementation only supports Chunks v2 (with CRC deferred to the end of the last chunk composing the message). +pub struct Chunk { + stream: Chain>, CRCPaddedStream>, +} + +impl Chunk { + /// Create a new single chunk for a message. + pub fn new_single(message: M) -> Self { + Self { + stream: Cursor::new(ChunkHeader::new_single().as_bytes().to_vec()) + .chain(CRCPaddedStream::new(message)), + } + } +} + +impl Read for Chunk { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.stream.read(buf) + } +} + +impl StreamLen for Chunk { + fn stream_len(&self) -> u64 { + self.stream.stream_len() + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use crate::mar::ChunkMessage; + + use super::*; + use std::io::copy; + + #[rstest] + fn test_single_chunk_message() { + // https://docs.memfault.com/docs/mcu/test-patterns-for-chunks-endpoint/#event-message-encoded-in-a-single-chunk + let known_good_chunk = [ + 0x8, 0x2, 0xa7, 0x2, 0x1, 0x3, 0x1, 0x7, 0x6a, 0x54, 0x45, 0x53, 0x54, 0x53, 0x45, + 0x52, 0x49, 0x41, 0x4c, 0xa, 0x6d, 0x74, 0x65, 0x73, 0x74, 0x2d, 0x73, 0x6f, 0x66, + 0x74, 0x77, 0x61, 0x72, 0x65, 0x9, 0x6a, 0x31, 0x2e, 0x30, 0x2e, 0x30, 0x2d, 0x74, + 0x65, 0x73, 0x74, 0x6, 0x6d, 0x74, 0x65, 0x73, 0x74, 0x2d, 0x68, 0x61, 0x72, 0x64, + 0x77, 0x61, 0x72, 0x65, 0x4, 0xa1, 0x1, 0xa1, 0x72, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x5f, + 0x74, 0x65, 0x73, 0x74, 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x1, 0x31, + 0xe4, + ]; + + // Remove 2 bytes chunk header and 2 bytes CRC + let data = known_good_chunk[2..known_good_chunk.len() - 2].to_vec(); + + let mut chunk = Chunk::new_single(ChunkMessage::new( + crate::mar::ChunkMessageType::Event, + Cursor::new(data), + )); + + let mut buf: Cursor> = Cursor::new(vec![]); + assert_eq!( + copy(&mut chunk, &mut buf).expect("copy ok"), + known_good_chunk.len() as u64 + ); + assert_eq!(buf.get_ref().as_slice(), known_good_chunk); + } + + impl StreamLen for std::io::Cursor> { + fn stream_len(&self) -> u64 { + self.get_ref().len() as u64 + } + } +} diff --git a/memfaultd/src/mar/chunks/chunk_header.rs b/memfaultd/src/mar/chunks/chunk_header.rs new file mode 100644 index 0000000..49408b4 --- /dev/null +++ b/memfaultd/src/mar/chunks/chunk_header.rs @@ -0,0 +1,59 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +/// Chunk protocol supports two types of chunks: Init and Cont. +#[derive(Copy, Clone, PartialEq, Eq)] +pub enum ChunkType { + /// First chunk of a message (with `more_data` bit set if there is more data to follow). + Init = 0, + /// All following chunks of a message (with more_data bit set, except for the last one). + #[allow(dead_code)] + Cont = 0x80, +} + +/// Header of one Memfault Chunk +pub struct ChunkHeader { + channel_id: u8, + chunk_type: ChunkType, + more_data: bool, + + // Note: we only support single messages for now so header is always 1 byte. + // We can use Box<[u8]> when we want to support including the length/offset in the header + as_bytes: [u8; 1], +} + +impl ChunkHeader { + const CHUNK_CHANNEL: u8 = 0; + const CHUNK_CHANNEL_MASK: u8 = 0x7; + const CHUNK_MORE_DATA_BIT: u8 = 0x40; + const CHUNK_CRC_DEFERRED_BIT: u8 = 0x08; + + /// Returns a header for one message encoded as a single chunk with deferred CRC. + pub fn new_single() -> Self { + let mut header = Self { + channel_id: Self::CHUNK_CHANNEL, + chunk_type: ChunkType::Init, + more_data: false, + as_bytes: [0], + }; + header.calculate_header_bytes(); + header + } + + fn calculate_header_bytes(&mut self) { + let byte = &mut self.as_bytes[0]; + *byte = 0; + *byte |= self.channel_id & Self::CHUNK_CHANNEL_MASK; + if self.more_data { + *byte |= Self::CHUNK_MORE_DATA_BIT; + } + *byte |= self.chunk_type as u8; + *byte |= Self::CHUNK_CRC_DEFERRED_BIT; + } + + /// Return a byte representation of the header + /// length can vary from 1 to 5 bytes (one byte + one varint) + pub fn as_bytes(&self) -> &[u8] { + &self.as_bytes + } +} diff --git a/memfaultd/src/mar/chunks/chunk_message.rs b/memfaultd/src/mar/chunks/chunk_message.rs new file mode 100644 index 0000000..ebd8ebb --- /dev/null +++ b/memfaultd/src/mar/chunks/chunk_message.rs @@ -0,0 +1,45 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{Chain, Cursor, Read}; + +use crate::util::io::StreamLen; + +/// A one-byte discriminator for the type of message being sent. +#[derive(Copy, Clone, PartialEq, Eq)] +#[allow(dead_code)] +pub enum ChunkMessageType { + Null = 0, + McuCoredump = 1, + Event = 2, + Logs = 3, + CustomDataRecording = 4, + Mar = 5, +} + +/// All data sent in chunks must be wrapped in a ChunkMessage. +/// +/// It adds a one-byte header indicating the type of message being sent. +pub struct ChunkMessage { + stream: Chain, R>, +} + +impl ChunkMessage { + pub fn new(message_type: ChunkMessageType, data: R) -> Self { + Self { + stream: Cursor::new([message_type as u8]).chain(data), + } + } +} + +impl Read for ChunkMessage { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.stream.read(buf) + } +} + +impl StreamLen for ChunkMessage { + fn stream_len(&self) -> u64 { + self.stream.stream_len() + } +} diff --git a/memfaultd/src/mar/chunks/chunk_wrapper.rs b/memfaultd/src/mar/chunks/chunk_wrapper.rs new file mode 100644 index 0000000..0dcd2f0 --- /dev/null +++ b/memfaultd/src/mar/chunks/chunk_wrapper.rs @@ -0,0 +1,43 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{Chain, Cursor, Read}; + +use crate::util::io::StreamLen; + +/// Add a little wrapper around each chunk containing an identifier and the length. +/// This makes it possible to concatenate multiple chunks in one file. +/// +/// Format: +/// - 4 identifier bytes: 'C' 'H' 'N' 'K' +/// - Chunk length (4 bytes, big endian) +/// +/// /!\ This format is not accepted by memfault API. You have to remove that +/// wrapper before sending the chunks to Memfault! +pub struct ChunkWrapper { + stream: Chain>, R>, +} + +impl ChunkWrapper { + pub fn new(chunk: R) -> Self { + let mut header: [u8; 8] = [0; 8]; + header[0..4].copy_from_slice(b"CHNK"); + header[4..8].copy_from_slice(&(chunk.stream_len() as u32).to_be_bytes()); + + Self { + stream: Cursor::new(header.to_vec()).chain(chunk), + } + } +} + +impl Read for ChunkWrapper { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.stream.read(buf) + } +} + +impl StreamLen for ChunkWrapper { + fn stream_len(&self) -> u64 { + self.stream.stream_len() + } +} diff --git a/memfaultd/src/mar/chunks/crc_padded_stream.rs b/memfaultd/src/mar/chunks/crc_padded_stream.rs new file mode 100644 index 0000000..39c6284 --- /dev/null +++ b/memfaultd/src/mar/chunks/crc_padded_stream.rs @@ -0,0 +1,64 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{Cursor, Read}; + +use crc::{Crc, Digest}; +use crc_catalog::CRC_16_XMODEM; +use once_cell::sync::Lazy; + +use crate::util::io::StreamLen; + +static CRC16_XMODEM: Lazy> = Lazy::new(|| Crc::::new(&CRC_16_XMODEM)); +static CRC16_XMODEM_LENGTH: u64 = 2; + +/// A stream which will be followed by a CRC. +pub struct CRCPaddedStream { + stream: R, + /// Keep a running CRC as the data is being read. + /// Will be None once all the stream data has been read. + crc: Option>, + /// Remaining crc bytes to be read. + crc_bytes: Cursor>, +} + +impl CRCPaddedStream { + pub fn new(stream: R) -> Self { + Self { + stream, + crc: Some(CRC16_XMODEM.digest()), + crc_bytes: Cursor::new(vec![]), + } + } +} + +impl Read for CRCPaddedStream { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let result = self.stream.read(buf)?; + + if result > 0 { + if let Some(crc) = &mut self.crc { + crc.update(&buf[..result]); + } else { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + "CRC already finalized", + )); + } + return Ok(result); + } + + // When done reading, write the CRC. + if let Some(crc) = self.crc.take() { + self.crc_bytes = Cursor::new(crc.finalize().to_le_bytes().to_vec()); + } + + self.crc_bytes.read(buf) + } +} + +impl StreamLen for CRCPaddedStream { + fn stream_len(&self) -> u64 { + self.stream.stream_len() + CRC16_XMODEM_LENGTH + } +} diff --git a/memfaultd/src/mar/clean.rs b/memfaultd/src/mar/clean.rs new file mode 100644 index 0000000..25fc279 --- /dev/null +++ b/memfaultd/src/mar/clean.rs @@ -0,0 +1,1072 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::{ + mar::MarEntry, + util::disk_size::{get_disk_space, get_size, DiskSize}, +}; +use eyre::{eyre, Result, WrapErr}; +use log::{debug, trace, warn}; +use std::fmt::{Display, Formatter, Result as FmtResult}; +use std::fs::read_dir; +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime}; + +pub struct MarStagingCleaner { + mar_staging_path: PathBuf, + max_total_size: DiskSize, + min_headroom: DiskSize, + max_age: Duration, +} + +impl MarStagingCleaner { + pub fn new( + mar_staging_path: &Path, + max_total_size: DiskSize, + min_headroom: DiskSize, + max_age: Duration, + ) -> Self { + Self { + mar_staging_path: mar_staging_path.to_owned(), + max_total_size, + min_headroom, + max_age, + } + } + + /// Cleans up MAR entries in the staging folder. + /// Returns the amount of space left until the max_total_size will be reached or until + /// min_headroom will be exceeded (the smallest of the two values). + pub fn clean(&self, required_space: DiskSize) -> Result { + trace!("Cleaning MAR staging area..."); + clean_mar_staging( + &self.mar_staging_path, + self.max_total_size.saturating_sub(required_space), + get_disk_space(&self.mar_staging_path).unwrap_or(DiskSize::ZERO), + self.min_headroom + required_space, + SystemTime::now(), + self.max_age, + ) + .map_err(|e| { + warn!("Unable to clean MAR entries: {}", e); + e + }) + } +} + +#[derive(Debug)] +enum DeletionReason { + Expired, + DiskQuota, +} +impl Display for DeletionReason { + fn fmt(&self, f: &mut Formatter) -> FmtResult { + match self { + DeletionReason::Expired => write!(f, "Expired"), + DeletionReason::DiskQuota => write!(f, "Disk quota"), + } + } +} + +#[derive(Debug, Clone)] +struct AgeSizePath { + age: Duration, + size: DiskSize, + path: PathBuf, +} + +impl AgeSizePath { + fn new( + path: &Path, + size: DiskSize, + timestamp: SystemTime, + reference_date: SystemTime, + ) -> AgeSizePath { + Self { + age: (reference_date.duration_since(timestamp)).unwrap_or(Duration::ZERO), + size, + path: path.to_owned(), + } + } +} + +fn clean_mar_staging( + mar_staging: &Path, + max_total_size: DiskSize, + available_space: DiskSize, + min_space: DiskSize, + reference_date: SystemTime, + max_age: Duration, +) -> Result { + let (entries, total_space_used) = collect_mar_entries(mar_staging, reference_date)?; + + let marked_entries = mark_entries_for_deletion( + entries, + total_space_used, + max_total_size, + available_space, + min_space, + max_age, + ); + + let space_freed = remove_marked_entries(marked_entries); + + let remaining_quota = + max_total_size.saturating_sub(total_space_used.saturating_sub(space_freed)); + let usable_space = (available_space + space_freed).saturating_sub(min_space); + + // Available space to write is the min of bytes and inodes remaining. + Ok(DiskSize::min(remaining_quota, usable_space)) +} + +fn collect_mar_entries( + mar_staging: &Path, + reference_date: SystemTime, +) -> Result<(Vec, DiskSize)> { + let entries: Vec = read_dir(mar_staging) + .wrap_err(eyre!( + "Unable to open MAR staging area: {}", + mar_staging.display() + ))? + .filter_map(|r| r.map_err(|e| warn!("Unable to read DirEntry: {}", e)).ok()) + .map(|dir_entry| match MarEntry::from_path(dir_entry.path()) { + // Use the collection time from the manifest or folder creation time if the manifest cannot be parsed: + Ok(entry) => AgeSizePath::new( + &entry.path, + get_size(&entry.path).unwrap_or(DiskSize::ZERO), + entry.manifest.collection_time.timestamp.into(), + reference_date, + ), + Err(_) => { + let path = dir_entry.path(); + let timestamp = path + .metadata() + .and_then(|m| m.created()) + .unwrap_or_else(|_| SystemTime::now()); + AgeSizePath::new( + &path, + get_size(&path).unwrap_or(DiskSize::ZERO), + timestamp, + reference_date, + ) + } + }) + .collect(); + let total_space_used = entries + .iter() + .fold(DiskSize::ZERO, |space_used, entry| entry.size + space_used); + Ok((entries, total_space_used)) +} + +fn mark_entries_for_deletion( + entries: Vec, + total_space_used: DiskSize, + max_total_size: DiskSize, + available_space: DiskSize, + min_space: DiskSize, + max_age: Duration, +) -> Vec<(AgeSizePath, DeletionReason)> { + // Sort entries with oldest first + let mut entries_by_age = entries; + entries_by_age.sort_by(|a, b| b.age.cmp(&a.age)); + + // Note that (total_space_used).exceeds(max_total_size) would not + // work here because it would be false when only one of bytes/inode is + // exceeded. (we have a test to verify this) + let max_total_size_exceeded = !max_total_size.exceeds(&(total_space_used)); + // Same here, min_space.exceeds(available_space) is sufficient but not necessary. + let min_headroom_exceeded = !available_space.exceeds(&min_space); + + // Calculate how much space needs to be freed based on quotas + let need_to_free = match (max_total_size_exceeded, min_headroom_exceeded) { + (false, false) => DiskSize::ZERO, + (true, false) => total_space_used.saturating_sub(max_total_size), + (false, true) => min_space.saturating_sub(available_space), + // If we have exceeded both quotas, need to free the larger difference + // for both inodes and bytes + (true, true) => DiskSize::max( + total_space_used.saturating_sub(max_total_size), + min_space.saturating_sub(available_space), + ), + }; + + let mut space_to_be_freed = DiskSize::ZERO; + + // Ignore max_age if it is configured to 0 + let delete_expired_entries = !max_age.is_zero(); + + // Since the vector is sorted from oldest to newest, + // older entries will be marked for deletion first + entries_by_age + .into_iter() + .filter_map(|entry| { + if need_to_free != DiskSize::ZERO && !space_to_be_freed.exceeds(&need_to_free) { + space_to_be_freed += entry.size; + Some((entry, DeletionReason::DiskQuota)) + } else if delete_expired_entries && entry.age > max_age { + space_to_be_freed += entry.size; + Some((entry, DeletionReason::Expired)) + } else { + None + } + }) + .collect() +} + +fn remove_marked_entries(marked_entries: Vec<(AgeSizePath, DeletionReason)>) -> DiskSize { + let mut space_freed = DiskSize::ZERO; + for (entry, deletion_reason) in marked_entries { + debug!( + "Cleaning up MAR entry: {} ({} bytes / {} inodes, ~{} seconds old). Deletion reason: {}", + entry.path.display(), + entry.size.bytes, + entry.size.inodes, + entry.age.as_secs(), + deletion_reason + ); + if let Err(e) = std::fs::remove_dir_all(&entry.path) { + warn!("Unable to remove MAR entry: {}", e); + } else { + debug!( + "Removed MAR entry: {} {:?}", + entry.path.display(), + entry.size + ); + space_freed += entry.size; + } + } + space_freed +} + +#[cfg(test)] +mod test { + use crate::mar::test_utils::MarCollectorFixture; + use crate::test_utils::create_file_with_size; + use crate::test_utils::setup_logger; + use rstest::{fixture, rstest}; + + use super::*; + #[rstest] + fn test_collect_entries(mut mar_fixture: MarCollectorFixture) { + let now = SystemTime::now(); + + let path_a = mar_fixture.create_logentry_with_size_and_age(100, now); + let path_b = + mar_fixture.create_logentry_with_size_and_age(100, now + Duration::from_secs(5)); + let path_c = + mar_fixture.create_logentry_with_size_and_age(100, now + Duration::from_secs(10)); + let (entries, _total_space_used) = + collect_mar_entries(&mar_fixture.mar_staging, SystemTime::now()).unwrap(); + assert!(entries.len() == 3); + assert!([path_a, path_b, path_c].iter().all(|path| entries + .iter() + .any(|entry| entry.path.to_str() == path.to_str()))); + } + + #[rstest] + fn test_no_delete_reasons_when_within_quota() { + let now = SystemTime::now(); + + let max_total_size = DiskSize::new_capacity(25000); + let min_headroom = DiskSize { + bytes: 20000, + inodes: 20, + }; + let available_space = DiskSize { + bytes: min_headroom.bytes, + inodes: 100, + }; + + let entries = vec![ + AgeSizePath::new( + Path::new("/mock/a"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now + Duration::from_secs(1), + now, + ), + AgeSizePath::new( + Path::new("/mock/b"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now + Duration::from_secs(10), + now, + ), + AgeSizePath::new( + Path::new("/mock/c"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now + Duration::from_secs(15), + now, + ), + ]; + + let total_space_used = entries + .iter() + .fold(DiskSize::ZERO, |space_used, entry| entry.size + space_used); + + let marked_entries = mark_entries_for_deletion( + entries, + total_space_used, + max_total_size, + available_space, + min_headroom, + Duration::from_secs(0), // No max age for this test + ); + + let do_not_deletes = ["/mock/a", "/mock/b"]; + + assert!(marked_entries.iter().all(|(entry, _reason)| !do_not_deletes + .iter() + .any(|&do_not_delete_path| entry.path.to_str().unwrap() == do_not_delete_path))); + } + + #[rstest] + #[case(DiskSize::new_capacity(2500))] // 1 entry over quota by bytes + #[case(DiskSize {bytes: 10000, inodes: 4})] // 1 entry over quota by inodes + fn test_oldest_marked_when_over_max_total_size(#[case] max_total_size: DiskSize) { + let now = SystemTime::now(); + let entries = vec![ + AgeSizePath::new( + Path::new("/mock/c"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(15), + now, + ), + AgeSizePath::new( + Path::new("/mock/a"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(1), + now, + ), + AgeSizePath::new( + Path::new("/mock/b"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(10), + now, + ), + ]; + + let total_space_used = entries + .iter() + .fold(DiskSize::ZERO, |space_used, entry| entry.size + space_used); + let marked_entries = mark_entries_for_deletion( + entries, + total_space_used, + max_total_size, + DiskSize::new_capacity(10000), + DiskSize::ZERO, // No min headroom quota for this test + Duration::from_secs(0), // No max age for this test + ); + + let do_not_deletes = ["/mock/a", "/mock/b"]; + + assert!(marked_entries.iter().all(|(entry, _reason)| !do_not_deletes + .iter() + .any(|&do_not_delete_path| entry.path.to_str().unwrap() == do_not_delete_path))); + + for (entry, reason) in marked_entries { + if entry.path.to_str().unwrap() == "/mock/c" { + assert!(matches!(reason, DeletionReason::DiskQuota)); + } + } + } + + #[rstest] + #[case(DiskSize {bytes: 3000, inodes: 100}, DiskSize::new_capacity(1500))] // 2 entries over quota by bytes + #[case(DiskSize {bytes: 10000, inodes: 4}, DiskSize {bytes: 10000, inodes: 0})] // 2 entries over quota by inodes + fn test_two_oldest_marked_when_under_min_headroom( + #[case] min_headroom: DiskSize, + #[case] available_space: DiskSize, + ) { + let now = SystemTime::now(); + let entries = vec![ + AgeSizePath::new( + Path::new("/mock/c"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(15), + now, + ), + AgeSizePath::new( + Path::new("/mock/a"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now, + now - Duration::from_secs(1), + ), + AgeSizePath::new( + Path::new("/mock/b"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(10), + now, + ), + ]; + + let total_space_used = entries + .iter() + .fold(DiskSize::ZERO, |space_used, entry| entry.size + space_used); + + let marked_entries = mark_entries_for_deletion( + entries, + total_space_used, + DiskSize::new_capacity(10000), + available_space, + min_headroom, + Duration::from_secs(0), // No max age for this test + ); + + assert!(marked_entries + .iter() + .all(|(entry, _reason)| entry.path.to_str().unwrap() != "/mock/a")); + + for (entry, reason) in marked_entries { + match entry.path.to_str().unwrap() { + "/mock/c" => assert!(matches!(reason, DeletionReason::DiskQuota)), + "/mock/b" => assert!(matches!(reason, DeletionReason::DiskQuota)), + _ => unreachable!(), + } + } + } + + #[rstest] + fn expired_entries_marked() { + let now = SystemTime::now(); + let entries = vec![ + AgeSizePath::new( + Path::new("/mock/c"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(15), + now, + ), + AgeSizePath::new( + Path::new("/mock/a"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(5), + now, + ), + AgeSizePath::new( + Path::new("/mock/b"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(10), + now, + ), + ]; + + let total_space_used = entries + .iter() + .fold(DiskSize::ZERO, |space_used, entry| entry.size + space_used); + let marked_entries = mark_entries_for_deletion( + entries, + total_space_used, + DiskSize::new_capacity(10000), + DiskSize::new_capacity(10000), + DiskSize::ZERO, // No min headroom quota for this test + Duration::from_secs(1), // low max age so all entries are marked as expired + ); + + assert!(marked_entries + .iter() + .all(|(_entry, reason)| matches!(reason, DeletionReason::Expired))); + } + + #[rstest] + // 2 oldest entries need to be deleted to get under quota + // 3rd oldest needs to be deleted because it is expired + // 2 most recent should not be deleted + fn test_marks_quota_and_expired_entries() { + let now = SystemTime::now(); + let max_total_size = DiskSize::new_capacity(3000); + + let entries = vec![ + AgeSizePath::new( + Path::new("/mock/c"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(200), + now, + ), + AgeSizePath::new( + Path::new("/mock/d"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(250), + now, + ), + AgeSizePath::new( + Path::new("/mock/e"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(300), + now, + ), + AgeSizePath::new( + Path::new("/mock/a"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(15), + now, + ), + AgeSizePath::new( + Path::new("/mock/b"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(30), + now, + ), + ]; + + let total_space_used = entries + .iter() + .fold(DiskSize::ZERO, |space_used, entry| entry.size + space_used); + let marked_entries = mark_entries_for_deletion( + entries, + total_space_used, + max_total_size, + DiskSize::new_capacity(10000), + DiskSize::ZERO, + Duration::from_secs(100), + ); + + let do_not_deletes = ["/mock/a", "/mock/b"]; + + assert!(marked_entries.iter().all(|(entry, _reason)| !do_not_deletes + .iter() + .any(|&do_not_delete_path| entry.path.to_str().unwrap() == do_not_delete_path))); + + for (entry, reason) in marked_entries { + match entry.path.to_str().unwrap() { + "/mock/c" => assert!(matches!(reason, DeletionReason::Expired)), + "/mock/d" => assert!(matches!(reason, DeletionReason::DiskQuota)), + "/mock/e" => assert!(matches!(reason, DeletionReason::DiskQuota)), + _ => unreachable!(), + } + } + } + + #[rstest] + // Over the max total space used quota by 2000 bytes + // and under the minimum headroom quota by 6 inodes + // Need to delete 3 entries altogether + fn test_marks_when_different_quotas_exceeded() { + let now = SystemTime::now(); + let max_total_size = DiskSize::new_capacity(3000); + let min_headroom = DiskSize { + bytes: 1024, + inodes: 10, + }; + let available_space = DiskSize { + bytes: 10000, + inodes: 6, + }; + + // 2 oldest entries need to be deleted to get under quota + // 3rd oldest needs to be deleted because it is expired + // 2 most recent should not be deleted + let entries = vec![ + AgeSizePath::new( + Path::new("/mock/c"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(200), + now, + ), + AgeSizePath::new( + Path::new("/mock/d"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(250), + now, + ), + AgeSizePath::new( + Path::new("/mock/e"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(300), + now, + ), + AgeSizePath::new( + Path::new("/mock/a"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(15), + now, + ), + AgeSizePath::new( + Path::new("/mock/b"), + DiskSize { + bytes: 1000, + inodes: 2, + }, + now - Duration::from_secs(30), + now, + ), + ]; + + let total_space_used = entries + .iter() + .fold(DiskSize::ZERO, |space_used, entry| entry.size + space_used); + let marked_entries = mark_entries_for_deletion( + entries, + total_space_used, + max_total_size, + available_space, + min_headroom, + Duration::from_secs(0), // No max age in this test + ); + + let do_not_deletes = ["/mock/a", "/mock/b"]; + + assert!(marked_entries.iter().all(|(entry, _reason)| !do_not_deletes + .iter() + .any(|&do_not_delete_path| entry.path.to_str().unwrap() == do_not_delete_path))); + for (entry, reason) in marked_entries { + match entry.path.to_str().unwrap() { + "/mock/c" => assert!(matches!(reason, DeletionReason::DiskQuota)), + "/mock/d" => assert!(matches!(reason, DeletionReason::DiskQuota)), + "/mock/e" => assert!(matches!(reason, DeletionReason::DiskQuota)), + _ => unreachable!(), + } + } + } + + #[rstest] + fn empty_staging_area( + mar_fixture: MarCollectorFixture, + max_total_size: DiskSize, + available_space: DiskSize, + min_headroom: DiskSize, + ) { + let size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + SystemTime::now(), + Duration::from_secs(604800), + ) + .unwrap(); + assert_eq!( + size_avail, + DiskSize::min(max_total_size, available_space.saturating_sub(min_headroom)) + ); + } + + #[rstest] + fn keeps_recent_unfinished_mar_entry( + mut mar_fixture: MarCollectorFixture, + max_total_size: DiskSize, + available_space: DiskSize, + min_headroom: DiskSize, + ) { + let path = mar_fixture.create_empty_entry(); + let size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + SystemTime::now(), + Duration::from_secs(604800), + ) + .unwrap(); + assert_eq!( + size_avail, + DiskSize::min(max_total_size, available_space.saturating_sub(min_headroom)) + ); + assert!(path.exists()); + } + + #[rstest] + fn removes_unfinished_mar_entry_exceeding_max_total_size( + mut mar_fixture: MarCollectorFixture, + max_total_size: DiskSize, + available_space: DiskSize, + min_headroom: DiskSize, + ) { + let path = mar_fixture.create_empty_entry(); + + create_file_with_size(&path.join("log.txt"), max_total_size.bytes + 1).unwrap(); + let size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space.saturating_sub(DiskSize { + bytes: 0, + inodes: 1, + }), + min_headroom, + SystemTime::now(), + Duration::from_secs(604800), + ) + .unwrap(); + assert_eq!( + size_avail, + DiskSize::min(max_total_size, available_space.saturating_sub(min_headroom)) + ); + assert!(!path.exists()); + } + + #[rstest] + fn keeps_recent_mar_entry( + mut mar_fixture: MarCollectorFixture, + max_total_size: DiskSize, + available_space: DiskSize, + min_headroom: DiskSize, + ) { + let now = SystemTime::now(); + let path = mar_fixture.create_logentry_with_size_and_age(1, now); + let size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + now, + Duration::from_secs(604800), + ) + .unwrap(); + assert!(max_total_size.exceeds(&size_avail)); + assert!(path.exists()); + } + + #[rstest] + fn removes_mar_entry_exceeding_max_total_size( + mut mar_fixture: MarCollectorFixture, + max_total_size: DiskSize, + available_space: DiskSize, + min_headroom: DiskSize, + ) { + let now = SystemTime::now(); + // NOTE: the entire directory will be larger than max_total_size due to the manifest.json. + let path = mar_fixture.create_logentry_with_size_and_age(max_total_size.bytes, now); + let size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space.saturating_sub(DiskSize { + bytes: 0, + inodes: 2, + }), + min_headroom, + now, + Duration::from_secs(604800), + ) + .unwrap(); + assert_eq!( + size_avail, + DiskSize::min(max_total_size, available_space.saturating_sub(min_headroom)) + ); + assert!(!path.exists()); + } + + #[rstest] + fn removes_mar_entry_exceeding_min_headroom(mut mar_fixture: MarCollectorFixture) { + let now = SystemTime::now(); + let max_total_size = DiskSize::new_capacity(4096); + let min_headroom = DiskSize { + bytes: 1024, + inodes: 10, + }; + let available_space = DiskSize { + bytes: min_headroom.bytes - 1, + inodes: 100, + }; + // NOTE: the entire directory will be larger than 1 byte due to the manifest.json. + let path = mar_fixture.create_logentry_with_size_and_age(1, now); + let size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + now, + Duration::from_secs(604800), + ) + .unwrap(); + assert!(size_avail.bytes >= 1); + assert!(!path.exists()); + } + + #[rstest] + fn removes_oldest_mar_entry_exceeding_max_total_size_when_multiple( + mut mar_fixture: MarCollectorFixture, + ) { + let now = SystemTime::now(); + let max_total_size = DiskSize::new_capacity(23000); + let min_headroom = DiskSize { + bytes: 1024, + inodes: 10, + }; + let available_space = DiskSize { + bytes: max_total_size.bytes * 2, + inodes: 100, + }; + let oldest = + mar_fixture.create_logentry_with_size_and_age(8000, now - Duration::from_secs(120)); + let middle = + mar_fixture.create_logentry_with_size_and_age(8000, now - Duration::from_secs(30)); + let most_recent = mar_fixture.create_logentry_with_size_and_age(8000, now); + let _size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + now, + Duration::from_secs(604800), + ) + .unwrap(); + assert!(!oldest.exists()); + assert!(middle.exists()); + assert!(most_recent.exists()); + } + + #[rstest] + fn removes_entries_exceeding_min_headroom_size_by_age( + mut mar_fixture: MarCollectorFixture, + _setup_logger: (), + ) { + let now = SystemTime::now(); + let max_total_size = DiskSize::new_capacity(80000); + let min_headroom = DiskSize { + bytes: 20000, + inodes: 20, + }; + let available_space = DiskSize { + bytes: min_headroom.bytes - 20000, + inodes: 100, + }; + let oldest = + mar_fixture.create_logentry_with_size_and_age(10000, now - Duration::from_secs(120)); + let second_oldest = + mar_fixture.create_logentry_with_size_and_age(10000, now - Duration::from_secs(30)); + let second_newest = + mar_fixture.create_logentry_with_size_and_age(10000, now - Duration::from_secs(10)); + let most_recent = mar_fixture.create_logentry_with_size_and_age(10000, now); + + // Need to delete 2 entries to free up required headroom + let _size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + now, + Duration::from_secs(604800), + ) + .unwrap(); + assert!(most_recent.exists()); + assert!(second_newest.exists()); + assert!(!second_oldest.exists()); + assert!(!oldest.exists()); + } + + #[rstest] + fn removes_entries_exceeding_max_total_size_by_age( + mut mar_fixture: MarCollectorFixture, + _setup_logger: (), + ) { + let now = SystemTime::now(); + let max_total_size = DiskSize::new_capacity(25000); + let min_headroom = DiskSize { + bytes: 1024, + inodes: 20, + }; + let available_space = DiskSize { + bytes: max_total_size.bytes * 2, + inodes: 100, + }; + let oldest = + mar_fixture.create_logentry_with_size_and_age(10000, now - Duration::from_secs(120)); + let second_oldest = + mar_fixture.create_logentry_with_size_and_age(10000, now - Duration::from_secs(30)); + let second_newest = + mar_fixture.create_logentry_with_size_and_age(10000, now - Duration::from_secs(10)); + let most_recent = mar_fixture.create_logentry_with_size_and_age(10000, now); + let _size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + now, + Duration::from_secs(604800), + ) + .unwrap(); + assert!(second_newest.exists()); + assert!(most_recent.exists()); + assert!(!oldest.exists()); + assert!(!second_oldest.exists()); + } + + #[rstest] + fn removes_mar_entry_exceeding_min_headroom_inodes( + _setup_logger: (), + mut mar_fixture: MarCollectorFixture, + ) { + let now = SystemTime::now(); + let max_total_size = DiskSize::new_capacity(10 * 1024 * 1024); + let min_headroom = DiskSize { + bytes: 1024, + inodes: 10, + }; + let available_space = DiskSize { + bytes: max_total_size.bytes, + inodes: 5, + }; + // NOTE: the entire directory will be larger than 1 byte due to the manifest.json. + let path = mar_fixture.create_logentry_with_size_and_age(1, now); + let size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + now, + Duration::from_secs(604800), + ) + .unwrap(); + assert!(size_avail.bytes >= 1); + assert!(!path.exists()); + } + + #[rstest] + fn removes_mar_entry_exceeding_max_age( + mut mar_fixture: MarCollectorFixture, + max_total_size: DiskSize, + available_space: DiskSize, + min_headroom: DiskSize, + ) { + let now = SystemTime::now(); + let thirty_seconds_ago = now - Duration::from_secs(30); + let path_unexpired = mar_fixture.create_logentry_with_size_and_age(1, thirty_seconds_ago); + let ten_min_ago = now - Duration::from_secs(600); + let path_expired = mar_fixture.create_logentry_with_size_and_age(1, ten_min_ago); + + let size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + now, + Duration::from_secs(60), + ) + .unwrap(); + assert!(size_avail.bytes >= 1); + assert!(path_unexpired.exists()); + assert!(!path_expired.exists()); + } + + #[rstest] + fn keeps_mar_entry_within_max_age( + mut mar_fixture: MarCollectorFixture, + max_total_size: DiskSize, + available_space: DiskSize, + min_headroom: DiskSize, + ) { + let now = SystemTime::now(); + let thirty_seconds_ago = now - Duration::from_secs(30); + let path = mar_fixture.create_logentry_with_size_and_age(1, thirty_seconds_ago); + let size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + now, + Duration::from_secs(60), + ) + .unwrap(); + assert!(size_avail.bytes >= 1); + assert!(path.exists()); + } + + #[rstest] + fn keeps_mar_entry_when_max_age_is_zero( + mut mar_fixture: MarCollectorFixture, + max_total_size: DiskSize, + available_space: DiskSize, + min_headroom: DiskSize, + ) { + let now = SystemTime::now(); + let over_one_week_ago = now - Duration::from_secs(604801); + let path = mar_fixture.create_logentry_with_size_and_age(1, over_one_week_ago); + let size_avail = clean_mar_staging( + &mar_fixture.mar_staging, + max_total_size, + available_space, + min_headroom, + now, + Duration::from_secs(0), + ) + .unwrap(); + assert!(size_avail.bytes >= 1); + assert!(path.exists()); + } + + #[fixture] + fn max_total_size() -> DiskSize { + DiskSize::new_capacity(1024) + } + + #[fixture] + fn available_space() -> DiskSize { + DiskSize { + bytes: u64::MAX / 2, + inodes: u64::MAX / 2, + } + } + + #[fixture] + fn min_headroom() -> DiskSize { + DiskSize { + bytes: 0, + inodes: 0, + } + } + + #[fixture] + fn mar_fixture() -> MarCollectorFixture { + MarCollectorFixture::new() + } +} diff --git a/memfaultd/src/mar/export.rs b/memfaultd/src/mar/export.rs new file mode 100644 index 0000000..80334d7 --- /dev/null +++ b/memfaultd/src/mar/export.rs @@ -0,0 +1,421 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use crate::{ + http_server::{ConvenientHeader, HttpHandler, HttpHandlerResult}, + mar::{ + gather_mar_entries_to_zip, Chunk, ChunkMessage, ChunkWrapper, ExportFormat, MarEntry, + MarZipContents, + }, + util::{io::StreamLen, zip::ZipEncoder}, +}; +use eyre::{eyre, Result}; +use log::{debug, trace, warn}; +use std::{ + collections::hash_map::DefaultHasher, + fs::remove_dir_all, + io::BufReader, + path::PathBuf, + sync::{Arc, Mutex}, +}; +use std::{hash::Hasher, os::unix::prelude::OsStrExt}; +use tiny_http::{Header, Method, Request, Response, ResponseBox, StatusCode}; + +pub const EXPORT_MAR_URL: &str = "/v1/export"; + +/// Information on the most recent export that was proposed to a client. +/// We will keep offering the same content until we receive a DELETE call. +struct ExportInfo { + content: MarZipContents, + hash: String, +} +impl ExportInfo { + fn new(content: MarZipContents) -> Self { + let mut hasher = DefaultHasher::new(); + for p in content.entry_paths.iter() { + hasher.write_usize(p.as_os_str().len()); + hasher.write(p.as_os_str().as_bytes()); + } + let hash = hasher.finish().to_string(); + ExportInfo { content, hash } + } + + /// Verifies if the files are still available on disk + fn is_valid(&self) -> bool { + self.content.entry_paths.iter().all(|p| p.exists()) + } +} + +#[derive(Clone)] +/// An `HttpHandler` to manage exporting memfaultd data to local clients. +/// +/// Clients should first make a GET request to download the data and write it to +/// disk. When the file has safely transmitted, they can all DELETE to delete +/// the data secured in the MAR staging directory. +/// +/// For additional security (if multiple clients are accidentally +/// reading/deleting concurrently), we recommend using the If-Match header when +/// calling DELETE and passing the ETag returned by the GET call. This will +/// confirm that the data being deleted is the data that was just saved. +pub struct MarExportHandler { + mar_staging: PathBuf, + current_export: Arc>>, +} + +impl MarExportHandler { + pub fn new(mar_staging: PathBuf) -> Self { + MarExportHandler { + mar_staging, + current_export: Arc::new(Mutex::new(None)), + } + } +} + +const DEFAULT_MAX_ZIP_FILE: usize = 10 * 1024 * 1024; + +impl MarExportHandler { + /// Looks at data in the mar_staging folder and returns content that should be included in next ZIP download + fn prepare_next_export(&self) -> Result> { + let mut entries = MarEntry::iterate_from_container(&self.mar_staging)?; + + let zip_files = gather_mar_entries_to_zip(&mut entries, DEFAULT_MAX_ZIP_FILE); + match zip_files.into_iter().next() { + Some(zip) => Ok(Some(ExportInfo::new(zip))), + None => Ok(None), + } + } + + fn handle_get_mar(&self, request: &Request) -> Result { + let mut export = self + .current_export + .lock() + .map_err(|e| eyre!("Export Mutex poisoned: {:#}", e))?; + + // If we have already prepared a package, make sure the file still exists on disk - or reset the package. + if let Some(false) = (export.as_ref()).map(|export| export.is_valid()) { + *export = None; + } + + // Prepare a new package if needed + if export.is_none() { + *export = self.prepare_next_export()?; + } + + // Parse the accept-header, return 406 NotAcceptable if the requested format is not supported. + let accept_header = request.headers().iter().find(|h| h.field.equiv("Accept")); + let format = match accept_header { + Some(header) => match ExportFormat::from_accept_header(header.value.as_str()) { + Ok(format) => format, + Err(_) => return Ok(Response::empty(406).boxed()), + }, + None => ExportFormat::default(), + }; + + // If we have data to serve, prime the ZIP encoder and stream the data + // Otherwise, return 204. + match &*export { + Some(export) => match format { + ExportFormat::Mar => Self::export_as_zip(export), + ExportFormat::Chunk => Self::export_as_chunk(export), + ExportFormat::ChunkWrapped => Self::export_as_chunk_wrapped(export), + }, + None => Ok(Response::empty(204).boxed()), + } + } + + fn export_as_zip(export: &ExportInfo) -> Result { + let zip_encoder = ZipEncoder::new(export.content.zip_infos.clone()); + let len = zip_encoder.stream_len(); + + Ok(Response::new( + StatusCode(200), + vec![ + Header::from_strings("Content-Type", "application/zip")?, + Header::from_strings("ETag", &format!("\"{}\"", export.hash))?, + ], + BufReader::new(zip_encoder), + Some(len as usize), + None, + ) + .boxed()) + } + + fn export_as_chunk(export: &ExportInfo) -> Result { + let zip_encoder = ZipEncoder::new(export.content.zip_infos.clone()); + + let chunk_stream = Chunk::new_single(ChunkMessage::new( + super::chunks::ChunkMessageType::Mar, + zip_encoder, + )); + + let len = chunk_stream.stream_len(); + + Ok(Response::new( + StatusCode(200), + vec![ + Header::from_strings("Content-Type", ExportFormat::Chunk.to_content_type())?, + Header::from_strings("ETag", &format!("\"{}\"", export.hash))?, + ], + BufReader::new(chunk_stream), + Some(len as usize), + None, + ) + .boxed()) + } + + fn export_as_chunk_wrapped(export: &ExportInfo) -> Result { + let zip_encoder = ZipEncoder::new(export.content.zip_infos.clone()); + + let chunk = ChunkWrapper::new(Chunk::new_single(ChunkMessage::new( + super::chunks::ChunkMessageType::Mar, + zip_encoder, + ))); + let len = chunk.stream_len(); + + Ok(Response::new( + StatusCode(200), + vec![ + Header::from_strings("Content-Type", ExportFormat::Chunk.to_content_type())?, + Header::from_strings("ETag", &format!("\"{}\"", export.hash))?, + ], + BufReader::new(chunk), + Some(len as usize), + None, + ) + .boxed()) + } + + fn handle_delete(&self, request: &Request) -> Result { + let mut export_opt = self + .current_export + .lock() + .map_err(|e| eyre!("Mutex poisoned: {:#}", e))?; + + if let Some(export) = export_opt.as_ref() { + // Optionally, check that the ETag matches (to confirm we are deleting the data client just read). + if let Some(if_match_header) = + request.headers().iter().find(|h| h.field.equiv("If-Match")) + { + if if_match_header.value != export.hash { + debug!( + "Delete error - Wrong hash. Got {}, expected {}", + if_match_header.value, export.hash + ); + return Ok(Response::from_string("Precondition Failed") + .with_status_code(412) + .boxed()); + } + } + + trace!("Deleting MAR entries: {:?}", export.content.entry_paths); + export.content.entry_paths.iter().for_each(|f| { + if let Err(e) = remove_dir_all(f) { + warn!("Error deleting MAR entry: {} ({})", f.display(), e) + } + }); + *export_opt = None; + Ok(Response::empty(204).boxed()) + } else { + trace!("Export delete called but no current content to delete."); + Ok(Response::empty(404).boxed()) + } + } +} + +impl HttpHandler for MarExportHandler { + // TODO: MFLT-11507 Handle locking the mar_cleaner while we are reading it! + fn handle_request(&self, request: &mut Request) -> HttpHandlerResult { + if request.url() == EXPORT_MAR_URL { + match *request.method() { + Method::Get => self.handle_get_mar(request).into(), + Method::Delete => self.handle_delete(request).into(), + _ => HttpHandlerResult::Response(Response::empty(405).boxed()), + } + } else { + HttpHandlerResult::NotHandled + } + } +} + +#[cfg(test)] +mod tests { + use std::{fs::remove_dir_all, str::FromStr}; + + use rstest::{fixture, rstest}; + use tiny_http::{Header, ResponseBox, StatusCode, TestRequest}; + + use crate::{ + http_server::HttpHandler, mar::test_utils::MarCollectorFixture, util::disk_size::get_size, + }; + + use super::{MarExportHandler, EXPORT_MAR_URL}; + + #[rstest] + fn answer_204_when_empty(mut fixture: Fixture) { + let r = fixture.do_download(); + assert_eq!(r.status_code(), StatusCode(204)); + assert_eq!(r.etag(), None); + } + + #[rstest] + fn download_zip(mut fixture: Fixture) { + fixture.mar_fixture.create_logentry_with_size(512); + + let r = fixture.do_download(); + assert_eq!(r.status_code(), StatusCode(200)); + + r.etag().expect("e-tag header should be included"); + + // Files should still be there. + assert!(fixture.count_mar_inodes() > 0); + } + + #[rstest] + fn download_twice(mut fixture: Fixture) { + fixture.mar_fixture.create_logentry_with_size(512); + + let r = fixture.do_download(); + assert_eq!(r.status_code(), StatusCode(200)); + + // Another GET should yield the same response - even if we have added files in between + fixture.mar_fixture.create_logentry_with_size(1024); + let r2 = fixture.do_download(); + + assert_eq!(r2.status_code(), StatusCode(200)); + assert_eq!(r.data_length().unwrap(), r2.data_length().unwrap()); + assert_eq!(r.etag().unwrap(), r2.etag().unwrap()); + } + + #[rstest] + fn download_reset_on_cleanup(mut fixture: Fixture) { + let log1 = fixture.mar_fixture.create_logentry_with_size(512); + + let r = fixture.do_download(); + assert_eq!(r.status_code(), StatusCode(200)); + + // Simulate mar cleaner removing some files + remove_dir_all(log1).expect("delete failed"); + + // Another GET should yield a new response (because the old files are not available anymore) + fixture.mar_fixture.create_logentry_with_size(1024); + let r2 = fixture.do_download(); + + assert_eq!(r2.status_code(), StatusCode(200)); + assert_ne!(r.data_length().unwrap(), r2.data_length().unwrap()); + assert_ne!(r.etag().unwrap(), r2.etag().unwrap()); + } + + #[rstest] + fn files_should_be_deleted_with_etag(mut fixture: Fixture) { + fixture.mar_fixture.create_logentry_with_size(512); + + let r = fixture.do_download(); + assert_eq!(r.status_code(), StatusCode(200)); + + let delete_response = fixture.do_delete(Some(r.etag().unwrap())); + assert_eq!(delete_response.status_code(), StatusCode(204)); + + // Files should have been deleted. + assert_eq!(fixture.count_mar_inodes(), 0); + } + + #[rstest] + fn files_should_be_deleted_without_etag(mut fixture: Fixture) { + fixture.mar_fixture.create_logentry_with_size(512); + + let r = fixture.do_download(); + assert_eq!(r.status_code(), StatusCode(200)); + + let delete_response = fixture.do_delete(None); + assert_eq!(delete_response.status_code(), StatusCode(204)); + + // Files should have been deleted. + assert_eq!(fixture.count_mar_inodes(), 0); + } + + #[rstest] + fn files_should_not_delete_if_etag_does_not_match(mut fixture: Fixture) { + fixture.mar_fixture.create_logentry_with_size(512); + + let r = fixture.do_download(); + assert_eq!(r.status_code(), StatusCode(200)); + + let delete_response = fixture.do_delete(Some("bogus".to_owned())); + assert_eq!(delete_response.status_code(), StatusCode(412)); + + // Files should NOT have been deleted. + assert!(fixture.count_mar_inodes() > 0); + } + + #[rstest] + fn error_404_for_deletes(mut fixture: Fixture) { + fixture.mar_fixture.create_logentry_with_size(512); + + // Not calling download before calling delete + + let delete_response = fixture.do_delete(None); + assert_eq!(delete_response.status_code(), StatusCode(404)); + + // Files should NOT have been deleted. + assert!(fixture.count_mar_inodes() > 0); + } + + struct Fixture { + mar_fixture: MarCollectorFixture, + handler: MarExportHandler, + } + + impl Fixture { + fn do_download(&mut self) -> ResponseBox { + let r = TestRequest::new() + .with_method(tiny_http::Method::Get) + .with_path(EXPORT_MAR_URL); + + self.handler + .handle_request(&mut r.into()) + .expect("should process the request") + } + + fn do_delete(&mut self, hash: Option) -> ResponseBox { + let mut r = TestRequest::new() + .with_method(tiny_http::Method::Delete) + .with_path(EXPORT_MAR_URL); + + if let Some(hash) = hash { + r = r.with_header(Header::from_str(&format!("If-Match: {}", hash)).unwrap()) + } + + self.handler + .handle_request(&mut r.into()) + .expect("should process the request") + } + + fn count_mar_inodes(&self) -> usize { + get_size(&self.mar_fixture.mar_staging) + .expect("count mar files") + .inodes as usize + } + } + + #[fixture] + fn fixture() -> Fixture { + let mar_fixture = MarCollectorFixture::new(); + + Fixture { + handler: MarExportHandler::new(mar_fixture.mar_staging.clone()), + mar_fixture, + } + } + + trait ResponseUtils { + fn etag(&self) -> Option; + } + impl ResponseUtils for ResponseBox { + fn etag(&self) -> Option { + self.headers() + .iter() + .find(|h| h.field.equiv("ETag")) + .map(|header| header.value.as_str().trim_matches('"').to_string()) + } + } +} diff --git a/memfaultd/src/mar/export_format.rs b/memfaultd/src/mar/export_format.rs new file mode 100644 index 0000000..b5674a0 --- /dev/null +++ b/memfaultd/src/mar/export_format.rs @@ -0,0 +1,78 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::{eyre, Result}; +use strum_macros::EnumString; + +#[derive(EnumString, Default, PartialEq, Eq, Debug)] +#[strum(serialize_all = "kebab-case")] +/// Supported export formats +pub enum ExportFormat { + #[default] + /// Default format is a MAR file (zip archive) + Mar, + /// Chunk is a memfault-specific format (it's a container on top of MAR). + /// It's mostly useful when you are already exporting memfault chunks from a + /// MCU device and want the Linux data in the same format. + Chunk, + /// Memfault chunks do not include a header to identify the format. They + /// also do not include the length of the data. This format adds a 'CHNK' + /// header and a length field. + ChunkWrapped, +} + +const CONTENT_TYPE_ZIP: &str = "application/zip"; +const CONTENT_TYPE_CHUNK: &str = "application/vnd.memfault.chunk"; +const CONTENT_TYPE_CHUNK_WRAPPED: &str = "application/vnd.memfault.chunk-wrapped"; + +impl ExportFormat { + pub fn to_content_type(&self) -> &'static str { + match self { + ExportFormat::Mar => CONTENT_TYPE_ZIP, + ExportFormat::Chunk => CONTENT_TYPE_CHUNK, + ExportFormat::ChunkWrapped => CONTENT_TYPE_CHUNK_WRAPPED, + } + } + + fn from_mime_type(value: &str) -> Option { + match value { + "*/*" => Some(Self::Mar), + CONTENT_TYPE_ZIP => Some(Self::Mar), + CONTENT_TYPE_CHUNK => Some(Self::Chunk), + CONTENT_TYPE_CHUNK_WRAPPED => Some(Self::ChunkWrapped), + _ => None, + } + } + + pub fn from_accept_header(value: &str) -> Result { + value + .split(',') + .find_map(|mime_type| { + let mime_type = mime_type.trim(); + Self::from_mime_type(mime_type) + }) + .ok_or_else(|| eyre!("Requested format not supported (Accept: {})", value)) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case("*/*", ExportFormat::Mar)] + #[case(CONTENT_TYPE_ZIP, ExportFormat::Mar)] + #[case(format!("{}, {}", CONTENT_TYPE_ZIP, CONTENT_TYPE_CHUNK), ExportFormat::Mar)] + #[case(format!("{}, {}", CONTENT_TYPE_CHUNK, CONTENT_TYPE_ZIP), ExportFormat::Chunk)] + fn test_accept_header_parser>( + #[case] header_value: H, + #[case] expected_format: ExportFormat, + ) { + assert_eq!( + ExportFormat::from_accept_header(&header_value.into()).unwrap(), + expected_format + ); + } +} diff --git a/memfaultd/src/mar/manifest.rs b/memfaultd/src/mar/manifest.rs new file mode 100644 index 0000000..d7a7fd3 --- /dev/null +++ b/memfaultd/src/mar/manifest.rs @@ -0,0 +1,463 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{collections::HashMap, time::Duration}; + +use chrono::Utc; +use eyre::Result; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use uuid::Uuid; + +use crate::{ + build_info::VERSION, + metrics::MetricStringKey, + metrics::{MetricReportType, MetricValue}, + network::DeviceConfigRevision, + network::NetworkConfig, + reboot::RebootReason, + util::serialization::{milliseconds_to_duration, optional_milliseconds_to_duration}, + util::system::{get_system_clock, read_system_boot_id, Clock}, +}; + +#[derive(Serialize, Deserialize)] +pub struct Manifest { + schema_version: u32, + pub collection_time: CollectionTime, + device: Device, + #[serde(default)] + producer: Producer, + #[serde(flatten)] + pub metadata: Metadata, +} + +#[derive(Serialize, Deserialize)] +pub struct CollectionTime { + pub timestamp: chrono::DateTime, + #[serde(rename = "uptime_ms", with = "milliseconds_to_duration")] + uptime: Duration, + linux_boot_id: Uuid, + #[serde(rename = "elapsed_realtime_ms", with = "milliseconds_to_duration")] + elapsed_realtime: Duration, + // TODO: MFLT-9012 - remove these android only fields + boot_count: u32, +} + +#[derive(Serialize, Deserialize)] +struct Device { + project_key: String, + hardware_version: String, + software_version: String, + software_type: String, + device_serial: String, +} + +#[derive(Serialize, Deserialize)] +pub struct Producer { + pub id: String, + pub version: String, +} + +impl Default for Producer { + fn default() -> Self { + Self { + id: "memfaultd".into(), + version: VERSION.to_owned(), + } + } +} + +#[derive(Serialize, Deserialize, Copy, Clone)] +pub enum CompressionAlgorithm { + None, + #[serde(rename = "zlib")] + Zlib, + #[serde(rename = "gzip")] + Gzip, +} + +impl CompressionAlgorithm { + pub fn is_none(&self) -> bool { + matches!(self, CompressionAlgorithm::None) + } +} + +#[derive(Serialize, Deserialize)] +#[serde(tag = "type", content = "metadata")] +pub enum Metadata { + #[serde(rename = "linux-logs")] + LinuxLogs { + format: LinuxLogsFormat, + // PathBuf.file_name() -> OsString but serde does not handle it well + // so we use a String here. + log_file_name: String, + #[serde(skip_serializing_if = "CompressionAlgorithm::is_none")] + compression: CompressionAlgorithm, + cid: Cid, + next_cid: Cid, + }, + #[serde(rename = "device-attributes")] + DeviceAttributes { attributes: Vec }, + #[serde(rename = "device-config")] + DeviceConfig { revision: DeviceConfigRevision }, + #[serde(rename = "elf-coredump")] + ElfCoredump { + coredump_file_name: String, + #[serde(skip_serializing_if = "CompressionAlgorithm::is_none")] + compression: CompressionAlgorithm, + }, + #[serde(rename = "linux-reboot")] + LinuxReboot { reason: RebootReason }, + // DEPRECATED but need to keep the variant for backwards compatibility + // with MARs produced by earlier SDK versions + #[serde(rename = "linux-heartbeat")] + LinuxHeartbeat { + #[serde(serialize_with = "crate::util::serialization::sorted_map::sorted_map")] + metrics: HashMap, + #[serde( + default, + rename = "duration_ms", + skip_serializing_if = "Option::is_none", + with = "optional_milliseconds_to_duration" + )] + duration: Option, + }, + #[serde(rename = "linux-metric-report")] + LinuxMetricReport { + #[serde(serialize_with = "crate::util::serialization::sorted_map::sorted_map")] + metrics: HashMap, + #[serde(rename = "duration_ms", with = "milliseconds_to_duration")] + duration: Duration, + report_type: MetricReportType, + }, + #[serde(rename = "linux-memfault-watch-logs")] + LinuxMemfaultWatch { + cmdline: Vec, + exit_code: i32, + duration: Duration, + stdio_log_file_name: String, + compression: CompressionAlgorithm, + }, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct LinuxLogsFormat { + id: String, + serialization: String, +} + +impl Default for LinuxLogsFormat { + fn default() -> Self { + Self { + id: "v1".into(), + serialization: "json-lines".into(), + } + } +} + +// Note: Memfault manifest defines Cid as an object containing a Uuid. +#[derive(Serialize, Deserialize)] +pub struct Cid { + uuid: Uuid, +} + +#[derive(Serialize, Deserialize, PartialEq, Eq, Debug)] +pub struct DeviceAttribute { + string_key: MetricStringKey, + value: Value, +} + +impl DeviceAttribute { + pub fn new(string_key: MetricStringKey, value: Value) -> Self { + Self { string_key, value } + } +} + +impl, V: Into> TryFrom<(K, V)> for DeviceAttribute { + type Error = String; + + fn try_from(value: (K, V)) -> std::result::Result { + Ok(DeviceAttribute { + string_key: str::parse(value.0.as_ref())?, + value: value.1.into(), + }) + } +} + +impl Metadata { + pub fn new_coredump(coredump_file_name: String, compression: CompressionAlgorithm) -> Self { + Self::ElfCoredump { + coredump_file_name, + compression, + } + } + + pub fn new_log( + log_file_name: String, + cid: Uuid, + next_cid: Uuid, + compression: CompressionAlgorithm, + ) -> Self { + Self::LinuxLogs { + log_file_name, + compression, + cid: Cid { uuid: cid }, + next_cid: Cid { uuid: next_cid }, + format: LinuxLogsFormat::default(), + } + } + + pub fn new_device_attributes(attributes: Vec) -> Self { + Self::DeviceAttributes { attributes } + } + + pub fn new_device_config(revision: DeviceConfigRevision) -> Self { + Self::DeviceConfig { revision } + } + + pub fn new_reboot(reason: RebootReason) -> Self { + Self::LinuxReboot { reason } + } + + pub fn new_metric_report( + metrics: HashMap, + duration: Duration, + report_type: MetricReportType, + ) -> Self { + Self::LinuxMetricReport { + metrics, + duration, + report_type, + } + } +} + +impl CollectionTime { + pub fn now() -> Result { + Ok(Self { + timestamp: Utc::now(), + linux_boot_id: read_system_boot_id()?, + uptime: get_system_clock(Clock::Monotonic)?, + elapsed_realtime: get_system_clock(Clock::Boottime)?, + // TODO: MFLT-9012 - remove these android only fields + boot_count: 0, + }) + } + + #[cfg(test)] + pub fn test_fixture() -> Self { + use chrono::TimeZone; + use uuid::uuid; + + Self { + timestamp: Utc.timestamp_millis_opt(1334250000000).unwrap(), + uptime: Duration::new(10, 0), + linux_boot_id: uuid!("413554b8-a727-11ed-b307-0317a0ffbea7"), + elapsed_realtime: Duration::new(10, 0), + // TODO: MFLT-9012 - remove these android only fields + boot_count: 0, + } + } +} + +impl From<&NetworkConfig> for Device { + fn from(config: &NetworkConfig) -> Self { + Self { + project_key: config.project_key.clone(), + device_serial: config.device_id.clone(), + hardware_version: config.hardware_version.clone(), + software_type: config.software_type.clone(), + software_version: config.software_version.clone(), + } + } +} + +impl Manifest { + pub fn new( + config: &NetworkConfig, + collection_time: CollectionTime, + metadata: Metadata, + ) -> Self { + Manifest { + collection_time, + device: Device::from(config), + producer: Producer::default(), + schema_version: 1, + metadata, + } + } + + pub fn attachments(&self) -> Vec { + match &self.metadata { + Metadata::ElfCoredump { + coredump_file_name, .. + } => vec![coredump_file_name.clone()], + Metadata::LinuxLogs { log_file_name, .. } => vec![log_file_name.clone()], + Metadata::DeviceAttributes { .. } => vec![], + Metadata::DeviceConfig { .. } => vec![], + Metadata::LinuxHeartbeat { .. } => vec![], + Metadata::LinuxMetricReport { .. } => vec![], + Metadata::LinuxReboot { .. } => vec![], + Metadata::LinuxMemfaultWatch { + stdio_log_file_name, + .. + } => vec![stdio_log_file_name.clone()], + } + } +} + +#[cfg(test)] +impl Metadata { + pub fn test_fixture() -> Self { + Metadata::new_device_config(0) + } +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, path::PathBuf, str::FromStr}; + + use crate::{ + mar::CompressionAlgorithm, + metrics::{MetricReportType, MetricValue}, + reboot::RebootReasonCode, + }; + use rstest::rstest; + use uuid::uuid; + + use crate::network::NetworkConfig; + use crate::reboot::RebootReason; + + use super::{CollectionTime, Manifest}; + + #[rstest] + #[case("coredump-gzip", CompressionAlgorithm::Gzip)] + #[case("coredump-none", CompressionAlgorithm::None)] + fn serialization_of_coredump(#[case] name: &str, #[case] compression: CompressionAlgorithm) { + let config = NetworkConfig::test_fixture(); + + let manifest = Manifest::new( + &config, + CollectionTime::test_fixture(), + super::Metadata::new_coredump("/tmp/core.elf".into(), compression), + ); + insta::assert_json_snapshot!(name, manifest, { ".producer.version" => "tests"}); + } + + #[rstest] + #[case("log-zlib", CompressionAlgorithm::Zlib)] + #[case("log-none", CompressionAlgorithm::None)] + fn serialization_of_log(#[case] name: &str, #[case] compression: CompressionAlgorithm) { + let config = NetworkConfig::test_fixture(); + + let this_cid = uuid!("99686390-a728-11ed-a68b-e7ff3cd0c7e7"); + let next_cid = uuid!("9e1ece10-a728-11ed-918e-5be35a10c7e7"); + let manifest = Manifest::new( + &config, + CollectionTime::test_fixture(), + super::Metadata::new_log("/var/log/syslog".into(), this_cid, next_cid, compression), + ); + insta::assert_json_snapshot!(name, manifest, { ".producer.version" => "tests" }); + } + + #[rstest] + fn serialization_of_device_attributes() { + let config = NetworkConfig::test_fixture(); + let manifest = Manifest::new( + &config, + CollectionTime::test_fixture(), + super::Metadata::new_device_attributes(vec![ + ("my_string", "foo").try_into().unwrap(), + ("my_int", 123).try_into().unwrap(), + ("my_float", 123.456).try_into().unwrap(), + ("my_bool", true).try_into().unwrap(), + ]), + ); + insta::assert_json_snapshot!(manifest, { ".producer.version" => "tests"}); + } + + #[rstest] + fn serialization_of_device_configc() { + let config = NetworkConfig::test_fixture(); + let manifest = Manifest::new( + &config, + CollectionTime::test_fixture(), + super::Metadata::new_device_config(42), + ); + insta::assert_json_snapshot!(manifest, { ".producer.version" => "tests"}); + } + + #[rstest] + fn serialization_of_reboot() { + let config = NetworkConfig::test_fixture(); + let manifest = Manifest::new( + &config, + CollectionTime::test_fixture(), + super::Metadata::new_reboot(RebootReason::from(RebootReasonCode::UserShutdown)), + ); + insta::assert_json_snapshot!(manifest, { ".producer.version" => "tests"}); + } + + #[rstest] + fn serialization_of_custom_reboot() { + let config = NetworkConfig::test_fixture(); + let manifest = Manifest::new( + &config, + CollectionTime::test_fixture(), + super::Metadata::new_reboot( + RebootReason::from_str("CustomRebootReason").unwrap_or_else(|e| panic!("{}", e)), + ), + ); + insta::assert_json_snapshot!(manifest, { ".producer.version" => "tests"}); + } + + #[rstest] + fn serialization_of_custom_unexpected_reboot() { + let config = NetworkConfig::test_fixture(); + let manifest = Manifest::new( + &config, + CollectionTime::test_fixture(), + super::Metadata::new_reboot( + RebootReason::from_str("!CustomUnexpectedRebootReason") + .unwrap_or_else(|e| panic!("{}", e)), + ), + ); + insta::assert_json_snapshot!(manifest, { ".producer.version" => "tests"}); + } + + #[rstest] + fn serialization_of_linux_heartbeat() { + let config = NetworkConfig::test_fixture(); + let manifest = Manifest::new( + &config, + CollectionTime::test_fixture(), + super::Metadata::LinuxMetricReport { + metrics: HashMap::from([ + ("n1".parse().unwrap(), MetricValue::Number(1.0)), + ("n2".parse().unwrap(), MetricValue::Number(42.0)), + ]), + duration: std::time::Duration::from_secs(42), + report_type: MetricReportType::Heartbeat, + }, + ); + insta::assert_json_snapshot!(manifest, { ".producer.version" => "tests"}); + } + + #[rstest] + #[case("heartbeat")] + #[case("heartbeat_with_duration")] + #[case("metric_report")] + #[case("device_config")] + #[case("reboot")] + #[case("attributes")] + #[case("elf_coredump")] + fn can_parse_test_manifests(#[case] name: &str) { + let input_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/mar/test-manifests") + .join(name) + .with_extension("json"); + let manifest_json = std::fs::read_to_string(input_path).unwrap(); + let manifest: Manifest = serde_json::from_str(manifest_json.as_str()).unwrap(); + insta::assert_json_snapshot!(name, manifest, { ".producer.version" => "tests"}); + } +} diff --git a/memfaultd/src/mar/mar_entry.rs b/memfaultd/src/mar/mar_entry.rs new file mode 100644 index 0000000..730cd34 --- /dev/null +++ b/memfaultd/src/mar/mar_entry.rs @@ -0,0 +1,152 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! MAR Entry +//! +//! Represents a MAR entry on disk and provides a parsing utility for the MAR staging area. +//! +//! A MAR entry is a folder with a unique name, a manifest and some optional attachments. +//! +use std::fs::{read_dir, File}; +use std::io::BufReader; +use std::iter::once; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::{collections::VecDeque, time::SystemTime}; + +use eyre::{eyre, Context, Result}; +use uuid::Uuid; + +use super::manifest::Manifest; + +/// A candidate folder for inclusion in a MAR zip file +pub struct MarEntry { + /// Path to the directory on disk where the MAR entry is stored. + pub path: PathBuf, + pub uuid: Uuid, + pub manifest: Manifest, +} + +/// An iterator over a list of paths that may contain a MarEntry. +/// Each is lazily transformed into a MarEntry. +pub struct MarEntryIterator { + directories: VecDeque, +} + +impl Iterator for MarEntryIterator { + type Item = Result; + + fn next(&mut self) -> Option { + self.directories.pop_front().map(MarEntry::from_path) + } + + fn size_hint(&self) -> (usize, Option) { + (self.directories.len(), Some(self.directories.len())) + } +} + +impl MarEntry { + /// Go through all files in our staging area and make a list of paths to valid + /// MAR entries to include in the next MAR file. + /// + /// A valid MAR entry is a directory, named with a valid uuid. and it must + /// contain a manifest.json file. To avoid synchronization issue, writers of + /// MAR entries should write to manifest.lock and rename manifest.json when + /// they are done (atomic operation). + pub fn iterate_from_container( + mar_staging: &Path, + ) -> Result>> { + let mut entries: Vec<(PathBuf, Option)> = read_dir(mar_staging) + .wrap_err(eyre!( + "Unable to open MAR staging area: {}", + mar_staging.display() + ))? + .filter_map(std::io::Result::ok) + // Keep only directories + .filter(|d| d.path().is_dir()) + // Collect the creation time so we can sort them + .map(|d| (d.path(), d.metadata().and_then(|m| m.created()).ok())) + .collect(); + + // Sort entries from oldest to newest + entries.sort_by(|a, b| a.1.cmp(&b.1)); + + Ok(MarEntryIterator { + directories: entries.into_iter().map(|e| e.0).collect(), + }) + } + + /// Creates a MarEntry instance from a directory containing a manifest.json file. + pub fn from_path(path: PathBuf) -> Result { + let uuid = Uuid::from_str( + path.file_name() + .ok_or_else(|| eyre!("{} is not a directory", path.display()))? + .to_str() + .ok_or_else(|| eyre!("{} is not a valid directory name", path.display()))?, + )?; + let manifest = path.join("manifest.json"); + if !manifest.exists() { + return Err(eyre!( + "{} does not contain a manifest file.", + path.display() + )); + } + let buf_reader = + BufReader::new(File::open(manifest).wrap_err("Error reading manifest file")?); + let manifest: Manifest = + serde_json::from_reader(buf_reader).wrap_err("Error parsing manifest file")?; + Ok(Self { + path, + uuid, + manifest, + }) + } + + /// Returns an iterator over the filenames of the manifest.json and attachments of this MAR entry. + pub fn filenames(&self) -> impl Iterator { + once("manifest.json".to_owned()).chain(self.manifest.attachments()) + } +} + +#[cfg(test)] +mod tests { + use rstest::{fixture, rstest}; + + use crate::mar::test_utils::MarCollectorFixture; + use crate::test_utils::setup_logger; + + use super::*; + + #[rstest] + fn collecting_from_empty_folder(_setup_logger: (), mar_fixture: MarCollectorFixture) { + assert_eq!( + MarEntry::iterate_from_container(&mar_fixture.mar_staging) + .unwrap() + .count(), + 0 + ) + } + + #[rstest] + fn collecting_from_folder_with_partial_entries( + _setup_logger: (), + mut mar_fixture: MarCollectorFixture, + ) { + mar_fixture.create_empty_entry(); + mar_fixture.create_logentry(); + + assert_eq!( + MarEntry::iterate_from_container(&mar_fixture.mar_staging) + .unwrap() + .filter(|e| e.is_ok()) + .count(), + // Only one entry should be picked up. The other one is ignored. + 1 + ) + } + + #[fixture] + fn mar_fixture() -> MarCollectorFixture { + MarCollectorFixture::new() + } +} diff --git a/memfaultd/src/mar/mar_entry_builder.rs b/memfaultd/src/mar/mar_entry_builder.rs new file mode 100644 index 0000000..1d0fc30 --- /dev/null +++ b/memfaultd/src/mar/mar_entry_builder.rs @@ -0,0 +1,272 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! MAR Entry Builder +//! +use crate::mar::{CollectionTime, Manifest, MarEntry, Metadata}; +use crate::network::NetworkConfig; +use crate::util::disk_size::DiskSize; +use crate::util::fs::move_file; +use eyre::WrapErr; +use std::fs::{create_dir, remove_dir_all, rename, File}; +use std::mem::take; +use std::path::{Path, PathBuf}; +use uuid::Uuid; + +const MAR_ENTRY_OVERHEAD_SIZE_ESTIMATE: u64 = 4096; + +/// A tool to build new MAR entries. Use one of the constructor functions and +/// call save() to write to disk. Any files attached to this MAR entry will +/// be moved when save is called. +pub struct MarEntryBuilder { + entry_dir: MarEntryDir, + uuid: Uuid, + collection_time: CollectionTime, + metadata: M, + attachments: Vec, +} + +pub struct NoMetadata; + +impl MarEntryBuilder { + fn entry_dir_path(&self) -> &Path { + &self.entry_dir.path + } + + pub fn make_attachment_path_in_entry_dir>(&self, filename: F) -> PathBuf { + self.entry_dir_path().join(filename.as_ref()) + } + + pub fn add_attachment(mut self, file: PathBuf) -> MarEntryBuilder { + assert!(file.is_file()); + assert!(file.is_absolute()); + self.attachments.push(file); + self + } +} + +impl MarEntryBuilder { + pub fn new(mar_staging: &Path) -> eyre::Result> { + let collection_time = CollectionTime::now()?; + + // Create a directory for this entry. Make sure this is the last fallible operation here, + // to avoid complicating cleanup in failure scenarios. + let uuid = Uuid::new_v4(); + let path = mar_staging.to_owned().join(uuid.to_string()); + create_dir(&path)?; + + Ok(Self { + entry_dir: MarEntryDir::new(path), + uuid, + collection_time, + metadata: NoMetadata, + attachments: vec![], + }) + } + + pub fn set_metadata(self, metadata: Metadata) -> MarEntryBuilder { + MarEntryBuilder { + entry_dir: self.entry_dir, + uuid: self.uuid, + collection_time: self.collection_time, + attachments: self.attachments, + metadata, + } + } +} + +impl MarEntryBuilder { + /// Consume this builder, writes the manifest and moves the attachment to the + /// MAR storage area and returns a MAR entry. + pub fn save(self, network_config: &NetworkConfig) -> eyre::Result { + // Move attachments + for filepath in self.attachments { + // We already check that attachments are file in the constructor so we ignore + // non-files here. + if let Some(filename) = filepath.file_name() { + let target = self.entry_dir.path.join(filename); + + // Note: if the attachment path was created using make_attachment_path_in_entry_dir(), + // filepath and target will be the same and this will be a no-op. + move_file(&filepath, &target)?; + } + } + + // Prepare manifest + let manifest = Manifest::new(network_config, self.collection_time, self.metadata); + + // Write the manifest to a temp file + let manifest_path = self.entry_dir.path.join("manifest.tmp"); + let manifest_file = File::create(&manifest_path) + .wrap_err_with(|| format!("Error opening manifest {}", manifest_path.display()))?; + serde_json::to_writer(manifest_file, &manifest)?; + + // Rename the manifest to signal that this folder is complete + let manifest_json_path = manifest_path.with_extension("json"); + rename(&manifest_path, &manifest_json_path).wrap_err_with(|| { + format!( + "Error renaming manifest {} to {}", + manifest_path.display(), + manifest_json_path.display() + ) + })?; + + Ok(MarEntry { + path: self.entry_dir.mark_saved(), + uuid: self.uuid, + manifest, + }) + } + + pub fn estimated_entry_size(&self) -> DiskSize { + let attachments_size_bytes: u64 = self + .attachments + .iter() + .filter_map(|p| p.metadata().ok()) + .map(|m| m.len()) + .sum(); + + // Add a bit extra for the overhead of the manifest.json and directory inode: + DiskSize { + bytes: attachments_size_bytes + MAR_ENTRY_OVERHEAD_SIZE_ESTIMATE, + inodes: self.attachments.len() as u64 + 1, + } + } + + pub fn get_metadata(&self) -> &Metadata { + &self.metadata + } +} + +/// Helper structure that will clean up the entry directory on Drop if mark_saved() was not called. +struct MarEntryDir { + path: PathBuf, + saved: bool, +} + +impl MarEntryDir { + fn new(path: PathBuf) -> Self { + Self { path, saved: false } + } + + fn mark_saved(mut self) -> PathBuf { + self.saved = true; + take(&mut self.path) + } +} + +impl Drop for MarEntryDir { + fn drop(&mut self) { + if !self.saved { + let _ = remove_dir_all(&self.path); + } + } +} + +#[cfg(test)] +mod tests { + use super::MAR_ENTRY_OVERHEAD_SIZE_ESTIMATE; + use crate::mar::MarEntryBuilder; + use crate::mar::Metadata; + use crate::network::NetworkConfig; + use crate::test_utils::create_file_with_size; + use rstest::{fixture, rstest}; + use std::path::PathBuf; + use tempfile::{tempdir, TempDir}; + + #[rstest] + fn cleans_up_entry_dir_when_save_was_not_called(fixture: Fixture) { + let builder = MarEntryBuilder::new(&fixture.mar_staging).unwrap(); + let entry_dir = builder.entry_dir_path().to_owned(); + assert!(entry_dir.exists()); + create_file_with_size(&entry_dir.join("attachment"), 1024).unwrap(); + drop(builder); + assert!(!entry_dir.exists()); + } + + #[rstest] + fn save_keeps_entry_dir_and_adds_manifest_json(fixture: Fixture) { + let mut entry_dir_option = None; + { + let builder = MarEntryBuilder::new(&fixture.mar_staging).unwrap(); + let _ = entry_dir_option.insert(builder.entry_dir_path().to_owned()); + builder + .set_metadata(Metadata::test_fixture()) + .save(&NetworkConfig::test_fixture()) + .unwrap(); + } + let entry_dir = entry_dir_option.unwrap(); + assert!(entry_dir.exists()); + assert!(entry_dir.join("manifest.json").exists()); + } + + #[rstest] + fn create_attachment_inside_entry_dir(fixture: Fixture) { + let builder = MarEntryBuilder::new(&fixture.mar_staging).unwrap(); + let orig_attachment_path = builder.make_attachment_path_in_entry_dir("attachment"); + create_file_with_size(&orig_attachment_path, 1024).unwrap(); + + builder + .add_attachment(orig_attachment_path.clone()) + .set_metadata(Metadata::test_fixture()) + .save(&NetworkConfig::test_fixture()) + .unwrap(); + + // Attachment is still where it was written: + assert!(orig_attachment_path.exists()); + } + + #[rstest] + fn attachment_outside_entry_dir_is_moved_into_entry_dir_upon_save(fixture: Fixture) { + let builder = MarEntryBuilder::new(&fixture.mar_staging).unwrap(); + let entry_dir = builder.entry_dir_path().to_owned(); + + let tempdir = tempdir().unwrap(); + let orig_attachment_path = tempdir.path().join("attachment"); + create_file_with_size(&orig_attachment_path, 1024).unwrap(); + + builder + .add_attachment(orig_attachment_path.clone()) + .set_metadata(Metadata::test_fixture()) + .save(&NetworkConfig::test_fixture()) + .unwrap(); + + // Attachment has been moved into the entry dir: + assert!(!orig_attachment_path.exists()); + assert!(entry_dir + .join(orig_attachment_path.file_name().unwrap()) + .exists()); + } + + #[rstest] + fn can_estimate_size_of_a_mar_entry(fixture: Fixture) { + let builder = MarEntryBuilder::new(&fixture.mar_staging).unwrap(); + let orig_attachment_path = builder.make_attachment_path_in_entry_dir("attachment"); + create_file_with_size(&orig_attachment_path, 1024).unwrap(); + + let builder = builder + .add_attachment(orig_attachment_path) + .set_metadata(Metadata::test_fixture()); + + assert_eq!( + builder.estimated_entry_size().bytes, + 1024 + MAR_ENTRY_OVERHEAD_SIZE_ESTIMATE + ); + assert_eq!(builder.estimated_entry_size().inodes, 2); + } + + struct Fixture { + _tempdir: TempDir, + mar_staging: PathBuf, + } + + #[fixture] + fn fixture() -> Fixture { + let tempdir = tempdir().unwrap(); + let mar_staging = tempdir.path().to_owned(); + Fixture { + _tempdir: tempdir, + mar_staging, + } + } +} diff --git a/memfaultd/src/mar/mod.rs b/memfaultd/src/mar/mod.rs new file mode 100644 index 0000000..00ff401 --- /dev/null +++ b/memfaultd/src/mar/mod.rs @@ -0,0 +1,25 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +pub mod clean; +pub mod manifest; +pub mod mar_entry; +pub mod mar_entry_builder; +pub mod upload; + +pub use clean::*; +pub use manifest::*; +pub use mar_entry::*; +pub use mar_entry_builder::*; +pub use upload::*; + +mod chunks; +mod export; +mod export_format; + +pub use chunks::{Chunk, ChunkMessage, ChunkMessageType, ChunkWrapper}; +pub use export::{MarExportHandler, EXPORT_MAR_URL}; +pub use export_format::ExportFormat; + +#[cfg(test)] +mod test_utils; diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__attributes.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__attributes.snap new file mode 100644 index 0000000..63c65dc --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__attributes.snap @@ -0,0 +1,46 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "1970-01-01T00:00:04.321Z", + "uptime_ms": 800000, + "linux_boot_id": "230295cb-04d4-40b8-8624-ec37089b9b75", + "elapsed_realtime_ms": 900000, + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "evt", + "software_version": "1.0.0", + "software_type": "linux-build", + "device_serial": "DEMOSERIAL" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "device-attributes", + "metadata": { + "attributes": [ + { + "string_key": "my_string", + "value": "foo" + }, + { + "string_key": "my_integer", + "value": 123 + }, + { + "string_key": "my_float", + "value": 123.456 + }, + { + "string_key": "my_bool", + "value": true + } + ] + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__coredump-gzip.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__coredump-gzip.snap new file mode 100644 index 0000000..a716082 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__coredump-gzip.snap @@ -0,0 +1,30 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "elf-coredump", + "metadata": { + "coredump_file_name": "/tmp/core.elf", + "compression": "gzip" + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__coredump-none.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__coredump-none.snap new file mode 100644 index 0000000..03cb837 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__coredump-none.snap @@ -0,0 +1,29 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "elf-coredump", + "metadata": { + "coredump_file_name": "/tmp/core.elf" + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__device_config.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__device_config.snap new file mode 100644 index 0000000..4f49a1c --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__device_config.snap @@ -0,0 +1,29 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "device-config", + "metadata": { + "revision": 42 + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__elf_coredump.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__elf_coredump.snap new file mode 100644 index 0000000..07b4b90 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__elf_coredump.snap @@ -0,0 +1,30 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "1970-01-01T00:00:04.321Z", + "uptime_ms": 800000, + "linux_boot_id": "230295cb-04d4-40b8-8624-ec37089b9b75", + "elapsed_realtime_ms": 900000, + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "evt", + "software_version": "1.0.0", + "software_type": "linux-build", + "device_serial": "DEMOSERIAL" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "elf-coredump", + "metadata": { + "coredump_file_name": "core-da01317a-902f-48f8-8c3f-aabf8b14facc.elf.gz", + "compression": "gzip" + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__heartbeat.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__heartbeat.snap new file mode 100644 index 0000000..e3b18fb --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__heartbeat.snap @@ -0,0 +1,52 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2023-06-08T23:47:39.021439227Z", + "uptime_ms": 35587, + "linux_boot_id": "ebf5f46f-fca6-4149-93cf-1c12a5e4d019", + "elapsed_realtime_ms": 35588, + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "qemuarm64", + "software_version": "0.0.1", + "software_type": "main", + "device_serial": "DEMOSERIAL" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "linux-heartbeat", + "metadata": { + "metrics": { + "cpu/sum/percent/idle": 482.959065568454, + "cpu/sum/percent/interrupt": 0.0, + "cpu/sum/percent/nice": 0.0, + "cpu/sum/percent/softirq": 0.403632694248234, + "cpu/sum/percent/steal": 0.0, + "cpu/sum/percent/system": 4.32826849339439, + "cpu/sum/percent/user": 15.6038091933719, + "cpu/sum/percent/wait": 0.702826448134218, + "df/media/df_complex/free": 416471040.0, + "df/media/df_complex/reserved": 27681792.0, + "df/media/df_complex/used": 19456.0, + "interface/enp0s1/if_dropped/rx": 0.0, + "interface/enp0s1/if_dropped/tx": 0.0, + "interface/enp0s1/if_errors/rx": 0.0, + "interface/enp0s1/if_errors/tx": 0.0, + "interface/enp0s1/if_octets/rx": 523.064016916174, + "interface/enp0s1/if_octets/tx": 152.289523564009, + "interface/enp0s1/if_packets/rx": 1.39990546332198, + "interface/enp0s1/if_packets/tx": 1.1999189685617, + "memory/memory/free": 260657152.0, + "memory/memory/used": 29261824.0, + "uptime/uptime": 26.0 + } + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__heartbeat_with_duration.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__heartbeat_with_duration.snap new file mode 100644 index 0000000..c9e46b4 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__heartbeat_with_duration.snap @@ -0,0 +1,53 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2023-06-08T23:47:39.021439227Z", + "uptime_ms": 35587, + "linux_boot_id": "ebf5f46f-fca6-4149-93cf-1c12a5e4d019", + "elapsed_realtime_ms": 35588, + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "qemuarm64", + "software_version": "0.0.1", + "software_type": "main", + "device_serial": "DEMOSERIAL" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "linux-heartbeat", + "metadata": { + "metrics": { + "cpu/sum/percent/idle": 482.959065568454, + "cpu/sum/percent/interrupt": 0.0, + "cpu/sum/percent/nice": 0.0, + "cpu/sum/percent/softirq": 0.403632694248234, + "cpu/sum/percent/steal": 0.0, + "cpu/sum/percent/system": 4.32826849339439, + "cpu/sum/percent/user": 15.6038091933719, + "cpu/sum/percent/wait": 0.702826448134218, + "df/media/df_complex/free": 416471040.0, + "df/media/df_complex/reserved": 27681792.0, + "df/media/df_complex/used": 19456.0, + "interface/enp0s1/if_dropped/rx": 0.0, + "interface/enp0s1/if_dropped/tx": 0.0, + "interface/enp0s1/if_errors/rx": 0.0, + "interface/enp0s1/if_errors/tx": 0.0, + "interface/enp0s1/if_octets/rx": 523.064016916174, + "interface/enp0s1/if_octets/tx": 152.289523564009, + "interface/enp0s1/if_packets/rx": 1.39990546332198, + "interface/enp0s1/if_packets/tx": 1.1999189685617, + "memory/memory/free": 260657152.0, + "memory/memory/used": 29261824.0, + "uptime/uptime": 26.0 + }, + "duration_ms": 14209 + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__log-none.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__log-none.snap new file mode 100644 index 0000000..ffc4361 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__log-none.snap @@ -0,0 +1,39 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "linux-logs", + "metadata": { + "format": { + "id": "v1", + "serialization": "json-lines" + }, + "log_file_name": "/var/log/syslog", + "cid": { + "uuid": "99686390-a728-11ed-a68b-e7ff3cd0c7e7" + }, + "next_cid": { + "uuid": "9e1ece10-a728-11ed-918e-5be35a10c7e7" + } + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__log-zlib.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__log-zlib.snap new file mode 100644 index 0000000..34fe055 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__log-zlib.snap @@ -0,0 +1,40 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "linux-logs", + "metadata": { + "format": { + "id": "v1", + "serialization": "json-lines" + }, + "log_file_name": "/var/log/syslog", + "compression": "zlib", + "cid": { + "uuid": "99686390-a728-11ed-a68b-e7ff3cd0c7e7" + }, + "next_cid": { + "uuid": "9e1ece10-a728-11ed-918e-5be35a10c7e7" + } + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__metric_report.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__metric_report.snap new file mode 100644 index 0000000..5176a6c --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__metric_report.snap @@ -0,0 +1,56 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2023-06-08T23:47:39.021439227Z", + "uptime_ms": 35587, + "linux_boot_id": "ebf5f46f-fca6-4149-93cf-1c12a5e4d019", + "elapsed_realtime_ms": 35588, + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "qemuarm64", + "software_version": "0.0.1", + "software_type": "main", + "device_serial": "DEMOSERIAL" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "linux-metric-report", + "metadata": { + "metrics": { + "cpu/sum/percent/idle": 482.959065568454, + "cpu/sum/percent/interrupt": 0.0, + "cpu/sum/percent/nice": 0.0, + "cpu/sum/percent/softirq": 0.403632694248234, + "cpu/sum/percent/steal": 0.0, + "cpu/sum/percent/system": 4.32826849339439, + "cpu/sum/percent/user": 15.6038091933719, + "cpu/sum/percent/wait": 0.702826448134218, + "df/media/df_complex/free": 416471040.0, + "df/media/df_complex/reserved": 27681792.0, + "df/media/df_complex/used": 19456.0, + "interface/enp0s1/if_dropped/rx": 0.0, + "interface/enp0s1/if_dropped/tx": 0.0, + "interface/enp0s1/if_errors/rx": 0.0, + "interface/enp0s1/if_errors/tx": 0.0, + "interface/enp0s1/if_octets/rx": 523.064016916174, + "interface/enp0s1/if_octets/tx": 152.289523564009, + "interface/enp0s1/if_packets/rx": 1.39990546332198, + "interface/enp0s1/if_packets/tx": 1.1999189685617, + "memory/memory/free": 260657152.0, + "memory/memory/used": 29261824.0, + "uptime/uptime": 26.0 + }, + "duration_ms": 14209, + "report_type": { + "session": "test-fixture-session" + } + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__reboot.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__reboot.snap new file mode 100644 index 0000000..7c92718 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__reboot.snap @@ -0,0 +1,29 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2023-05-24T18:20:21.371852623Z", + "uptime_ms": 17445527, + "linux_boot_id": "48af0b97-79c3-4e95-ba80-8c436b4b9e8f", + "elapsed_realtime_ms": 17445527, + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "qemuarm64", + "software_version": "0.0.1", + "software_type": "main", + "device_serial": "DEMOSERIAL" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "linux-reboot", + "metadata": { + "reason": 3 + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_custom_reboot.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_custom_reboot.snap new file mode 100644 index 0000000..f911a44 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_custom_reboot.snap @@ -0,0 +1,32 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "linux-reboot", + "metadata": { + "reason": { + "unexpected": false, + "name": "CustomRebootReason" + } + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_custom_unexpected_reboot.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_custom_unexpected_reboot.snap new file mode 100644 index 0000000..6319932 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_custom_unexpected_reboot.snap @@ -0,0 +1,32 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "linux-reboot", + "metadata": { + "reason": { + "unexpected": true, + "name": "CustomUnexpectedRebootReason" + } + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_device_attributes.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_device_attributes.snap new file mode 100644 index 0000000..c7d93f9 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_device_attributes.snap @@ -0,0 +1,46 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "device-attributes", + "metadata": { + "attributes": [ + { + "string_key": "my_string", + "value": "foo" + }, + { + "string_key": "my_int", + "value": 123 + }, + { + "string_key": "my_float", + "value": 123.456 + }, + { + "string_key": "my_bool", + "value": true + } + ] + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_device_configc.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_device_configc.snap new file mode 100644 index 0000000..4f49a1c --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_device_configc.snap @@ -0,0 +1,29 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "device-config", + "metadata": { + "revision": 42 + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_linux_heartbeat.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_linux_heartbeat.snap new file mode 100644 index 0000000..4bec52e --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_linux_heartbeat.snap @@ -0,0 +1,34 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "linux-metric-report", + "metadata": { + "metrics": { + "n1": 1.0, + "n2": 42.0 + }, + "duration_ms": 42000, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_reboot.snap b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_reboot.snap new file mode 100644 index 0000000..cea2c31 --- /dev/null +++ b/memfaultd/src/mar/snapshots/memfaultd__mar__manifest__tests__serialization_of_reboot.snap @@ -0,0 +1,29 @@ +--- +source: memfaultd/src/mar/manifest.rs +expression: manifest +--- +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "producer": { + "id": "memfaultd", + "version": "tests" + }, + "type": "linux-reboot", + "metadata": { + "reason": 1 + } +} diff --git a/memfaultd/src/mar/test-manifests/attributes.json b/memfaultd/src/mar/test-manifests/attributes.json new file mode 100644 index 0000000..af5fdbb --- /dev/null +++ b/memfaultd/src/mar/test-manifests/attributes.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "collection_time": { + "timestamp": "1970-01-01T00:00:04.321Z", + "uptime_ms": 800000, + "elapsed_realtime_ms": 900000, + "linux_boot_id": "230295cb-04d4-40b8-8624-ec37089b9b75", + "boot_count": 0 + }, + "type": "device-attributes", + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "evt", + "software_version": "1.0.0", + "software_type": "linux-build", + "device_serial": "DEMOSERIAL" + }, + "metadata": { + "attributes": [ + { + "string_key": "my_string", + "value": "foo" + }, + { + "string_key": "my_integer", + "value": 123 + }, + { + "string_key": "my_float", + "value": 123.456 + }, + { + "string_key": "my_bool", + "value": true + } + ] + } +} diff --git a/memfaultd/src/mar/test-manifests/device_config.json b/memfaultd/src/mar/test-manifests/device_config.json new file mode 100644 index 0000000..9833f09 --- /dev/null +++ b/memfaultd/src/mar/test-manifests/device_config.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2012-04-12T17:00:00Z", + "uptime_ms": 10000, + "linux_boot_id": "413554b8-a727-11ed-b307-0317a0ffbea7", + "elapsed_realtime_ms": 10000, + "boot_count": 0 + }, + "device": { + "project_key": "abcd", + "hardware_version": "DVT", + "software_version": "1.0.0", + "software_type": "test", + "device_serial": "001" + }, + "type": "device-config", + "metadata": { + "revision": 42 + } +} diff --git a/memfaultd/src/mar/test-manifests/elf_coredump.json b/memfaultd/src/mar/test-manifests/elf_coredump.json new file mode 100644 index 0000000..b00bc62 --- /dev/null +++ b/memfaultd/src/mar/test-manifests/elf_coredump.json @@ -0,0 +1,22 @@ +{ + "schema_version": 1, + "collection_time": { + "timestamp": "1970-01-01T00:00:04.321Z", + "uptime_ms": 800000, + "elapsed_realtime_ms": 900000, + "linux_boot_id": "230295cb-04d4-40b8-8624-ec37089b9b75", + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "evt", + "software_version": "1.0.0", + "software_type": "linux-build", + "device_serial": "DEMOSERIAL" + }, + "type": "elf-coredump", + "metadata": { + "coredump_file_name": "core-da01317a-902f-48f8-8c3f-aabf8b14facc.elf.gz", + "compression": "gzip" + } +} diff --git a/memfaultd/src/mar/test-manifests/heartbeat.json b/memfaultd/src/mar/test-manifests/heartbeat.json new file mode 100644 index 0000000..8cf4cba --- /dev/null +++ b/memfaultd/src/mar/test-manifests/heartbeat.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2023-06-08T23:47:39.021439227Z", + "uptime_ms": 35587, + "linux_boot_id": "ebf5f46f-fca6-4149-93cf-1c12a5e4d019", + "elapsed_realtime_ms": 35588, + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "qemuarm64", + "software_version": "0.0.1", + "software_type": "main", + "device_serial": "DEMOSERIAL" + }, + "producer": { + "id": "memfaultd", + "version": "dev" + }, + "type": "linux-heartbeat", + "metadata": { + "metrics": { + "cpu/sum/percent/idle": 482.959065568454, + "cpu/sum/percent/interrupt": 0, + "cpu/sum/percent/nice": 0, + "cpu/sum/percent/softirq": 0.403632694248234, + "cpu/sum/percent/steal": 0, + "cpu/sum/percent/system": 4.32826849339439, + "cpu/sum/percent/user": 15.6038091933719, + "cpu/sum/percent/wait": 0.702826448134218, + "df/media/df_complex/free": 416471040, + "df/media/df_complex/reserved": 27681792, + "df/media/df_complex/used": 19456, + "interface/enp0s1/if_dropped/rx": 0, + "interface/enp0s1/if_dropped/tx": 0, + "interface/enp0s1/if_errors/rx": 0, + "interface/enp0s1/if_errors/tx": 0, + "interface/enp0s1/if_octets/rx": 523.064016916174, + "interface/enp0s1/if_octets/tx": 152.289523564009, + "interface/enp0s1/if_packets/rx": 1.39990546332198, + "interface/enp0s1/if_packets/tx": 1.1999189685617, + "memory/memory/free": 260657152, + "memory/memory/used": 29261824, + "uptime/uptime": 26 + } + } +} diff --git a/memfaultd/src/mar/test-manifests/heartbeat_with_duration.json b/memfaultd/src/mar/test-manifests/heartbeat_with_duration.json new file mode 100644 index 0000000..e17b93a --- /dev/null +++ b/memfaultd/src/mar/test-manifests/heartbeat_with_duration.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2023-06-08T23:47:39.021439227Z", + "uptime_ms": 35587, + "linux_boot_id": "ebf5f46f-fca6-4149-93cf-1c12a5e4d019", + "elapsed_realtime_ms": 35588, + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "qemuarm64", + "software_version": "0.0.1", + "software_type": "main", + "device_serial": "DEMOSERIAL" + }, + "producer": { + "id": "memfaultd", + "version": "dev" + }, + "type": "linux-heartbeat", + "metadata": { + "metrics": { + "cpu/sum/percent/idle": 482.959065568454, + "cpu/sum/percent/interrupt": 0, + "cpu/sum/percent/nice": 0, + "cpu/sum/percent/softirq": 0.403632694248234, + "cpu/sum/percent/steal": 0, + "cpu/sum/percent/system": 4.32826849339439, + "cpu/sum/percent/user": 15.6038091933719, + "cpu/sum/percent/wait": 0.702826448134218, + "df/media/df_complex/free": 416471040, + "df/media/df_complex/reserved": 27681792, + "df/media/df_complex/used": 19456, + "interface/enp0s1/if_dropped/rx": 0, + "interface/enp0s1/if_dropped/tx": 0, + "interface/enp0s1/if_errors/rx": 0, + "interface/enp0s1/if_errors/tx": 0, + "interface/enp0s1/if_octets/rx": 523.064016916174, + "interface/enp0s1/if_octets/tx": 152.289523564009, + "interface/enp0s1/if_packets/rx": 1.39990546332198, + "interface/enp0s1/if_packets/tx": 1.1999189685617, + "memory/memory/free": 260657152, + "memory/memory/used": 29261824, + "uptime/uptime": 26 + }, + "duration_ms": 14209 + } +} diff --git a/memfaultd/src/mar/test-manifests/log.json b/memfaultd/src/mar/test-manifests/log.json new file mode 100644 index 0000000..cfc461f --- /dev/null +++ b/memfaultd/src/mar/test-manifests/log.json @@ -0,0 +1,25 @@ +{ + "schema_version": 1, + "collection_time": { + "timestamp": "1970-01-01T00:00:04.321Z", + "uptime_ms": 800000, + "elapsed_realtime_ms": 900000, + "linux_boot_id": "230295cb-04d4-40b8-8624-ec37089b9b75", + "boot_count": 0 + }, + "type": "linux-logs", + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "evt", + "software_version": "1.0.0", + "software_type": "linux-build", + "device_serial": "DEMOSERIAL" + }, + "metadata": { + "format": { "id": "v1", "serialization": "json-lines" }, + "producer": { "id": "memfaultd", "version": "2.0.0" }, + "log_file_name": "logs.txt", + "cid": { "uuid": "271e5249-1e3f-4c89-b4a5-ed96cc8f42a7" }, + "next_cid": { "uuid": "4276a26b-ba89-4c57-832f-d9e90e9629dd" } + } +} diff --git a/memfaultd/src/mar/test-manifests/metric_report.json b/memfaultd/src/mar/test-manifests/metric_report.json new file mode 100644 index 0000000..af64563 --- /dev/null +++ b/memfaultd/src/mar/test-manifests/metric_report.json @@ -0,0 +1,52 @@ +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2023-06-08T23:47:39.021439227Z", + "uptime_ms": 35587, + "linux_boot_id": "ebf5f46f-fca6-4149-93cf-1c12a5e4d019", + "elapsed_realtime_ms": 35588, + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "qemuarm64", + "software_version": "0.0.1", + "software_type": "main", + "device_serial": "DEMOSERIAL" + }, + "producer": { + "id": "memfaultd", + "version": "dev" + }, + "type": "linux-metric-report", + "metadata": { + "metrics": { + "cpu/sum/percent/idle": 482.959065568454, + "cpu/sum/percent/interrupt": 0, + "cpu/sum/percent/nice": 0, + "cpu/sum/percent/softirq": 0.403632694248234, + "cpu/sum/percent/steal": 0, + "cpu/sum/percent/system": 4.32826849339439, + "cpu/sum/percent/user": 15.6038091933719, + "cpu/sum/percent/wait": 0.702826448134218, + "df/media/df_complex/free": 416471040, + "df/media/df_complex/reserved": 27681792, + "df/media/df_complex/used": 19456, + "interface/enp0s1/if_dropped/rx": 0, + "interface/enp0s1/if_dropped/tx": 0, + "interface/enp0s1/if_errors/rx": 0, + "interface/enp0s1/if_errors/tx": 0, + "interface/enp0s1/if_octets/rx": 523.064016916174, + "interface/enp0s1/if_octets/tx": 152.289523564009, + "interface/enp0s1/if_packets/rx": 1.39990546332198, + "interface/enp0s1/if_packets/tx": 1.1999189685617, + "memory/memory/free": 260657152, + "memory/memory/used": 29261824, + "uptime/uptime": 26 + }, + "duration_ms": 14209, + "report_type": { + "session": "test-fixture-session" + } + } +} diff --git a/memfaultd/src/mar/test-manifests/reboot.json b/memfaultd/src/mar/test-manifests/reboot.json new file mode 100644 index 0000000..a927a63 --- /dev/null +++ b/memfaultd/src/mar/test-manifests/reboot.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "collection_time": { + "timestamp": "2023-05-24T18:20:21.371852623Z", + "uptime_ms": 17445527, + "linux_boot_id": "48af0b97-79c3-4e95-ba80-8c436b4b9e8f", + "elapsed_realtime_ms": 17445527, + "boot_count": 0 + }, + "device": { + "project_key": "INSERT_PROJECT_KEY_HERE", + "hardware_version": "qemuarm64", + "software_version": "0.0.1", + "software_type": "main", + "device_serial": "DEMOSERIAL" + }, + "type": "linux-reboot", + "metadata": { + "reason": 3 + } +} diff --git a/memfaultd/src/mar/test_utils.rs b/memfaultd/src/mar/test_utils.rs new file mode 100644 index 0000000..25688d2 --- /dev/null +++ b/memfaultd/src/mar/test_utils.rs @@ -0,0 +1,243 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + collections::HashMap, + time::{Duration, SystemTime}, +}; +use std::{ + fs::{create_dir, create_dir_all, set_permissions, File}, + io::{BufWriter, Write}, + os::unix::prelude::PermissionsExt, + path::PathBuf, +}; + +use crate::{ + mar::{CompressionAlgorithm, DeviceAttribute}, + metrics::{MetricReportType, MetricStringKey, MetricValue}, + reboot::RebootReason, +}; +use tempfile::{tempdir, TempDir}; +use uuid::Uuid; + +use crate::network::NetworkConfig; +use crate::test_utils::create_file_with_size; +use crate::util::zip::ZipEncoder; + +use super::manifest::{CollectionTime, Manifest, Metadata}; + +pub struct MarCollectorFixture { + pub mar_staging: PathBuf, + // Keep a reference to the tempdir so it is automatically + // deleted *after* the fixture + _tempdir: TempDir, + config: NetworkConfig, +} + +impl MarCollectorFixture { + pub fn new() -> Self { + let tempdir = tempdir().unwrap(); + let mar_staging = tempdir.path().to_owned(); + create_dir_all(&mar_staging).unwrap(); + Self { + mar_staging, + _tempdir: tempdir, + config: NetworkConfig::test_fixture(), + } + } + + pub fn create_empty_entry(&mut self) -> PathBuf { + let uuid = Uuid::new_v4(); + let path = self.mar_staging.join(uuid.to_string()); + create_dir(&path).unwrap(); + path + } + + pub fn create_device_attributes_entry( + &mut self, + attributes: Vec, + timestamp: SystemTime, + ) -> PathBuf { + let path = self.create_empty_entry(); + let manifest_path = path.join("manifest.json"); + + let manifest_file = File::create(manifest_path).unwrap(); + + let mut collection_time = CollectionTime::test_fixture(); + collection_time.timestamp = timestamp.into(); + + let manifest = Manifest::new( + &self.config, + collection_time, + Metadata::new_device_attributes(attributes), + ); + serde_json::to_writer(BufWriter::new(manifest_file), &manifest).unwrap(); + + path + } + + pub fn create_logentry_with_size(&mut self, size: u64) -> PathBuf { + self.create_logentry_with_size_and_age(size, SystemTime::now()) + } + + pub fn create_logentry_with_size_and_age( + &mut self, + size: u64, + timestamp: SystemTime, + ) -> PathBuf { + let path = self.create_empty_entry(); + let manifest_path = path.join("manifest.json"); + + let log_name = "system.log".to_owned(); + let log_path = path.join(&log_name); + create_file_with_size(&log_path, size).unwrap(); + + let manifest_file = File::create(manifest_path).unwrap(); + + let mut collection_time = CollectionTime::test_fixture(); + collection_time.timestamp = timestamp.into(); + + let manifest = Manifest::new( + &self.config, + collection_time, + Metadata::new_log( + log_name, + Uuid::new_v4(), + Uuid::new_v4(), + CompressionAlgorithm::Zlib, + ), + ); + serde_json::to_writer(BufWriter::new(manifest_file), &manifest).unwrap(); + + path + } + + pub fn create_logentry(&mut self) -> PathBuf { + self.create_logentry_with_size_and_age(0, SystemTime::now()) + } + + pub fn create_logentry_with_unreadable_attachment(&mut self) -> PathBuf { + let path = self.create_empty_entry(); + let manifest_path = path.join("manifest.json"); + + let log_name = "system.log".to_owned(); + let log_path = path.join(&log_name); + let log = File::create(&log_path).unwrap(); + drop(log); + + let mut permissions = log_path.metadata().unwrap().permissions(); + permissions.set_mode(0o0); + set_permissions(&log_path, permissions).unwrap(); + + let manifest_file = File::create(manifest_path).unwrap(); + let manifest = Manifest::new( + &self.config, + CollectionTime::test_fixture(), + Metadata::new_log( + log_name, + Uuid::new_v4(), + Uuid::new_v4(), + CompressionAlgorithm::Zlib, + ), + ); + serde_json::to_writer(BufWriter::new(manifest_file), &manifest).unwrap(); + + path + } + + pub fn create_entry_with_bogus_json(&mut self) -> PathBuf { + let path = self.create_empty_entry(); + let manifest_path = path.join("manifest.json"); + File::create(manifest_path) + .unwrap() + .write_all(b"BOGUS") + .unwrap(); + path + } + + pub fn create_entry_without_directory_read_permission(&mut self) -> PathBuf { + let path = self.create_empty_entry(); + let manifest_path = path.join("manifest.json"); + File::create(manifest_path) + .unwrap() + .write_all(b"BOGUS") + .unwrap(); + + let mut permissions = path.metadata().unwrap().permissions(); + permissions.set_mode(0o0); + set_permissions(&path, permissions).unwrap(); + path + } + + pub fn create_entry_without_manifest_read_permission(&mut self) -> PathBuf { + let path = self.create_empty_entry(); + let manifest_path = path.join("manifest.json"); + File::create(&manifest_path) + .unwrap() + .write_all(b"BOGUS") + .unwrap(); + + let mut permissions = manifest_path.metadata().unwrap().permissions(); + permissions.set_mode(0o0); + set_permissions(manifest_path, permissions).unwrap(); + path + } + + pub fn create_metric_report_entry( + &mut self, + metrics: HashMap, + duration: Duration, + report_type: MetricReportType, + ) -> PathBuf { + let path = self.create_empty_entry(); + let manifest_path = path.join("manifest.json"); + + let manifest_file = File::create(manifest_path).unwrap(); + let manifest = Manifest::new( + &self.config, + CollectionTime::test_fixture(), + Metadata::new_metric_report(metrics, duration, report_type), + ); + serde_json::to_writer(BufWriter::new(manifest_file), &manifest).unwrap(); + + path + } + + pub fn create_reboot_entry(&mut self, reason: RebootReason) -> PathBuf { + let path = self.create_empty_entry(); + let manifest_path = path.join("manifest.json"); + + let manifest_file = File::create(manifest_path).unwrap(); + let manifest = Manifest::new( + &self.config, + CollectionTime::test_fixture(), + Metadata::new_reboot(reason), + ); + serde_json::to_writer(BufWriter::new(manifest_file), &manifest).unwrap(); + + path + } +} + +/// Check the content of a MAR zip encoder against a list of expected files. +/// The first (in zip order) entry name is renamed from "some_uuid/" to +/// "/" before matching and the list is sorted alphabetically. +/// Eg: ZIP(abcd42/manifest.json abcd42/file.txt) => [/file.txt, /manifest.json] +pub fn assert_mar_content_matches(zip_encoder: &ZipEncoder, expected_files: Vec<&str>) -> bool { + // Get the folder name for the first entry, we will s/(entry_uuid)// to make matching friendlier + let file_names = zip_encoder.file_names(); + assert!(!file_names.is_empty()); + + let entry_name = file_names[0] + .split(std::path::MAIN_SEPARATOR) + .next() + .unwrap(); + let mut files_list = file_names + .iter() + .map(|filename| filename.replace(entry_name, "")) + .collect::>(); + files_list.sort(); + + assert_eq!(files_list, *expected_files); + true +} diff --git a/memfaultd/src/mar/upload.rs b/memfaultd/src/mar/upload.rs new file mode 100644 index 0000000..df40622 --- /dev/null +++ b/memfaultd/src/mar/upload.rs @@ -0,0 +1,514 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Collect and upload MAR entries. +//! +//! This module provides the functionality to collect all valid MAR entries, upload them and delete them on success. +//! +//! Whether or not an entry is uploaded depends on the sampling configuration. Each type of entry can have a different +//! level of configuration with device config and reboots always being uploaded. All other will be uploaded based on +//! the below rules: +//! +//! +=================+=====+=====+========+======+ +//! | MAR Type | Off | Low | Medium | High | +//! +=================+=====+=====+========+======+ +//! | heartbeat | | | x | x | +//! +-----------------+-----+-----+--------+------+ +//! | daily-heartbeat | | x | x | x | +//! +-----------------+-----+-----+--------+------+ +//! | session | | | x | x | +//! +-----------------+-----+-----+--------+------+ +//! | attributes | | | x | x | +//! +-----------------+-----+-----+--------+------+ +//! | coredump | | | x | x | +//! +-----------------+-----+-----+--------+------+ +//! | logs | | | x | x | +//! +-----------------+-----+-----+--------+------+ + +use std::fs::{remove_dir_all, File}; +use std::io::BufReader; +use std::path::{Path, PathBuf}; + +use eyre::{Context, Result}; +use itertools::Itertools; +use log::{trace, warn}; + +use crate::{ + config::{Resolution, Sampling}, + mar::{MarEntry, Metadata}, + metrics::MetricReportType, + network::NetworkClient, + util::zip::{zip_stream_len_empty, zip_stream_len_for_file, ZipEncoder, ZipEntryInfo}, +}; + +/// Collect all valid MAR entries, upload them and delete them on success. +/// +/// Returns the number of MAR entries that were uploaded. +/// +/// This function will not do anything with invalid MAR entries (we assume they are "under construction"). +pub fn collect_and_upload( + mar_staging: &Path, + client: &impl NetworkClient, + max_zip_size: usize, + sampling: Sampling, +) -> Result { + let mut entries = MarEntry::iterate_from_container(mar_staging)? + // Apply fleet sampling to the MAR entries + .filter(|entry_result| match entry_result { + Ok(entry) => should_upload(&entry.manifest.metadata, &sampling), + _ => true, + }); + + upload_mar_entries(&mut entries, client, max_zip_size, |included_entries| { + trace!("Uploaded {:?} - deleting...", included_entries); + included_entries.iter().for_each(|f| { + let _ = remove_dir_all(f); + }) + }) +} + +/// Given the current sampling configuration determine if the given MAR entry should be uploaded. +fn should_upload(metadata: &Metadata, sampling: &Sampling) -> bool { + match metadata { + Metadata::DeviceAttributes { .. } => sampling.monitoring_resolution >= Resolution::Normal, + Metadata::DeviceConfig { .. } => true, // Always upload device config + Metadata::ElfCoredump { .. } => sampling.debugging_resolution >= Resolution::Normal, + Metadata::LinuxHeartbeat { .. } => sampling.monitoring_resolution >= Resolution::Normal, + Metadata::LinuxMetricReport { report_type, .. } => match report_type { + MetricReportType::Heartbeat => sampling.monitoring_resolution >= Resolution::Normal, + MetricReportType::Session(_) => sampling.monitoring_resolution >= Resolution::Normal, + MetricReportType::DailyHeartbeat => sampling.monitoring_resolution >= Resolution::Low, + }, + Metadata::LinuxLogs { .. } => sampling.logging_resolution >= Resolution::Normal, + Metadata::LinuxReboot { .. } => true, // Always upload reboots + Metadata::LinuxMemfaultWatch { exit_code, .. } => { + let is_crash = exit_code != &0; + if is_crash { + sampling.debugging_resolution >= Resolution::Normal + || sampling.logging_resolution >= Resolution::Normal + } else { + sampling.logging_resolution >= Resolution::Normal + } + } + } +} + +/// Describes the contents for a single MAR file to upload. +pub struct MarZipContents { + /// All the MAR entry directories to to be included in this file. + pub entry_paths: Vec, + /// All the ZipEntryInfos to be included in this file. + pub zip_infos: Vec, +} + +/// Gather MAR entries and associated ZipEntryInfos, consuming items from the iterator. +/// +/// Return a list of MarZipContents, each containing the list of folders that are included in the +/// zip (and can be deleted after upload) and the list of ZipEntryInfos. +/// Invalid folders will not trigger an error and they will not be included in the returned lists. +pub fn gather_mar_entries_to_zip( + entries: &mut impl Iterator>, + max_zip_size: usize, +) -> Vec { + let entry_paths_with_zip_infos = entries.filter_map(|entry_result| match entry_result { + Ok(entry) => { + trace!("Adding {:?}", &entry.path); + let zip_infos: Option> = (&entry) + .try_into() + .wrap_err_with(|| format!("Unable to add entry {}.", &entry.path.display())) + .ok(); + let entry_and_infos: Option<(PathBuf, Vec)> = + zip_infos.map(|infos| (entry.path, infos)); + entry_and_infos + } + Err(e) => { + warn!("Invalid folder in MAR staging: {:?}", e); + None + } + }); + + let mut zip_size = zip_stream_len_empty(); + let mut zip_file_index: usize = 0; + let grouper = entry_paths_with_zip_infos.group_by(|(_, zip_infos)| { + let entry_zipped_size = zip_infos.iter().map(zip_stream_len_for_file).sum::(); + if zip_size + entry_zipped_size > max_zip_size { + zip_size = zip_stream_len_empty() + entry_zipped_size; + zip_file_index += 1; + } else { + zip_size += entry_zipped_size; + } + zip_file_index + }); + + grouper + .into_iter() + .map(|(_zip_file_index, group)| { + // Convert from Vec<(PathBuf, Vec)> to MarZipContents: + let (entry_paths, zip_infos): (Vec, Vec>) = group.unzip(); + MarZipContents { + entry_paths, + zip_infos: zip_infos + .into_iter() + .flatten() + .collect::>(), + } + }) + .collect() +} + +impl TryFrom<&MarEntry> for Vec { + type Error = eyre::Error; + + fn try_from(entry: &MarEntry) -> Result { + let entry_path = entry.path.clone(); + entry + .filenames() + .map(move |filename| { + let path = entry_path.join(&filename); + + // Open the file to check that it exists and is readable. This is a best effort to avoid + // starting to upload a MAR file only to find out half way through that a file was not + // readable. Yes, this is prone to a race condition where it is no longer readable by + // the time is going to be read by the zip writer, but it is better than nothing. + let file = + File::open(&path).wrap_err_with(|| format!("Error opening {:?}", filename))?; + drop(file); + + let base = entry_path.parent().unwrap(); + ZipEntryInfo::new(path, base) + .wrap_err_with(|| format!("Error adding {:?}", filename)) + }) + .collect::>>() + } +} + +/// Progressively upload the MAR entries. The callback will be called for each batch that is uploaded. +fn upload_mar_entries( + entries: &mut impl Iterator>, + client: &impl NetworkClient, + max_zip_size: usize, + callback: fn(entries: Vec) -> (), +) -> Result { + let zip_files = gather_mar_entries_to_zip(entries, max_zip_size); + let count = zip_files.len(); + + for MarZipContents { + entry_paths, + zip_infos, + } in zip_files.into_iter() + { + client.upload_mar_file(BufReader::new(ZipEncoder::new(zip_infos)))?; + callback(entry_paths); + } + Ok(count) +} + +#[cfg(test)] +mod tests { + use rstest::{fixture, rstest}; + use std::str::FromStr; + use std::{ + collections::HashMap, + time::{Duration, SystemTime}, + }; + + use crate::reboot::{RebootReason, RebootReasonCode}; + use crate::{ + mar::test_utils::{assert_mar_content_matches, MarCollectorFixture}, + metrics::SessionName, + network::MockNetworkClient, + }; + use crate::{ + metrics::{MetricStringKey, MetricValue}, + test_utils::setup_logger, + }; + + use super::*; + + #[rstest] + fn collecting_from_empty_folder(_setup_logger: (), mar_fixture: MarCollectorFixture) { + assert_eq!( + MarEntry::iterate_from_container(&mar_fixture.mar_staging) + .unwrap() + .count(), + 0 + ) + } + + #[rstest] + fn collecting_from_folder_with_partial_entries( + _setup_logger: (), + mut mar_fixture: MarCollectorFixture, + ) { + mar_fixture.create_empty_entry(); + mar_fixture.create_logentry(); + + assert_eq!( + MarEntry::iterate_from_container(&mar_fixture.mar_staging) + .unwrap() + .filter(|e| e.is_ok()) + .count(), + // Only one entry should be picked up. The other one is ignored. + 1 + ) + } + + #[rstest] + fn zipping_two_entries(_setup_logger: (), mut mar_fixture: MarCollectorFixture) { + // Add one valid entry so we can verify that this one is readable. + mar_fixture.create_logentry(); + mar_fixture.create_logentry(); + + let mut entries = MarEntry::iterate_from_container(&mar_fixture.mar_staging) + .expect("We should still be able to collect."); + + let mars = gather_mar_entries_to_zip(&mut entries, usize::MAX); + + assert_eq!(mars.len(), 1); + assert_eq!(mars[0].entry_paths.len(), 2); + assert_eq!(mars[0].zip_infos.len(), 4); // for each entry: manifest.json + log file + } + + #[rstest] + #[case::not_json(MarCollectorFixture::create_entry_with_bogus_json)] + #[case::unreadable_dir(MarCollectorFixture::create_entry_without_directory_read_permission)] + #[case::unreadable_manifest(MarCollectorFixture::create_entry_without_manifest_read_permission)] + fn zipping_with_skipped_entries( + _setup_logger: (), + mut mar_fixture: MarCollectorFixture, + #[case] create_bogus_entry: fn(&mut MarCollectorFixture) -> PathBuf, + ) { + create_bogus_entry(&mut mar_fixture); + // Add one valid entry so we can verify that this one is readable. + mar_fixture.create_logentry(); + + let mut entries = MarEntry::iterate_from_container(&mar_fixture.mar_staging) + .expect("We should still be able to collect."); + + let mars = gather_mar_entries_to_zip(&mut entries, usize::MAX); + + assert_eq!(mars.len(), 1); + assert_eq!(mars[0].entry_paths.len(), 1); + assert_eq!(mars[0].zip_infos.len(), 2); // manifest.json + log file + } + + #[rstest] + fn zipping_an_unreadable_attachment(_setup_logger: (), mut mar_fixture: MarCollectorFixture) { + // Add one valid entry so we can verify that this one is readable. + mar_fixture.create_logentry_with_unreadable_attachment(); + + let mut entries = MarEntry::iterate_from_container(&mar_fixture.mar_staging) + .expect("We should still be able to collect."); + + let mars = gather_mar_entries_to_zip(&mut entries, usize::MAX); + + // No MAR should be created because the attachment is unreadable. + assert_eq!(mars.len(), 0); + } + + #[rstest] + fn new_mar_when_size_limit_is_reached(_setup_logger: (), mut mar_fixture: MarCollectorFixture) { + let max_zip_size = 1024; + mar_fixture.create_logentry_with_size(max_zip_size / 2); + mar_fixture.create_logentry_with_size(max_zip_size); + // Note: the next entry exceeds the size limit, but it is still added to a MAR of its own: + mar_fixture.create_logentry_with_size(max_zip_size * 2); + + let mut entries = MarEntry::iterate_from_container(&mar_fixture.mar_staging) + .expect("We should still be able to collect."); + + let mars = gather_mar_entries_to_zip(&mut entries, max_zip_size as usize); + + // 3 MARs should be created because the size limit was reached after every entry: + assert_eq!(mars.len(), 3); + for contents in mars { + assert_eq!(contents.entry_paths.len(), 1); + assert_eq!(contents.zip_infos.len(), 2); // for each entry: manifest.json + log file + } + } + + #[rstest] + fn uploading_empty_list( + _setup_logger: (), + client: MockNetworkClient, + mar_fixture: MarCollectorFixture, + ) { + // We do not set an expectation on client => it will panic if client.upload_mar is called + collect_and_upload( + &mar_fixture.mar_staging, + &client, + usize::MAX, + Sampling { + debugging_resolution: Resolution::Normal, + logging_resolution: Resolution::Normal, + monitoring_resolution: Resolution::Normal, + }, + ) + .unwrap(); + } + + #[rstest] + #[case::off(Resolution::Off, false)] + #[case::low(Resolution::Low, false)] + #[case::normal(Resolution::Normal, true)] + #[case::high(Resolution::High, true)] + fn uploading_logs( + #[case] resolution: Resolution, + #[case] should_upload: bool, + _setup_logger: (), + client: MockNetworkClient, + mut mar_fixture: MarCollectorFixture, + ) { + mar_fixture.create_logentry(); + + let expected_files = + should_upload.then(|| vec!["/manifest.json", "/system.log"]); + let sampling_config = Sampling { + debugging_resolution: Resolution::Off, + logging_resolution: resolution, + monitoring_resolution: Resolution::Off, + }; + upload_and_verify(mar_fixture, client, sampling_config, expected_files); + } + + #[rstest] + #[case::off(Resolution::Off, false)] + #[case::low(Resolution::Low, false)] + #[case::normal(Resolution::Normal, true)] + #[case::high(Resolution::High, true)] + fn uploading_device_attributes( + #[case] resolution: Resolution, + #[case] should_upload: bool, + _setup_logger: (), + client: MockNetworkClient, + mut mar_fixture: MarCollectorFixture, + ) { + mar_fixture.create_device_attributes_entry(vec![], SystemTime::now()); + + let sampling_config = Sampling { + debugging_resolution: Resolution::Off, + logging_resolution: Resolution::Off, + monitoring_resolution: resolution, + }; + let expected_files = should_upload.then(|| vec!["/manifest.json"]); + upload_and_verify(mar_fixture, client, sampling_config, expected_files); + } + + #[rstest] + // Verify that reboots are always uploaded + #[case::off(Resolution::Off, true)] + #[case::low(Resolution::Low, true)] + #[case::normal(Resolution::Normal, true)] + #[case::high(Resolution::High, true)] + fn uploading_reboots( + #[case] resolution: Resolution, + #[case] should_upload: bool, + _setup_logger: (), + client: MockNetworkClient, + mut mar_fixture: MarCollectorFixture, + ) { + mar_fixture.create_reboot_entry(RebootReason::Code(RebootReasonCode::Unknown)); + + let sampling_config = Sampling { + debugging_resolution: resolution, + logging_resolution: Resolution::Off, + monitoring_resolution: Resolution::Off, + }; + let expected_files = should_upload.then(|| vec!["/manifest.json"]); + upload_and_verify(mar_fixture, client, sampling_config, expected_files); + } + + #[rstest] + // Heartbeat cases + #[case::heartbeat_off(MetricReportType::Heartbeat, Resolution::Off, false)] + #[case::heartbeat_low(MetricReportType::Heartbeat, Resolution::Low, false)] + #[case::heartbeat_normal(MetricReportType::Heartbeat, Resolution::Normal, true)] + #[case::heartbeat_high(MetricReportType::Heartbeat, Resolution::High, true)] + // Daily heartbeat cases + #[case::daily_heartbeat_off(MetricReportType::DailyHeartbeat, Resolution::Off, false)] + #[case::daily_heartbeat_low(MetricReportType::DailyHeartbeat, Resolution::Low, true)] + #[case::daily_heartbeat_normal(MetricReportType::DailyHeartbeat, Resolution::Normal, true)] + #[case::daily_heartbeat_high(MetricReportType::DailyHeartbeat, Resolution::High, true)] + // Session cases + #[case::session_off( + MetricReportType::Session(SessionName::from_str("test").unwrap()), + Resolution::Off, + false + )] + #[case::session_low( + MetricReportType::Session(SessionName::from_str("test").unwrap()), + Resolution::Low, + false + )] + #[case::session_normal( + MetricReportType::Session(SessionName::from_str("test").unwrap()), + Resolution::Normal, + true + )] + #[case::session_high( + MetricReportType::Session(SessionName::from_str("test").unwrap()), + Resolution::High, + true + )] + fn uploading_metric_reports( + #[case] report_type: MetricReportType, + #[case] resolution: Resolution, + #[case] should_upload: bool, + _setup_logger: (), + client: MockNetworkClient, + mut mar_fixture: MarCollectorFixture, + ) { + let duration = Duration::from_secs(1); + let metrics: HashMap = vec![( + MetricStringKey::from_str("foo").unwrap(), + MetricValue::Number(1.0), + )] + .into_iter() + .collect(); + + mar_fixture.create_metric_report_entry(metrics, duration, report_type); + + let sampling_config = Sampling { + debugging_resolution: Resolution::Off, + logging_resolution: Resolution::Off, + monitoring_resolution: resolution, + }; + let expected_files = should_upload.then(|| vec!["/manifest.json"]); + upload_and_verify(mar_fixture, client, sampling_config, expected_files); + } + + fn upload_and_verify( + mar_fixture: MarCollectorFixture, + mut client: MockNetworkClient, + sampling_config: Sampling, + expected_files: Option>, + ) { + if let Some(expected_files) = expected_files { + client + .expect_upload_mar_file::>() + .withf(move |buf_reader| { + let zip_encoder = buf_reader.get_ref(); + assert_mar_content_matches(zip_encoder, expected_files.clone()) + }) + .once() + .returning(|_| Ok(())); + } + collect_and_upload( + &mar_fixture.mar_staging, + &client, + usize::MAX, + sampling_config, + ) + .unwrap(); + } + + #[fixture] + fn client() -> MockNetworkClient { + MockNetworkClient::default() + } + + #[fixture] + fn mar_fixture() -> MarCollectorFixture { + MarCollectorFixture::new() + } +} diff --git a/memfaultd/src/memfaultd.rs b/memfaultd/src/memfaultd.rs new file mode 100644 index 0000000..3a7840b --- /dev/null +++ b/memfaultd/src/memfaultd.rs @@ -0,0 +1,534 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::process::Command; +use std::sync::Arc; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Mutex, +}; +use std::thread::{sleep, spawn}; +use std::time::Duration; +use std::{fs::create_dir_all, time::Instant}; + +use eyre::Result; +use eyre::{eyre, Context}; +use log::{error, info, trace, warn}; + +use crate::metrics::{ + BatteryMonitor, BatteryReadingHandler, ConnectivityMonitor, MetricReportType, + PeriodicMetricReportDumper, ReportSyncEventHandler, SessionEventHandler, + SystemMetricsCollector, +}; + +use crate::{ + config::Config, + mar::upload::collect_and_upload, + metrics::{ + core_metrics::{METRIC_MF_SYNC_FAILURE, METRIC_MF_SYNC_SUCCESS}, + CrashFreeIntervalTracker, MetricReportManager, StatsDServer, + }, +}; +use crate::{http_server::HttpHandler, util::UpdateStatus}; +use crate::{ + http_server::HttpServer, + network::{NetworkClientImpl, NetworkConfig}, +}; +use crate::{ + mar::MarExportHandler, + util::{ + can_connect::TcpConnectionChecker, + task::{loop_with_exponential_error_backoff, LoopContinuation}, + }, +}; +use crate::{mar::MarStagingCleaner, service_manager::get_service_manager}; +use crate::{reboot::RebootReasonTracker, util::disk_size::DiskSize}; + +#[cfg(feature = "collectd")] +use crate::collectd::CollectdHandler; + +#[cfg(feature = "logging")] +use crate::{ + fluent_bit::{FluentBitConfig, FluentBitConnectionHandler}, + logs::{CompletedLog, FluentBitAdapter, HeadroomLimiter, LogCollector, LogCollectorConfig}, + mar::{MarEntryBuilder, Metadata}, + util::disk_size::get_disk_space, +}; + +const CONFIG_REFRESH_INTERVAL: Duration = Duration::from_secs(60 * 120); +const DAILY_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(60 * 60 * 24); + +#[derive(PartialEq, Eq)] +pub enum MemfaultLoopResult { + Terminate, + Relaunch, +} + +pub fn memfaultd_loop Result<()>>( + config: Config, + ready_callback: C, +) -> Result { + // Register a flag which will be set when one of these signals is received. + let term_signals = [signal_hook::consts::SIGINT, signal_hook::consts::SIGTERM]; + let term = Arc::new(AtomicBool::new(false)); + for signal in term_signals { + signal_hook::flag::register(signal, Arc::clone(&term))?; + } + + // This flag will be set when we get the SIGHUP signal to reload (currently reload = restart) + let reload = Arc::new(AtomicBool::new(false)); + signal_hook::flag::register(signal_hook::consts::SIGHUP, Arc::clone(&reload))?; + + // Register a flag to be set when we are woken up by SIGUSR1 + let force_sync = Arc::new(AtomicBool::new(false)); + signal_hook::flag::register(signal_hook::consts::SIGUSR1, Arc::clone(&force_sync))?; + + // Load configuration and device information. This has already been done by the C code but + // we are preparing for a future where there is no more C code. + let client = NetworkClientImpl::new(NetworkConfig::from(&config)) + .wrap_err(eyre!("Unable to prepare network client"))?; + + let service_manager = get_service_manager(); + + // Make sure the MAR staging area exists + create_dir_all(config.mar_staging_path()).wrap_err_with(|| { + eyre!( + "Unable to create MAR staging area {}", + &config.mar_staging_path().display(), + ) + })?; + + let mar_cleaner = Arc::new(MarStagingCleaner::new( + &config.mar_staging_path(), + config.tmp_dir_max_size(), + config.tmp_dir_min_headroom(), + config.mar_entry_max_age(), + )); + + // List of tasks to run before syncing with server + let mut sync_tasks: Vec Result<()>>> = vec![]; + // List of tasks to run before shutting down + let mut shutdown_tasks: Vec Result<()>>> = vec![]; + + // List of http handlers + #[allow(unused_mut, /* reason = "Can be unused when some features are disabled." */)] + let mut http_handlers: Vec> = + vec![Box::new(MarExportHandler::new(config.mar_staging_path()))]; + + // Metric store + let metric_report_manager = match config.session_configs() { + Some(session_configs) => Arc::new(Mutex::new( + MetricReportManager::new_with_session_configs(session_configs), + )), + None => Arc::new(Mutex::new(MetricReportManager::new())), + }; + + let battery_monitor = Arc::new(Mutex::new(BatteryMonitor::::new( + metric_report_manager.clone(), + ))); + let battery_reading_handler = BatteryReadingHandler::new( + config.config_file.enable_data_collection, + battery_monitor.clone(), + ); + http_handlers.push(Box::new(battery_reading_handler)); + + let report_sync_event_handler = ReportSyncEventHandler::new( + config.config_file.enable_data_collection, + metric_report_manager.clone(), + ); + http_handlers.push(Box::new(report_sync_event_handler)); + + let session_event_handler = SessionEventHandler::new( + config.config_file.enable_data_collection, + metric_report_manager.clone(), + config.mar_staging_path(), + NetworkConfig::from(&config), + ); + http_handlers.push(Box::new(session_event_handler)); + + #[cfg(feature = "collectd")] + { + let collectd_handler = CollectdHandler::new( + config.config_file.enable_data_collection, + config.builtin_system_metric_collection_enabled(), + metric_report_manager.clone(), + ); + http_handlers.push(Box::new(collectd_handler)); + } + + // Start a thread to dump the metrics precisely every heartbeat interval + { + let net_config = NetworkConfig::from(&config); + let mar_staging_path = config.mar_staging_path(); + let heartbeat_interval = config.config_file.heartbeat_interval; + let metric_report_manager = metric_report_manager.clone(); + spawn(move || { + let periodic_metric_report_dumper = PeriodicMetricReportDumper::new( + mar_staging_path, + net_config, + metric_report_manager, + heartbeat_interval, + MetricReportType::Heartbeat, + ); + + periodic_metric_report_dumper.start(); + }); + } + + // Start statsd server + if config.statsd_server_enabled() { + let statsd_server = StatsDServer::new(); + if let Ok(bind_address) = config.statsd_server_address() { + if let Err(e) = statsd_server.start(bind_address, metric_report_manager.clone()) { + warn!("Couldn't start StatsD server: {}", e); + }; + } + } + + // Start system metric collector thread + if config.builtin_system_metric_collection_enabled() { + let metric_report_manager = metric_report_manager.clone(); + let poll_interval = config.system_metric_poll_interval(); + let system_metric_config = config.system_metric_config(); + spawn(move || { + let mut sys_metric_collector = SystemMetricsCollector::new(system_metric_config); + sys_metric_collector.run(poll_interval, metric_report_manager) + }); + } + + // Start a thread to dump metrics every 24 hours + if config.config_file.metrics.enable_daily_heartbeats { + let net_config = NetworkConfig::from(&config); + let mar_staging_path = config.mar_staging_path(); + let metric_report_manager = metric_report_manager.clone(); + spawn(move || { + let periodic_metric_report_dumper = PeriodicMetricReportDumper::new( + mar_staging_path, + net_config, + metric_report_manager, + DAILY_HEARTBEAT_INTERVAL, + MetricReportType::DailyHeartbeat, + ); + + periodic_metric_report_dumper.start(); + }); + } + + // Start a thread to update battery metrics + // periodically if enabled by configuration + if config.battery_monitor_periodic_update_enabled() { + let battery_monitor_interval = config.battery_monitor_interval(); + let battery_info_command_str = config.battery_monitor_battery_info_command().to_string(); + spawn(move || { + let mut next_battery_interval = Instant::now() + battery_monitor_interval; + loop { + while Instant::now() < next_battery_interval { + sleep(next_battery_interval - Instant::now()); + } + next_battery_interval += battery_monitor_interval; + let battery_info_command = Command::new(&battery_info_command_str); + if let Err(e) = battery_monitor + .lock() + .unwrap() + .update_via_command(battery_info_command) + { + warn!("Error updating battery monitor metrics: {}", e); + } + } + }); + } + // Connected time monitor is only enabled if config is defined + if let Some(connectivity_monitor_config) = config.connectivity_monitor_config() { + let mut connectivity_monitor = ConnectivityMonitor::::new( + connectivity_monitor_config, + metric_report_manager.clone(), + ); + spawn(move || { + let mut next_connectivity_reading_time = + Instant::now() + connectivity_monitor.interval_seconds(); + loop { + while Instant::now() < next_connectivity_reading_time { + sleep(next_connectivity_reading_time - Instant::now()); + } + next_connectivity_reading_time += connectivity_monitor.interval_seconds(); + if let Err(e) = connectivity_monitor.update_connected_time() { + warn!("Failed to update connected time metrics: {}", e); + } + } + }); + } + // Schedule a task to dump the metrics when a sync is forced + { + let net_config = NetworkConfig::from(&config); + let mar_staging_path = config.mar_staging_path(); + + let heartbeat_report_manager = metric_report_manager.clone(); + sync_tasks.push(Box::new(move |forced| match forced { + true => MetricReportManager::dump_report_to_mar_entry( + &heartbeat_report_manager, + &mar_staging_path, + &net_config, + &MetricReportType::Heartbeat, + ), + false => Ok(()), + })); + + let daily_heartbeat_report_manager = metric_report_manager.clone(); + let net_config = NetworkConfig::from(&config); + let mar_staging_path = config.mar_staging_path(); + sync_tasks.push(Box::new(move |forced| match forced { + true => { + trace!("Dumping daily heartbeat metrics"); + MetricReportManager::dump_report_to_mar_entry( + &daily_heartbeat_report_manager, + &mar_staging_path, + &net_config, + &MetricReportType::DailyHeartbeat, + ) + } + false => Ok(()), + })); + } + // Schedule a task to dump the metrics when we are shutting down + { + let net_config = NetworkConfig::from(&config); + let mar_staging_path = config.mar_staging_path(); + + let metric_report_manager = metric_report_manager.clone(); + shutdown_tasks.push(Box::new(move || { + MetricReportManager::dump_metric_reports( + &metric_report_manager, + &mar_staging_path, + &net_config, + ) + })); + } + // Schedule a task to compute operational and crashfree hours + { + let metric_report_manager = metric_report_manager.clone(); + + let mut crashfree_tracker = CrashFreeIntervalTracker::::new_hourly(); + http_handlers.push(crashfree_tracker.http_handler()); + spawn(move || { + let interval = Duration::from_secs(60); + loop { + let metrics = crashfree_tracker.wait_and_update(interval); + let mut store = metric_report_manager.lock().unwrap(); + trace!("Crashfree hours metrics: {:?}", metrics); + for m in metrics { + if let Err(e) = store.add_metric(m) { + warn!("Unable to add crashfree metric: {}", e); + } + } + } + }); + } + + #[cfg(feature = "logging")] + { + use crate::config::LogSource; + #[cfg(feature = "systemd")] + use crate::logs::journald_provider::start_journald_provider; + use crate::logs::log_entry::LogEntry; + use log::debug; + + let fluent_bit_config = FluentBitConfig::from(&config); + if config.config_file.enable_data_collection { + let log_source = config.config_file.logs.source; + let log_receiver: Box + Send> = match log_source { + LogSource::FluentBit => { + let (_, fluent_bit_receiver) = + FluentBitConnectionHandler::start(fluent_bit_config)?; + Box::new(FluentBitAdapter::new( + fluent_bit_receiver, + &config.config_file.fluent_bit.extra_fluentd_attributes, + )) + } + #[cfg(feature = "systemd")] + LogSource::Journald => Box::new(start_journald_provider(config.tmp_dir())), + }; + + let mar_cleaner = mar_cleaner.clone(); + + let network_config = NetworkConfig::from(&config); + let mar_staging_path = config.mar_staging_path(); + let on_log_completion = move |CompletedLog { + path, + cid, + next_cid, + compression, + }| + -> Result<()> { + // Prepare the MAR entry + let file_name = path + .file_name() + .ok_or(eyre!("Logfile should be a file."))? + .to_str() + .ok_or(eyre!("Invalid log filename."))? + .to_owned(); + let mar_builder = MarEntryBuilder::new(&mar_staging_path)? + .set_metadata(Metadata::new_log(file_name, cid, next_cid, compression)) + .add_attachment(path); + + mar_cleaner.clean(mar_builder.estimated_entry_size())?; + + // Move the log in the mar_staging area and add a manifest + let mar_entry = mar_builder.save(&network_config)?; + debug!( + "Logfile (cid: {}) saved as MAR entry: {}", + cid, + mar_entry.path.display() + ); + + Ok(()) + }; + let log_config = LogCollectorConfig::from(&config); + let headroom_limiter = { + let tmp_folder = log_config.log_tmp_path.clone(); + HeadroomLimiter::new(config.tmp_dir_min_headroom(), move || { + get_disk_space(&tmp_folder) + }) + }; + let mut log_collector = LogCollector::open( + log_config, + on_log_completion, + headroom_limiter, + metric_report_manager.clone(), + )?; + log_collector.spawn_collect_from(log_receiver); + + let crash_log_handler = log_collector.crash_log_handler(); + http_handlers.push(Box::new(crash_log_handler)); + + sync_tasks.push(Box::new(move |forced_sync| { + // Check if we have received a signal to force-sync and reset the flag. + if forced_sync { + trace!("Flushing logs"); + log_collector.flush_logs()?; + } else { + // If not force-flushing - we still want to make sure this file + // did not get too old. + log_collector.rotate_if_needed()?; + } + Ok(()) + })); + } else { + FluentBitConnectionHandler::start_null(fluent_bit_config)?; + } + } + + let reboot_tracker = RebootReasonTracker::new(&config, &service_manager); + if let Err(e) = reboot_tracker.track_reboot() { + error!("Unable to track reboot reason: {:#}", e); + } + + // Start the http server + let mut http_server = HttpServer::new(http_handlers); + http_server.start(config.config_file.http_server.bind_address)?; + + // Run the ready callback (creates the PID file) + ready_callback()?; + + let mut last_device_config_refresh = Option::::None; + + // If upload_interval is zero, we are only uploading on manual syncs. + let forced_sync_only = config.config_file.upload_interval.is_zero(); + // If we are only uploading on manual syncs, we still need to run the mar cleaner periodically. In + // this case set the the upload interval to 15 minutes. + let upload_interval = if forced_sync_only { + Duration::from_secs(60 * 15) + } else { + config.config_file.upload_interval + }; + loop_with_exponential_error_backoff( + || { + // Reset the forced sync flag before doing any work so we can detect + // if it's set again while we run and RerunImmediately. + let forced = force_sync.swap(false, Ordering::Relaxed); + let enable_data_collection = config.config_file.enable_data_collection; + + // Refresh device config if needed. In cases where we are only syncing on demand, we + // short-circuit this check. + if enable_data_collection + && (!forced_sync_only + && (last_device_config_refresh.is_none() + || last_device_config_refresh.unwrap() + CONFIG_REFRESH_INTERVAL + < Instant::now()) + || forced) + { + // Refresh device config from the server + match config.refresh_device_config(&client) { + Err(e) => { + warn!("Unable to refresh device config: {}", e); + // We continue processing the pending uploads on errors. + // We expect rate limiting errors here. + } + Ok(UpdateStatus::Updated) => { + info!("Device config updated"); + last_device_config_refresh = Some(Instant::now()) + } + Ok(UpdateStatus::Unchanged) => { + trace!("Device config unchanged"); + last_device_config_refresh = Some(Instant::now()) + } + } + } + + for task in &mut sync_tasks { + if let Err(e) = task(forced) { + warn!("{:#}", e); + } + } + + mar_cleaner.clean(DiskSize::ZERO).unwrap(); + + if enable_data_collection && !forced_sync_only || forced { + trace!("Collect MAR entries..."); + let result = collect_and_upload( + &config.mar_staging_path(), + &client, + config.config_file.mar.mar_file_max_size, + config.sampling(), + ); + let _metric_result = match result { + Ok(0) => Ok(()), + Ok(_count) => metric_report_manager + .lock() + .unwrap() + .increment_counter(METRIC_MF_SYNC_SUCCESS), + Err(_) => metric_report_manager + .lock() + .unwrap() + .increment_counter(METRIC_MF_SYNC_FAILURE), + }; + return result.map(|_| ()); + } + Ok(()) + }, + || match ( + term.load(Ordering::Relaxed) || reload.load(Ordering::Relaxed), + force_sync.load(Ordering::Relaxed), + ) { + // Stop when we receive a term signal + (true, _) => LoopContinuation::Stop, + // If we received a SIGUSR1 signal while we were in the loop, rerun immediately. + (false, true) => LoopContinuation::RerunImmediately, + // Otherwise, keep running normally + (false, false) => LoopContinuation::KeepRunning, + }, + upload_interval, + Duration::new(60, 0), + ); + info!("Memfaultd shutting down..."); + for task in &mut shutdown_tasks { + if let Err(e) = task() { + warn!("Error while shutting down: {}", e); + } + } + + if reload.load(Ordering::Relaxed) { + Ok(MemfaultLoopResult::Relaunch) + } else { + Ok(MemfaultLoopResult::Terminate) + } +} diff --git a/memfaultd/src/metrics/battery/battery_monitor.rs b/memfaultd/src/metrics/battery/battery_monitor.rs new file mode 100644 index 0000000..bbb84af --- /dev/null +++ b/memfaultd/src/metrics/battery/battery_monitor.rs @@ -0,0 +1,365 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use chrono::{DateTime, Utc}; + +use std::ops::Sub; +use std::process::Command; +use std::str::FromStr; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use eyre::{eyre, ErrReport, Result}; + +use crate::metrics::{ + core_metrics::{ + METRIC_BATTERY_DISCHARGE_DURATION_MS, METRIC_BATTERY_SOC_PCT, METRIC_BATTERY_SOC_PCT_DROP, + }, + KeyedMetricReading, MetricReading, MetricReportManager, MetricStringKey, +}; +use crate::util::time_measure::TimeMeasure; + +// These states are based off the valid values for +// sys/class/power_supply//status +// Read more here: +// https://www.kernel.org/doc/Documentation/ABI/testing/sysfs-class-power +enum ChargingState { + Charging, + Discharging, + Full, + Unknown, + NotCharging, + Invalid, +} + +// A single reading that describes +// the state of the device's battery +pub struct BatteryMonitorReading { + battery_soc_pct: f64, + battery_charging_state: ChargingState, +} + +impl BatteryMonitorReading { + fn new(battery_soc_pct: f64, battery_charging_state: ChargingState) -> BatteryMonitorReading { + BatteryMonitorReading { + battery_soc_pct, + battery_charging_state, + } + } +} + +impl FromStr for BatteryMonitorReading { + type Err = ErrReport; + + fn from_str(s: &str) -> Result { + if let Some((state_str, pct_str)) = s.trim().split_once(':') { + let charging_state = match state_str { + "Charging" => ChargingState::Charging, + "Discharging" => ChargingState::Discharging, + "Full" => ChargingState::Full, + "Not charging" => ChargingState::NotCharging, + "Unknown" => ChargingState::Unknown, + _ => ChargingState::Invalid, + }; + let pct = pct_str.parse::(); + match (charging_state, pct) { + (ChargingState::Invalid, _) => Err(eyre!("Invalid charging state: {}", state_str)), + (_, Err(e)) => Err(eyre!("Couldn't parse battery percentage: {}", e)), + (charging_state, Ok(p)) => { + if (0.0..=100.0).contains(&p) { + Ok(BatteryMonitorReading::new(p, charging_state)) + } else { + Err(eyre!( + "Battery SOC percentage value {} is not in the range [0.0, 100.0]!", + p + )) + } + } + } + } else { + Err(eyre!( + "Invalid output from command configured via `battery_percentage_command`" + )) + } + } +} + +// Since some of the battery metrics recorded +// are calculated based on both the current and previous reading +// (such as battery_soc_pct_drop), this struct needs to +// store the previous battery percentage as well as when that +// perecentage was recorded. +pub struct BatteryMonitor { + previous_reading: Option, + last_reading_time: T, + heartbeat_manager: Arc>, +} + +impl BatteryMonitor +where + T: TimeMeasure + Copy + Ord + Sub, +{ + pub fn new(heartbeat_manager: Arc>) -> Self { + Self { + previous_reading: None, + last_reading_time: T::now(), + heartbeat_manager, + } + } + + // Writes new values for battery_discharge_duration_ms, battery_soc_pct_drop, + // and battery_soc_pct to the in memory metric store + fn update_metrics( + &mut self, + battery_monitor_reading: BatteryMonitorReading, + reading_time: T, + wall_time: DateTime, + ) -> Result<()> { + let reading_duration = reading_time.since(&self.last_reading_time); + match ( + &battery_monitor_reading.battery_charging_state, + &self.previous_reading, + ) { + // Update battery discharge metrics only when there is a previous + // reading and both the previous AND current + // charging state are Discharging + ( + ChargingState::Discharging, + Some(BatteryMonitorReading { + battery_soc_pct: previous_soc_pct, + battery_charging_state: ChargingState::Discharging, + }), + ) => { + let soc_pct = battery_monitor_reading.battery_soc_pct; + let soc_pct_discharged = + (previous_soc_pct - battery_monitor_reading.battery_soc_pct).max(0.0); + + let mut heartbeat_manager = self.heartbeat_manager.lock().expect("Mutex Poisoned"); + heartbeat_manager.add_to_counter( + METRIC_BATTERY_DISCHARGE_DURATION_MS, + reading_duration.as_millis() as f64, + )?; + + heartbeat_manager + .add_to_counter(METRIC_BATTERY_SOC_PCT_DROP, soc_pct_discharged)?; + + let battery_soc_pct_key = MetricStringKey::from_str(METRIC_BATTERY_SOC_PCT) + .unwrap_or_else(|_| panic!("Invalid metric name: {}", METRIC_BATTERY_SOC_PCT)); + heartbeat_manager.add_metric(KeyedMetricReading::new( + battery_soc_pct_key, + MetricReading::TimeWeightedAverage { + value: soc_pct, + timestamp: wall_time, + interval: chrono::Duration::from_std(reading_duration)?, + }, + ))?; + } + // In all other cases only update the SoC percent + _ => { + let soc_pct = battery_monitor_reading.battery_soc_pct; + + let mut heartbeat_manager = self.heartbeat_manager.lock().expect("Mutex Poisoned"); + + // Add 0.0 to these counters so if the device is charging + // for the full heartbeat duration these metrics are still + // populated + heartbeat_manager.add_to_counter(METRIC_BATTERY_DISCHARGE_DURATION_MS, 0.0)?; + heartbeat_manager.add_to_counter(METRIC_BATTERY_SOC_PCT_DROP, 0.0)?; + + let battery_soc_pct_key = MetricStringKey::from_str(METRIC_BATTERY_SOC_PCT) + .unwrap_or_else(|_| panic!("Invalid metric name: {}", METRIC_BATTERY_SOC_PCT)); + heartbeat_manager.add_metric(KeyedMetricReading::new( + battery_soc_pct_key, + MetricReading::TimeWeightedAverage { + value: soc_pct, + timestamp: wall_time, + interval: chrono::Duration::from_std(reading_duration)?, + }, + ))?; + } + } + + self.previous_reading = Some(battery_monitor_reading); + self.last_reading_time = reading_time; + + Ok(()) + } + + pub fn update_via_command(&mut self, mut battery_info_command: Command) -> Result<()> { + let battery_info_output = battery_info_command.output()?; + if !battery_info_output.status.success() { + Err(eyre!( + "Failed to execute {}. Battery percentage was not captured.", + battery_info_command.get_program().to_string_lossy() + )) + } else { + let output_string = String::from_utf8(battery_info_output.stdout)?; + let battery_monitor_reading = BatteryMonitorReading::from_str(&output_string)?; + self.add_new_reading(battery_monitor_reading)?; + Ok(()) + } + } + + pub fn add_new_reading( + &mut self, + battery_monitor_reading: BatteryMonitorReading, + ) -> Result<()> { + self.update_metrics(battery_monitor_reading, T::now(), Utc::now())?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + + use super::*; + use crate::metrics::MetricValue; + use crate::test_utils::TestInstant; + use rstest::rstest; + + #[rstest] + #[case("Charging:80", true)] + #[case("Discharging:80", true)] + #[case("Not charging:80", true)] + #[case("Isn't charging:80", false)] + #[case("Charging:EIGHTY", false)] + #[case("Charging:42.5", true)] + #[case("Charging:42.five", false)] + #[case("Charging:42.3.5", false)] + #[case("Charging:-1", false)] + #[case("Discharging:100.1", false)] + #[case("Discharging:100.0", true)] + #[case("Discharging:0.0", true)] + #[case("Discharging:-0.1", false)] + #[case("Full:100.0", true)] + #[case("Unknown:80", true)] + fn test_parse(#[case] cmd_output: &str, #[case] is_ok: bool) { + assert_eq!(BatteryMonitorReading::from_str(cmd_output).is_ok(), is_ok); + } + + #[rstest] + // Single reading results in expected metrics + #[case(vec![BatteryMonitorReading::new(90.0, ChargingState::Charging)], 30, 90.0, 0.0, 0.0)] + #[case(vec![BatteryMonitorReading::new(90.0, ChargingState::Charging), BatteryMonitorReading::new(100.0, ChargingState::Charging)], 30, 95.0, 0.0, 0.0)] + // Battery discharges between readings + #[case(vec![BatteryMonitorReading::new(90.0, ChargingState::Discharging), BatteryMonitorReading::new(85.0, ChargingState::Discharging)], 30, 87.5, 5.0, 30000.0)] + // Battery is discharging, then charging, then discharging again + #[case(vec![BatteryMonitorReading::new(90.0, ChargingState::Discharging), + BatteryMonitorReading::new(85.0, ChargingState::Discharging), + BatteryMonitorReading::new(90.0, ChargingState::Charging), + BatteryMonitorReading::new(90.0, ChargingState::Discharging), + BatteryMonitorReading::new(80.0, ChargingState::Discharging)], + 30, + 87.0, + 15.0, + 60000.0)] + // Continuous discharge + #[case(vec![BatteryMonitorReading::new(90.0, ChargingState::Discharging), + BatteryMonitorReading::new(80.0, ChargingState::Discharging), + BatteryMonitorReading::new(70.0, ChargingState::Discharging), + BatteryMonitorReading::new(60.0, ChargingState::Discharging)], + 30, + 75.0, + 30.0, + 90000.0)] + // Continuous charge + #[case(vec![BatteryMonitorReading::new(60.0, ChargingState::Charging), + BatteryMonitorReading::new(70.0, ChargingState::Charging), + BatteryMonitorReading::new(80.0, ChargingState::Charging), + BatteryMonitorReading::new(90.0, ChargingState::Charging)], + 30, + 75.0, + 0.0, + 0.0)] + // Battery was charged in between monitoring calls + #[case(vec![BatteryMonitorReading::new(60.0, ChargingState::Discharging), + BatteryMonitorReading::new(80.0, ChargingState::Discharging),], + 30, + 70.0, + 0.0, + 30000.0)] + // Discharge then charge to full + #[case(vec![BatteryMonitorReading::new(90.0, ChargingState::Discharging), + BatteryMonitorReading::new(80.0, ChargingState::Discharging), + BatteryMonitorReading::new(70.0, ChargingState::Discharging), + BatteryMonitorReading::new(80.0, ChargingState::Charging), + BatteryMonitorReading::new(100.0, ChargingState::Full)], + 30, + 84.0, + 20.0, + 60000.0)] + // Check unknown and not charging states + #[case(vec![BatteryMonitorReading::new(60.0, ChargingState::Charging), + BatteryMonitorReading::new(70.0, ChargingState::Charging), + BatteryMonitorReading::new(80.0, ChargingState::Unknown), + BatteryMonitorReading::new(90.0, ChargingState::NotCharging)], + 30, + 75.0, + 0.0, + 0.0)] + // Check measurements with 0 seconds between + #[case(vec![BatteryMonitorReading::new(80.0, ChargingState::Charging), + BatteryMonitorReading::new(85.0, ChargingState::NotCharging), + BatteryMonitorReading::new(90.0, ChargingState::NotCharging)], + 0, + f64::NAN, + 0.0, + 0.0)] + fn test_update_metrics_soc_pct( + #[case] battery_monitor_readings: Vec, + #[case] seconds_between_readings: u64, + #[case] expected_soc_pct: f64, + #[case] expected_soc_pct_discharge: f64, + #[case] expected_discharge_duration: f64, + ) { + let now = TestInstant::now(); + let heartbeat_manager = Arc::new(Mutex::new(MetricReportManager::new())); + let mut battery_monitor = BatteryMonitor { + heartbeat_manager, + last_reading_time: now, + previous_reading: None, + }; + + let mut ts = Utc::now(); + for reading in battery_monitor_readings { + TestInstant::sleep(Duration::from_secs(seconds_between_readings)); + ts += chrono::Duration::seconds(seconds_between_readings as i64); + battery_monitor + .update_metrics(reading, TestInstant::now(), ts) + .unwrap(); + } + let metrics = battery_monitor + .heartbeat_manager + .lock() + .unwrap() + .take_heartbeat_metrics(); + let soc_pct_key = METRIC_BATTERY_SOC_PCT.parse::().unwrap(); + + match metrics.get(&soc_pct_key).unwrap() { + MetricValue::Number(e) => { + if expected_soc_pct.is_finite() { + assert_eq!(*e, expected_soc_pct); + } else { + assert!(e.is_nan()); + } + } + _ => panic!("This test only expects number metric values!"), + } + + let soc_pct_discharge_key = METRIC_BATTERY_SOC_PCT_DROP + .parse::() + .unwrap(); + match metrics.get(&soc_pct_discharge_key).unwrap() { + MetricValue::Number(e) => assert_eq!(*e, expected_soc_pct_discharge), + _ => panic!("This test only expects number metric values!"), + } + + let soc_discharge_duration_key = METRIC_BATTERY_DISCHARGE_DURATION_MS + .parse::() + .unwrap(); + match metrics.get(&soc_discharge_duration_key).unwrap() { + MetricValue::Number(e) => assert_eq!(*e, expected_discharge_duration), + _ => panic!("This test only expects number metric values!"), + } + } +} diff --git a/memfaultd/src/metrics/battery/battery_reading_handler.rs b/memfaultd/src/metrics/battery/battery_reading_handler.rs new file mode 100644 index 0000000..ba9d177 --- /dev/null +++ b/memfaultd/src/metrics/battery/battery_reading_handler.rs @@ -0,0 +1,188 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + io::Read, + ops::Sub, + str::{from_utf8, FromStr}, + sync::{Arc, Mutex}, + time::Duration, +}; + +use eyre::Result; +use tiny_http::{Method, Request, Response}; + +use crate::util::time_measure::TimeMeasure; +use crate::{ + http_server::{HttpHandler, HttpHandlerResult}, + metrics::{BatteryMonitor, BatteryMonitorReading}, +}; + +/// A server that listens for battery reading pushes and stores them in memory. +#[derive(Clone)] +pub struct BatteryReadingHandler { + data_collection_enabled: bool, + battery_monitor: Arc>>, +} + +impl BatteryReadingHandler +where + T: TimeMeasure + Copy + Ord + Sub + Send + Sync, +{ + pub fn new( + data_collection_enabled: bool, + battery_monitor: Arc>>, + ) -> Self { + Self { + data_collection_enabled, + battery_monitor, + } + } + + fn parse_request(stream: &mut dyn Read) -> Result { + let mut buf = vec![]; + stream.read_to_end(&mut buf)?; + let reading = BatteryMonitorReading::from_str(from_utf8(&buf)?)?; + Ok(reading) + } +} + +impl HttpHandler for BatteryReadingHandler +where + T: TimeMeasure + Copy + Ord + Sub + Send + Sync, +{ + fn handle_request(&self, request: &mut Request) -> HttpHandlerResult { + if request.url() != "/v1/battery/add_reading" || *request.method() != Method::Post { + return HttpHandlerResult::NotHandled; + } + if self.data_collection_enabled { + match Self::parse_request(request.as_reader()) { + Ok(reading) => { + match self + .battery_monitor + .lock() + .unwrap() + .add_new_reading(reading) + { + Ok(()) => HttpHandlerResult::Response(Response::empty(200).boxed()), + Err(e) => HttpHandlerResult::Error(format!( + "Failed to add battery reading to metrics: {:#}", + e + )), + } + } + Err(e) => HttpHandlerResult::Error(format!( + "Failed to parse battery reading string: {}", + e + )), + } + } else { + HttpHandlerResult::Response(Response::empty(200).boxed()) + } + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, + time::Duration, + }; + + use insta::assert_json_snapshot; + use rstest::rstest; + use tiny_http::{Method, TestRequest}; + + use crate::test_utils::TestInstant; + use crate::{ + http_server::{HttpHandler, HttpHandlerResult}, + metrics::{BatteryMonitor, MetricReportManager}, + }; + + use super::BatteryReadingHandler; + #[rstest] + fn handle_push() { + let heartbeat_manager = Arc::new(Mutex::new(MetricReportManager::new())); + let handler = BatteryReadingHandler::new( + true, + Arc::new(Mutex::new(BatteryMonitor::::new( + heartbeat_manager.clone(), + ))), + ); + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/battery/add_reading") + .with_body("Charging:80"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + let metrics = heartbeat_manager.lock().unwrap().take_heartbeat_metrics(); + + // Need to sort the map so the JSON string is consistent + let sorted_metrics: BTreeMap<_, _> = metrics.iter().collect(); + + assert_json_snapshot!(&sorted_metrics); + } + + // Need to include a test_name string parameter here due to + // a known issue using insta and rstest together: + // https://github.com/la10736/rstest/issues/183 + #[rstest] + #[case(vec!["Charging:80", "Charging:90", "Full:100", "Discharging:95", "Discharging:85"], 30, "charging_then_discharging")] + #[case(vec!["Full:100", "Discharging:90", "Discharging:50", "Not charging:50", "Discharging:30", "Discharging:10", "Charging:50"], 30, "nonconsecutive_discharges")] + #[case(vec!["Charging:90", "Charging:92.465", "Unknown:91.78", "Discharging:90", "Discharging:80"], 30, "non_integer_percentages")] + fn handle_push_of_multiple_readings( + #[case] readings: Vec<&'static str>, + #[case] seconds_between_readings: u64, + #[case] test_name: &str, + ) { + let heartbeat_manager = Arc::new(Mutex::new(MetricReportManager::new())); + let handler = BatteryReadingHandler::new( + true, + Arc::new(Mutex::new(BatteryMonitor::::new( + heartbeat_manager.clone(), + ))), + ); + for reading in readings { + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/battery/add_reading") + .with_body(reading); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + TestInstant::sleep(Duration::from_secs(seconds_between_readings)); + } + + let metrics = heartbeat_manager.lock().unwrap().take_heartbeat_metrics(); + + // Need to sort the map so the JSON string is consistent + let sorted_metrics: BTreeMap<_, _> = metrics.iter().collect(); + + // Set battery_soc_pct to 0.0 to avoid flakiness due to it being weighted by wall time + assert_json_snapshot!(test_name, &sorted_metrics, {".battery_soc_pct" => 0.0 }); + } + + #[rstest] + fn errors_when_body_is_invalid() { + let heartbeat_manager = Arc::new(Mutex::new(MetricReportManager::new())); + let handler = BatteryReadingHandler::::new( + true, + Arc::new(Mutex::new(BatteryMonitor::::new( + heartbeat_manager, + ))), + ); + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/battery/add_reading") + .with_body("{\"state\": \"Charging\", \"percent\":80}"); + matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Error(_) + ); + } +} diff --git a/memfaultd/src/metrics/battery/mod.rs b/memfaultd/src/metrics/battery/mod.rs new file mode 100644 index 0000000..e78f15a --- /dev/null +++ b/memfaultd/src/metrics/battery/mod.rs @@ -0,0 +1,9 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +mod battery_reading_handler; +pub use battery_reading_handler::BatteryReadingHandler; + +mod battery_monitor; +pub use battery_monitor::BatteryMonitor; +pub use battery_monitor::BatteryMonitorReading; diff --git a/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__charging_then_discharging.snap b/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__charging_then_discharging.snap new file mode 100644 index 0000000..b792236 --- /dev/null +++ b/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__charging_then_discharging.snap @@ -0,0 +1,9 @@ +--- +source: memfaultd/src/metrics/battery/battery_reading_handler.rs +expression: "&sorted_metrics" +--- +{ + "battery_discharge_duration_ms": 30000.0, + "battery_soc_pct": 0.0, + "battery_soc_pct_drop": 10.0 +} diff --git a/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__handle_push.snap b/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__handle_push.snap new file mode 100644 index 0000000..1d4cf88 --- /dev/null +++ b/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__handle_push.snap @@ -0,0 +1,9 @@ +--- +source: memfaultd/src/metrics/battery/battery_reading_handler.rs +expression: "serde_json::to_string_pretty(&sorted_metrics).expect(\"metric_store should be serializable\")" +--- +{ + "battery_discharge_duration_ms": 0.0, + "battery_soc_pct": null, + "battery_soc_pct_drop": 0.0 +} diff --git a/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__ignores_data_when_data_collection_is_off.snap b/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__ignores_data_when_data_collection_is_off.snap new file mode 100644 index 0000000..4d04193 --- /dev/null +++ b/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__ignores_data_when_data_collection_is_off.snap @@ -0,0 +1,5 @@ +--- +source: memfaultd/src/metrics/battery/battery_reading_handler.rs +expression: "serde_json::to_string_pretty(&metrics).expect(\"metric_store should be serializable\")" +--- +{} diff --git a/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__non_integer_percentages.snap b/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__non_integer_percentages.snap new file mode 100644 index 0000000..b792236 --- /dev/null +++ b/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__non_integer_percentages.snap @@ -0,0 +1,9 @@ +--- +source: memfaultd/src/metrics/battery/battery_reading_handler.rs +expression: "&sorted_metrics" +--- +{ + "battery_discharge_duration_ms": 30000.0, + "battery_soc_pct": 0.0, + "battery_soc_pct_drop": 10.0 +} diff --git a/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__nonconsecutive_discharges.snap b/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__nonconsecutive_discharges.snap new file mode 100644 index 0000000..07dd8ff --- /dev/null +++ b/memfaultd/src/metrics/battery/snapshots/memfaultd__metrics__battery__battery_reading_handler__tests__nonconsecutive_discharges.snap @@ -0,0 +1,9 @@ +--- +source: memfaultd/src/metrics/battery/battery_reading_handler.rs +expression: "&sorted_metrics" +--- +{ + "battery_discharge_duration_ms": 60000.0, + "battery_soc_pct": 0.0, + "battery_soc_pct_drop": 60.0 +} diff --git a/memfaultd/src/metrics/connectivity/connectivity_monitor.rs b/memfaultd/src/metrics/connectivity/connectivity_monitor.rs new file mode 100644 index 0000000..79e0081 --- /dev/null +++ b/memfaultd/src/metrics/connectivity/connectivity_monitor.rs @@ -0,0 +1,217 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + ops::Sub, + sync::{Arc, Mutex}, + time::Duration, +}; + +use eyre::Result; + +use crate::{ + config::{ConnectivityMonitorConfig, ConnectivityMonitorTarget}, + metrics::{ + core_metrics::{METRIC_CONNECTED_TIME, METRIC_EXPECTED_CONNECTED_TIME}, + MetricReportManager, + }, + util::{can_connect::CanConnect, time_measure::TimeMeasure}, +}; + +pub struct ConnectivityMonitor { + targets: Vec, + interval: Duration, + last_checked_at: Option, + heartbeat_manager: Arc>, + connection_checker: U, +} + +impl ConnectivityMonitor +where + T: TimeMeasure + Copy + Ord + Sub, + U: CanConnect, +{ + pub fn new( + config: &ConnectivityMonitorConfig, + heartbeat_manager: Arc>, + ) -> Self { + Self { + targets: config.targets.clone(), + interval: config.interval_seconds, + last_checked_at: None, + heartbeat_manager, + connection_checker: U::new(config.timeout_seconds), + } + } + + fn is_connected(&self) -> bool { + self.targets + .iter() + .any(|ConnectivityMonitorTarget { host, port, .. }| { + self.connection_checker.can_connect(host, *port).is_ok() + }) + } + + pub fn update_connected_time(&mut self) -> Result<()> { + let now = T::now(); + let since_last_reading = self.last_checked_at.unwrap_or(now).elapsed(); + let connected_duration = if self.is_connected() { + since_last_reading + } else { + Duration::ZERO + }; + + let mut store = self.heartbeat_manager.lock().expect("Mutex Poisoned"); + store.add_to_counter(METRIC_CONNECTED_TIME, connected_duration.as_millis() as f64)?; + store.add_to_counter( + METRIC_EXPECTED_CONNECTED_TIME, + since_last_reading.as_millis() as f64, + )?; + + self.last_checked_at = Some(now); + + Ok(()) + } + + pub fn interval_seconds(&self) -> Duration { + self.interval + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::BTreeMap, + net::IpAddr, + str::FromStr, + sync::{Arc, Mutex}, + time::Duration, + }; + + use insta::assert_json_snapshot; + use rstest::rstest; + + use super::ConnectivityMonitor; + use crate::test_utils::{TestConnectionChecker, TestInstant}; + use crate::{ + config::{ConnectionCheckProtocol, ConnectivityMonitorConfig, ConnectivityMonitorTarget}, + metrics::MetricReportManager, + }; + + #[rstest] + fn test_while_connected() { + let heartbeat_manager = Arc::new(Mutex::new(MetricReportManager::new())); + let config = ConnectivityMonitorConfig { + targets: vec![ConnectivityMonitorTarget { + host: IpAddr::from_str("8.8.8.8").unwrap(), + port: 443, + protocol: ConnectionCheckProtocol::Tcp, + }], + interval_seconds: Duration::from_secs(15), + timeout_seconds: Duration::from_secs(10), + }; + let mut connectivity_monitor = + ConnectivityMonitor::::new( + &config, + heartbeat_manager.clone(), + ); + + TestConnectionChecker::connect(); + + connectivity_monitor + .update_connected_time() + .expect("Couldn't update connected time monitor!"); + + TestInstant::sleep(Duration::from_secs(30)); + connectivity_monitor + .update_connected_time() + .expect("Couldn't update connected time monitor!"); + + let metrics = heartbeat_manager.lock().unwrap().take_heartbeat_metrics(); + + // Need to sort the map so the JSON string is consistent + let sorted_metrics: BTreeMap<_, _> = metrics.iter().collect(); + + assert_json_snapshot!(sorted_metrics); + } + + #[rstest] + fn test_half_connected_half_disconnected() { + let heartbeat_manager = Arc::new(Mutex::new(MetricReportManager::new())); + let config = ConnectivityMonitorConfig { + targets: vec![ConnectivityMonitorTarget { + host: IpAddr::from_str("8.8.8.8").unwrap(), + port: 443, + protocol: ConnectionCheckProtocol::Tcp, + }], + interval_seconds: Duration::from_secs(15), + timeout_seconds: Duration::from_secs(10), + }; + let mut connectivity_monitor = + ConnectivityMonitor::::new( + &config, + heartbeat_manager.clone(), + ); + + TestConnectionChecker::connect(); + + // Initial reading + connectivity_monitor.update_connected_time().unwrap(); + + TestInstant::sleep(Duration::from_secs(30)); + connectivity_monitor + .update_connected_time() + .expect("Couldn't update connected time monitor!"); + + TestConnectionChecker::disconnect(); + + TestInstant::sleep(Duration::from_secs(30)); + connectivity_monitor + .update_connected_time() + .expect("Couldn't update connected time monitor!"); + let metrics = heartbeat_manager.lock().unwrap().take_heartbeat_metrics(); + + // Need to sort the map so the JSON string is consistent + let sorted_metrics: BTreeMap<_, _> = metrics.iter().collect(); + assert_json_snapshot!(sorted_metrics); + } + + #[rstest] + fn test_fully_disconnected() { + let heartbeat_manager = Arc::new(Mutex::new(MetricReportManager::new())); + let config = ConnectivityMonitorConfig { + targets: vec![ConnectivityMonitorTarget { + host: IpAddr::from_str("8.8.8.8").unwrap(), + port: 443, + protocol: ConnectionCheckProtocol::Tcp, + }], + interval_seconds: Duration::from_secs(15), + timeout_seconds: Duration::from_secs(10), + }; + let mut connectivity_monitor = + ConnectivityMonitor::::new( + &config, + heartbeat_manager.clone(), + ); + + TestConnectionChecker::disconnect(); + + // Initial reading + connectivity_monitor.update_connected_time().unwrap(); + + TestInstant::sleep(Duration::from_secs(30)); + connectivity_monitor + .update_connected_time() + .expect("Couldn't update connected time monitor!"); + TestInstant::sleep(Duration::from_secs(30)); + connectivity_monitor + .update_connected_time() + .expect("Couldn't update connected time monitor!"); + let metrics = heartbeat_manager.lock().unwrap().take_heartbeat_metrics(); + + // Need to sort the map so the JSON string is consistent + let sorted_metrics: BTreeMap<_, _> = metrics.iter().collect(); + + assert_json_snapshot!(sorted_metrics); + } +} diff --git a/memfaultd/src/metrics/connectivity/mod.rs b/memfaultd/src/metrics/connectivity/mod.rs new file mode 100644 index 0000000..6f4aa52 --- /dev/null +++ b/memfaultd/src/metrics/connectivity/mod.rs @@ -0,0 +1,8 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +mod connectivity_monitor; +pub use connectivity_monitor::ConnectivityMonitor; + +mod report_sync_event_handler; +pub use report_sync_event_handler::ReportSyncEventHandler; diff --git a/memfaultd/src/metrics/connectivity/report_sync_event_handler.rs b/memfaultd/src/metrics/connectivity/report_sync_event_handler.rs new file mode 100644 index 0000000..52e1cba --- /dev/null +++ b/memfaultd/src/metrics/connectivity/report_sync_event_handler.rs @@ -0,0 +1,150 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::sync::{Arc, Mutex}; + +use log::warn; +use tiny_http::{Method, Request, Response}; + +use crate::{ + http_server::{HttpHandler, HttpHandlerResult}, + metrics::{ + core_metrics::{METRIC_SYNC_FAILURE, METRIC_SYNC_SUCCESS}, + MetricReportManager, + }, +}; + +/// A server that listens for collectd JSON pushes and stores them in memory. +#[derive(Clone)] +pub struct ReportSyncEventHandler { + data_collection_enabled: bool, + metrics_store: Arc>, +} + +impl ReportSyncEventHandler { + pub fn new( + data_collection_enabled: bool, + metrics_store: Arc>, + ) -> Self { + Self { + data_collection_enabled, + metrics_store, + } + } +} + +impl HttpHandler for ReportSyncEventHandler { + fn handle_request(&self, request: &mut Request) -> HttpHandlerResult { + if (request.url() != "/v1/sync/success" && request.url() != "/v1/sync/failure") + || *request.method() != Method::Post + { + return HttpHandlerResult::NotHandled; + } + if self.data_collection_enabled { + if request.url() == "/v1/sync/success" { + let mut metrics_store = self.metrics_store.lock().unwrap(); + if let Err(e) = metrics_store.increment_counter(METRIC_SYNC_SUCCESS) { + warn!("Couldn't increment sync_success counter: {:#}", e); + } + } else if request.url() == "/v1/sync/failure" { + let mut metrics_store = self.metrics_store.lock().unwrap(); + if let Err(e) = metrics_store.increment_counter(METRIC_SYNC_FAILURE) { + warn!("Couldn't increment sync_failure counter: {:#}", e); + } + } + } + HttpHandlerResult::Response(Response::empty(200).boxed()) + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, + }; + + use insta::assert_json_snapshot; + use rstest::{fixture, rstest}; + use tiny_http::{Method, TestRequest}; + + use crate::{ + http_server::{HttpHandler, HttpHandlerResult}, + metrics::MetricReportManager, + }; + + use super::ReportSyncEventHandler; + + #[rstest] + fn handle_sync_success(handler: ReportSyncEventHandler) { + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/sync/success"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + let metrics = handler + .metrics_store + .lock() + .unwrap() + .take_heartbeat_metrics(); + assert_json_snapshot!(&metrics); + } + + #[rstest] + fn handle_sync_failure(handler: ReportSyncEventHandler) { + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/sync/failure"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + let metrics = handler + .metrics_store + .lock() + .unwrap() + .take_heartbeat_metrics(); + assert_json_snapshot!(&metrics); + } + + #[rstest] + fn handle_multiple_sync_events(handler: ReportSyncEventHandler) { + for _ in 0..10 { + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/sync/failure"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + } + + for _ in 0..90 { + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/sync/success"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + } + let metrics = handler + .metrics_store + .lock() + .unwrap() + .take_heartbeat_metrics(); + // Need to sort the map so the JSON string is consistent + let sorted_metrics: BTreeMap<_, _> = metrics.iter().collect(); + + assert_json_snapshot!(&sorted_metrics); + } + + #[fixture] + fn handler() -> ReportSyncEventHandler { + ReportSyncEventHandler::new(true, Arc::new(Mutex::new(MetricReportManager::new()))) + } +} diff --git a/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__connectivity_monitor__tests__fully_disconnected.snap b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__connectivity_monitor__tests__fully_disconnected.snap new file mode 100644 index 0000000..70b69e3 --- /dev/null +++ b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__connectivity_monitor__tests__fully_disconnected.snap @@ -0,0 +1,8 @@ +--- +source: memfaultd/src/metrics/connectivity/connectivity_monitor.rs +expression: sorted_metrics +--- +{ + "connectivity_connected_time_ms": 0.0, + "connectivity_expected_time_ms": 60000.0 +} diff --git a/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__connectivity_monitor__tests__half_connected_half_disconnected.snap b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__connectivity_monitor__tests__half_connected_half_disconnected.snap new file mode 100644 index 0000000..e411ed1 --- /dev/null +++ b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__connectivity_monitor__tests__half_connected_half_disconnected.snap @@ -0,0 +1,8 @@ +--- +source: memfaultd/src/metrics/connectivity/connectivity_monitor.rs +expression: sorted_metrics +--- +{ + "connectivity_connected_time_ms": 30000.0, + "connectivity_expected_time_ms": 60000.0 +} diff --git a/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__connectivity_monitor__tests__while_connected.snap b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__connectivity_monitor__tests__while_connected.snap new file mode 100644 index 0000000..87fa147 --- /dev/null +++ b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__connectivity_monitor__tests__while_connected.snap @@ -0,0 +1,8 @@ +--- +source: memfaultd/src/metrics/connectivity/connectivity_monitor.rs +expression: sorted_metrics +--- +{ + "connectivity_connected_time_ms": 30000.0, + "connectivity_expected_time_ms": 30000.0 +} diff --git a/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__report_sync_event_handler__tests__handle_multiple_sync_events.snap b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__report_sync_event_handler__tests__handle_multiple_sync_events.snap new file mode 100644 index 0000000..83861ef --- /dev/null +++ b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__report_sync_event_handler__tests__handle_multiple_sync_events.snap @@ -0,0 +1,8 @@ +--- +source: memfaultd/src/metrics/connectivity/report_sync_event_handler.rs +expression: "serde_json::to_string_pretty(&sorted_metrics).expect(\"metric_store should be serializable\")" +--- +{ + "sync_failure": 10.0, + "sync_successful": 90.0 +} diff --git a/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__report_sync_event_handler__tests__handle_sync_failure.snap b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__report_sync_event_handler__tests__handle_sync_failure.snap new file mode 100644 index 0000000..d6079b6 --- /dev/null +++ b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__report_sync_event_handler__tests__handle_sync_failure.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/metrics/connectivity/report_sync_event_handler.rs +expression: "serde_json::to_string_pretty(&metrics).expect(\"metric_store should be serializable\")" +--- +{ + "sync_failure": 1.0 +} diff --git a/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__report_sync_event_handler__tests__handle_sync_success.snap b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__report_sync_event_handler__tests__handle_sync_success.snap new file mode 100644 index 0000000..9e75119 --- /dev/null +++ b/memfaultd/src/metrics/connectivity/snapshots/memfaultd__metrics__connectivity__report_sync_event_handler__tests__handle_sync_success.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/metrics/connectivity/report_sync_event_handler.rs +expression: "serde_json::to_string_pretty(&metrics).expect(\"metric_store should be serializable\")" +--- +{ + "sync_successful": 1.0 +} diff --git a/memfaultd/src/metrics/core_metrics.rs b/memfaultd/src/metrics/core_metrics.rs new file mode 100644 index 0000000..408f96c --- /dev/null +++ b/memfaultd/src/metrics/core_metrics.rs @@ -0,0 +1,20 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +// Connectivity metrics +pub const METRIC_MF_SYNC_SUCCESS: &str = "sync_memfault_successful"; +pub const METRIC_MF_SYNC_FAILURE: &str = "sync_memfault_failure"; +pub const METRIC_CONNECTED_TIME: &str = "connectivity_connected_time_ms"; +pub const METRIC_EXPECTED_CONNECTED_TIME: &str = "connectivity_expected_time_ms"; +pub const METRIC_SYNC_SUCCESS: &str = "sync_successful"; +pub const METRIC_SYNC_FAILURE: &str = "sync_failure"; + +// Stability metrics +pub const METRIC_OPERATIONAL_HOURS: &str = "operational_hours"; +pub const METRIC_OPERATIONAL_CRASHFREE_HOURS: &str = "operational_crashfree_hours"; +pub const METRIC_OPERATIONAL_CRASHES: &str = "operational_crashes"; + +// Battery metrics +pub const METRIC_BATTERY_DISCHARGE_DURATION_MS: &str = "battery_discharge_duration_ms"; +pub const METRIC_BATTERY_SOC_PCT_DROP: &str = "battery_soc_pct_drop"; +pub const METRIC_BATTERY_SOC_PCT: &str = "battery_soc_pct"; diff --git a/memfaultd/src/metrics/crashfree_interval.rs b/memfaultd/src/metrics/crashfree_interval.rs new file mode 100644 index 0000000..a574b25 --- /dev/null +++ b/memfaultd/src/metrics/crashfree_interval.rs @@ -0,0 +1,413 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + cmp::max, + iter::once, + sync::mpsc::{channel, Receiver, Sender}, + time::Duration, +}; + +use chrono::Utc; +use tiny_http::{Method, Request, Response}; + +use crate::{ + http_server::{HttpHandler, HttpHandlerResult}, + metrics::{ + core_metrics::{ + METRIC_OPERATIONAL_CRASHES, METRIC_OPERATIONAL_CRASHFREE_HOURS, + METRIC_OPERATIONAL_HOURS, + }, + MetricReading, + }, + util::time_measure::TimeMeasure, +}; + +use super::{KeyedMetricReading, MetricStringKey}; + +pub struct CrashFreeIntervalTracker { + last_interval_mark: T, + last_crashfree_interval_mark: T, + crash_count: u32, + sender: Sender, + receiver: Receiver, + interval: Duration, + elapsed_intervals_key: MetricStringKey, + crashfree_intervals_key: MetricStringKey, + crash_count_key: MetricStringKey, +} + +#[derive(Debug, PartialEq, Eq)] +struct TimeMod { + count: u32, + mark: T, +} + +impl CrashFreeIntervalTracker +where + T: TimeMeasure + Copy + Ord + std::ops::Add + Send + Sync + 'static, +{ + pub fn new( + interval: Duration, + elapsed_intervals_key: MetricStringKey, + crashfree_intervals_key: MetricStringKey, + crash_count_key: MetricStringKey, + ) -> Self { + let (sender, receiver) = channel(); + Self { + last_crashfree_interval_mark: T::now(), + last_interval_mark: T::now(), + sender, + receiver, + crash_count: 0, + interval, + elapsed_intervals_key, + crashfree_intervals_key, + crash_count_key, + } + } + + /// Returns a tracker with an hourly interval + pub fn new_hourly() -> Self { + Self::new( + Duration::from_secs(3600), + METRIC_OPERATIONAL_HOURS.parse().unwrap(), + METRIC_OPERATIONAL_CRASHFREE_HOURS.parse().unwrap(), + METRIC_OPERATIONAL_CRASHES.parse().unwrap(), + ) + } + + /// Wait for the next crash or update the metrics if the wait duration has passed. + /// + /// This allows us to have instant updates on crashes and hourly updates on the metrics, but + /// also allows us to periodically update the metrics so that we don't have to wait for a crash. + pub fn wait_and_update(&mut self, wait_duration: Duration) -> Vec { + if let Ok(crash_ts) = self.receiver.recv_timeout(wait_duration) { + // Drain the receiver to get all crashes that happened since the last update + self.receiver + .try_iter() + .chain(once(crash_ts)) + .for_each(|ts| { + self.crash_count += 1; + self.last_crashfree_interval_mark = max(self.last_crashfree_interval_mark, ts); + }); + } + + // Since timing out just means no crashes occurred in the `wait_duration`, + // update even when the receiver times out. + self.update() + } + + fn update(&mut self) -> Vec { + let TimeMod { + count: count_op_interval, + mark: last_counted_op_interval, + } = Self::full_interval_elapsed_since(self.interval, &self.last_interval_mark); + let TimeMod { + count: count_crashfree_interval, + mark: last_counted_crashfree_interval, + } = Self::full_interval_elapsed_since(self.interval, &self.last_crashfree_interval_mark); + + self.last_interval_mark = last_counted_op_interval; + self.last_crashfree_interval_mark = last_counted_crashfree_interval; + + let crashes = self.crash_count; + self.crash_count = 0; + + let metrics_ts = Utc::now(); + vec![ + KeyedMetricReading::new( + self.elapsed_intervals_key.clone(), + MetricReading::Counter { + value: count_op_interval as f64, + timestamp: metrics_ts, + }, + ), + KeyedMetricReading::new( + self.crashfree_intervals_key.clone(), + MetricReading::Counter { + value: count_crashfree_interval as f64, + timestamp: metrics_ts, + }, + ), + KeyedMetricReading::new( + self.crash_count_key.clone(), + MetricReading::Counter { + value: crashes as f64, + timestamp: metrics_ts, + }, + ), + ] + } + + pub fn http_handler(&mut self) -> Box { + Box::new(CrashFreeIntervalHttpHandler { + channel: self.sender.clone(), + }) + } + + pub fn capture_crash(&self) { + self.sender + .send(T::now()) + .expect("Failed to send crash timestamp"); + } + + /// Count how many `interval` have elapsed since `since`. + /// + /// This returns the number of intervals that have elapsed since `since`, and the timestamp of the end of the last interval + /// that was counted. This is the value you should pass as `since` next time you call this function. + /// + /// See unit test for examples. + fn full_interval_elapsed_since(interval: Duration, since: &T) -> TimeMod { + let now = T::now(); + if *since > now { + return TimeMod { + count: 0, + mark: T::now(), + }; + } + + let duration = now.since(since); + let count_interval_elapsed = (duration.as_nanos() / interval.as_nanos()) as u32; + TimeMod { + count: count_interval_elapsed, + mark: since.add(interval * count_interval_elapsed), + } + } +} + +struct CrashFreeIntervalHttpHandler { + channel: Sender, +} + +impl HttpHandler for CrashFreeIntervalHttpHandler +where + T: TimeMeasure + Copy + Ord + std::ops::Add + Send + Sync, +{ + fn handle_request(&self, request: &mut Request) -> HttpHandlerResult { + if request.url() == "/v1/crash/report" && request.method() == &Method::Post { + self.channel.send(T::now()).unwrap(); + HttpHandlerResult::Response(Response::from_string("OK").boxed()) + } else { + HttpHandlerResult::NotHandled + } + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use rstest::rstest; + + use crate::{ + metrics::{ + crashfree_interval::{ + METRIC_OPERATIONAL_CRASHES, METRIC_OPERATIONAL_CRASHFREE_HOURS, + METRIC_OPERATIONAL_HOURS, + }, + KeyedMetricReading, MetricReading, + }, + test_utils::TestInstant, + }; + + use super::CrashFreeIntervalTracker; + use super::TimeMod; + + #[rstest] + fn test_counting_intervals() { + use std::time::Duration; + + // move the clock forward so we can go backwards below + TestInstant::sleep(Duration::from_secs(3600)); + let now = TestInstant::now(); + + let d10 = Duration::from_secs(10); + assert_eq!( + CrashFreeIntervalTracker::full_interval_elapsed_since(d10, &now), + TimeMod { + count: 0, + mark: now + } + ); + assert_eq!( + CrashFreeIntervalTracker::full_interval_elapsed_since( + d10, + &(now - Duration::from_secs(10)) + ), + TimeMod { + count: 1, + mark: now + } + ); + assert_eq!( + CrashFreeIntervalTracker::full_interval_elapsed_since( + d10, + &(now - Duration::from_secs(25)) + ), + TimeMod { + count: 2, + mark: now - Duration::from_secs(5) + } + ); + } + + #[rstest] + fn test_counting_hours() { + let mut crashfree_tracker = CrashFreeIntervalTracker::::new_hourly(); + + TestInstant::sleep(Duration::from_secs(7200)); + + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 2, + 2, + ); + } + + #[rstest] + fn test_counting_minutes() { + let mut crashfree_tracker = CrashFreeIntervalTracker::::new( + Duration::from_secs(60), + METRIC_OPERATIONAL_HOURS.parse().unwrap(), + METRIC_OPERATIONAL_CRASHFREE_HOURS.parse().unwrap(), + METRIC_OPERATIONAL_CRASHES.parse().unwrap(), + ); + + TestInstant::sleep(Duration::from_secs(3600)); + crashfree_tracker.capture_crash(); + TestInstant::sleep(Duration::from_secs(3600)); + + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 120, + 60, + ); + } + + #[rstest] + fn test_30min_heartbeat() { + let mut crashfree_tracker = CrashFreeIntervalTracker::::new_hourly(); + + TestInstant::sleep(Duration::from_secs(1800)); + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 0, + 0, + ); + + TestInstant::sleep(Duration::from_secs(1800)); + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 1, + 1, + ); + } + + #[rstest] + fn test_30min_heartbeat_with_crash() { + let mut crashfree_tracker = CrashFreeIntervalTracker::::new_hourly(); + + TestInstant::sleep(Duration::from_secs(1800)); + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 0, + 0, + ); + + // Crash at t0 + 30min + crashfree_tracker.capture_crash(); + + // After 30' we should be ready to mark an operational hour + TestInstant::sleep(Duration::from_secs(1800)); + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 1, + 0, + ); + + // After another 30' we should be ready to mark another crashfree hour + TestInstant::sleep(Duration::from_secs(1800)); + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 0, + 1, + ); + + // After another 30' we should be ready to mark another operational hour + TestInstant::sleep(Duration::from_secs(1800)); + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 1, + 0, + ); + } + + #[rstest] + fn test_180min_heartbeat_with_one_crash() { + let mut crashfree_tracker = CrashFreeIntervalTracker::::new_hourly(); + + // Basic test + TestInstant::sleep(Duration::from_secs(3600 * 3)); + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 3, + 3, + ); + + // Crash at interval + 170' + TestInstant::sleep(Duration::from_secs(170 * 60)); + crashfree_tracker.capture_crash(); + + // Another 10' to the heartbeat mark + // We will count 0 operational hour here. That is a consequence of the heartbeat being larger than the hour + // To avoid this bug, we need to make sure we call the `update` at least once per hour! + TestInstant::sleep(Duration::from_secs(10 * 60)); + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 3, + 0, + ); + + // However, doing the crash at interval +10' then waiting for 170' will record 2 crashfree hours + TestInstant::sleep(Duration::from_secs(10 * 60)); + crashfree_tracker.capture_crash(); + TestInstant::sleep(Duration::from_secs(170 * 60)); + assert_operational_metrics( + crashfree_tracker.wait_and_update(Duration::from_secs(0)), + 3, + 2, + ); + } + + fn assert_operational_metrics( + metrics: Vec, + expected_op_hours: u32, + expected_crashfree_hours: u32, + ) { + assert_eq!(metrics.len(), 3); + let op_hours = metrics + .iter() + .find(|m| m.name.as_str() == METRIC_OPERATIONAL_HOURS) + .unwrap(); + let crash_free_hours = metrics + .iter() + .find(|m| m.name.as_str() == METRIC_OPERATIONAL_CRASHFREE_HOURS) + .unwrap(); + + let op_hours_value = match op_hours.value { + MetricReading::Counter { value, .. } => value, + _ => panic!("Unexpected metric type"), + }; + + let crashfree_hours_value = match crash_free_hours.value { + MetricReading::Counter { value, .. } => value, + _ => panic!("Unexpected metric type"), + }; + + assert_eq!( + (op_hours_value as u32, crashfree_hours_value as u32), + (expected_op_hours, expected_crashfree_hours) + ); + } +} diff --git a/memfaultd/src/metrics/metric_reading.rs b/memfaultd/src/metrics/metric_reading.rs new file mode 100644 index 0000000..df0f51d --- /dev/null +++ b/memfaultd/src/metrics/metric_reading.rs @@ -0,0 +1,255 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::str::FromStr; + +use chrono::{Duration, Utc}; +use eyre::{eyre, ErrReport}; +use nom::{ + branch::alt, + character::complete::char, + combinator::value, + number::complete::double, + sequence::separated_pair, + Finish, + {bytes::complete::tag, IResult}, +}; +use serde::{Deserialize, Serialize}; + +use super::{MetricStringKey, MetricTimestamp}; + +use crate::util::serialization::float_to_duration; + +/// A typed value and timestamp pair that represents +/// an individual reading value for a metric. This type does +/// not have a notion of which key it is associated it +/// and is purely the "value" in a metric reading. +/// For the full metric reading type that includes the key, +/// use KeyedMetricReading. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum MetricReading { + /// TimeWeightedAverage readings will be aggregated based on + /// the time the reading was captured over. + TimeWeightedAverage { + value: f64, + timestamp: MetricTimestamp, + /// Time period considered for this reading. This is only used to give a "time-weight" to the first + /// value in the series. For future values we will use the time + /// difference we measure between the two points + /// In doubt, it's safe to use Duration::from_secs(0) here. This means the first value will be ignored. + #[serde(with = "float_to_duration")] + interval: Duration, + }, + /// A non-decreasing monotonic sum. Within a metric report, Counter readings are summed together. + Counter { + value: f64, + timestamp: MetricTimestamp, + }, + /// Gauges are absolute values. We keep the latest value collected during a metric report. + Gauge { + value: f64, + timestamp: MetricTimestamp, + }, + /// Histogram readings are averaged together by dividing the sum of the values + /// by the number of readings over the duration of a metric report. + Histogram { + value: f64, + timestamp: MetricTimestamp, + }, + /// ReportTags are string values associated with the MetricReport they are captured in. + /// We keep the latest value collected during a metric report for a key and drop the older + /// ones. + ReportTag { + value: String, + timestamp: MetricTimestamp, + }, +} + +impl MetricReading { + /// Parse a metric reading in the format f64|. + /// The timestamp will be set to Utc::now(). + /// This is the suffix of a full StatsD reading of the following format: + /// : + /// + /// Examples of valid readings: + /// 64|h + /// 100.0|c + /// -89.5|g + fn parse(input: &str) -> IResult<&str, MetricReading> { + let (remaining, (value, statsd_type)) = + separated_pair(double, tag("|"), StatsDMetricType::parse)(input)?; + let timestamp = Utc::now(); + match statsd_type { + StatsDMetricType::Histogram => { + Ok((remaining, MetricReading::Histogram { value, timestamp })) + } + StatsDMetricType::Counter => { + Ok((remaining, MetricReading::Counter { value, timestamp })) + } + StatsDMetricType::Timer => Ok((remaining, MetricReading::Counter { value, timestamp })), + StatsDMetricType::Gauge => Ok((remaining, MetricReading::Gauge { value, timestamp })), + } + } +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct KeyedMetricReading { + pub name: MetricStringKey, + pub value: MetricReading, +} + +impl KeyedMetricReading { + pub fn new(name: MetricStringKey, value: MetricReading) -> Self { + Self { name, value } + } + + pub fn new_gauge(name: MetricStringKey, value: f64) -> Self { + Self { + name, + value: MetricReading::Gauge { + value, + timestamp: Utc::now(), + }, + } + } + + pub fn new_histogram(name: MetricStringKey, value: f64) -> Self { + Self { + name, + value: MetricReading::Histogram { + value, + timestamp: Utc::now(), + }, + } + } + + /// Construct a KeyedMetricReading from a string in the StatsD format + /// | + /// + /// Examples of valid keyed metric readings: + /// testCounter:1|c + /// test_counter:1.0|c + /// test_histo:100|h + /// test_gauge:1.7|g + /// cpu3_idle:100.9898|g + pub fn from_statsd_str(s: &str) -> Result { + match Self::parse_statsd(s).finish() { + Ok((_, reading)) => Ok(reading), + Err(e) => Err(eyre!( + "Failed to parse string \"{}\" as a StatsD metric reading: {}", + s, + e + )), + } + } + + /// Helper that handles `nom` details for parsing a StatsD string as + /// a KeyedMetricReading + fn parse_statsd(input: &str) -> IResult<&str, KeyedMetricReading> { + let (remaining, (name, value)) = + separated_pair(MetricStringKey::parse, tag(":"), MetricReading::parse)(input)?; + Ok((remaining, KeyedMetricReading { name, value })) + } + + /// Deserialize a string in the form = to a Gauge metric reading + /// + /// Currently deserialization to a KeyedMetricReading with any other type of MetricReading + /// as its value is not supported + fn from_arg_str(s: &str) -> Result { + let (key, value_str) = s.split_once('=').ok_or(eyre!( + "Attached metric reading should be specified as KEY=VALUE" + ))?; + + // Let's ensure the key is valid first: + let metric_key = MetricStringKey::from_str(key).map_err(|e| eyre!(e))?; + if let Ok(value) = f64::from_str(value_str) { + let reading = MetricReading::Gauge { + value, + timestamp: Utc::now(), + }; + Ok(KeyedMetricReading::new(metric_key, reading)) + } else { + let reading = MetricReading::ReportTag { + value: value_str.to_string(), + timestamp: Utc::now(), + }; + Ok(KeyedMetricReading::new(metric_key, reading)) + } + } +} + +impl FromStr for KeyedMetricReading { + type Err = ErrReport; + + fn from_str(s: &str) -> Result { + match Self::from_statsd_str(s) { + Ok(reading) => Ok(reading), + Err(_) => match Self::from_arg_str(s) { + Ok(reading) => Ok(reading), + Err(e) => Err(eyre!("Couldn't parse \"{}\" as a Gauge metric: {}", s, e)), + }, + } + } +} + +#[derive(Debug, Clone)] +enum StatsDMetricType { + Counter, + Histogram, + Gauge, + Timer, +} + +impl StatsDMetricType { + /// Parse a StatsDMetricType, which must be one of 'c', 'g', or 'h' + /// + /// 'c' indicates a Counter reading + /// 'g' indicates a Gauge reading + /// 'h' indicates a Histogram reading + fn parse(input: &str) -> IResult<&str, StatsDMetricType> { + alt(( + value(StatsDMetricType::Counter, char('c')), + value(StatsDMetricType::Histogram, char('h')), + value(StatsDMetricType::Gauge, char('g')), + value(StatsDMetricType::Timer, tag("ms")), + ))(input) + } +} + +#[cfg(test)] +mod tests { + + use super::*; + use crate::test_utils::setup_logger; + use rstest::rstest; + + #[rstest] + #[case("testCounter=1")] + #[case("hello=world")] + #[case("float=100.0")] + fn parse_valid_arg_reading(#[case] reading_str: &str, _setup_logger: ()) { + assert!(KeyedMetricReading::from_str(reading_str).is_ok()) + } + + #[rstest] + #[case("testCounter:1|c")] + #[case("test_counter:1.0|c")] + #[case("test_histo:100|h")] + #[case("test_gauge:1.7|g")] + #[case("cpu3_idle:100.9898|g")] + #[case("some_negative_gauge:-87.55|g")] + #[case("test_timer:3600000|ms")] + fn parse_valid_statsd_reading(#[case] reading_str: &str, _setup_logger: ()) { + assert!(KeyedMetricReading::from_str(reading_str).is_ok()) + } + + #[rstest] + #[case("test Counter:1|c")] + #[case("{test_counter:1.0|c}")] + #[case("\"test_counter\":1.0|c}")] + #[case("test_gauge:\"string-value\"|g")] + #[case("test_gauge:string-value|g")] + fn fail_on_invalid_statsd_reading(#[case] reading_str: &str, _setup_logger: ()) { + assert!(KeyedMetricReading::from_str(reading_str).is_err()) + } +} diff --git a/memfaultd/src/metrics/metric_report.rs b/memfaultd/src/metrics/metric_report.rs new file mode 100644 index 0000000..86e8579 --- /dev/null +++ b/memfaultd/src/metrics/metric_report.rs @@ -0,0 +1,337 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use chrono::Utc; +use eyre::{eyre, Result}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{HashMap, HashSet}, + path::Path, + str::FromStr, + time::{Duration, Instant}, +}; + +use crate::{ + mar::{MarEntryBuilder, Metadata}, + metrics::{ + core_metrics::{ + METRIC_BATTERY_DISCHARGE_DURATION_MS, METRIC_BATTERY_SOC_PCT_DROP, + METRIC_CONNECTED_TIME, METRIC_EXPECTED_CONNECTED_TIME, METRIC_MF_SYNC_FAILURE, + METRIC_MF_SYNC_SUCCESS, METRIC_OPERATIONAL_CRASHES, METRIC_SYNC_FAILURE, + METRIC_SYNC_SUCCESS, + }, + metric_reading::KeyedMetricReading, + timeseries::{Counter, Gauge, Histogram, TimeSeries, TimeWeightedAverage}, + MetricReading, MetricStringKey, MetricValue, SessionName, + }, +}; + +use super::timeseries::ReportTag; + +pub enum CapturedMetrics { + All, + Metrics(HashSet), +} + +pub const HEARTBEAT_REPORT_TYPE: &str = "heartbeat"; +pub const DAILY_HEARTBEAT_REPORT_TYPE: &str = "daily-heartbeat"; + +const SESSION_CORE_METRICS: &[&str; 9] = &[ + METRIC_MF_SYNC_FAILURE, + METRIC_MF_SYNC_SUCCESS, + METRIC_BATTERY_DISCHARGE_DURATION_MS, + METRIC_BATTERY_SOC_PCT_DROP, + METRIC_CONNECTED_TIME, + METRIC_EXPECTED_CONNECTED_TIME, + METRIC_SYNC_FAILURE, + METRIC_SYNC_SUCCESS, + METRIC_OPERATIONAL_CRASHES, +]; + +#[derive(Serialize, Deserialize, Clone, PartialEq, Eq)] +pub enum MetricReportType { + #[serde(rename = "heartbeat")] + Heartbeat, + #[serde(rename = "session")] + Session(SessionName), + #[serde(rename = "daily-heartbeat")] + DailyHeartbeat, +} + +impl MetricReportType { + pub fn as_str(&self) -> &str { + match self { + MetricReportType::Heartbeat => HEARTBEAT_REPORT_TYPE, + MetricReportType::Session(session_name) => session_name.as_str(), + MetricReportType::DailyHeartbeat => DAILY_HEARTBEAT_REPORT_TYPE, + } + } +} + +pub struct MetricReport { + /// In-memory metric store for this report + metrics: HashMap>, + /// Point in time when capture of metrics currently in + /// report's metric store began + start: Instant, + /// Configuration of which metrics this report should capture + captured_metrics: CapturedMetrics, + /// Indicates whether this is a heartbeat metric report or + /// session metric report (with session name) + report_type: MetricReportType, +} + +struct MetricReportSnapshot { + duration: Duration, + metrics: HashMap, +} + +impl MetricReport { + pub fn new( + report_type: MetricReportType, + configured_captured_metrics: CapturedMetrics, + ) -> Self { + // Always include session core metrics regardless of configuration + let captured_metrics = match configured_captured_metrics { + CapturedMetrics::All => CapturedMetrics::All, + CapturedMetrics::Metrics(metrics) => { + let mut merged_set = SESSION_CORE_METRICS + .iter() + .map(|core_metric_key| { + // This expect should never be hit as SESSION_CORE_METRICS is + // an array of static strings + MetricStringKey::from_str(core_metric_key) + .expect("Invalid Metric Key in SESSION_CORE_METRICS") + }) + .collect::>(); + merged_set.extend(metrics); + CapturedMetrics::Metrics(merged_set) + } + }; + + Self { + metrics: HashMap::new(), + start: Instant::now(), + captured_metrics, + report_type, + } + } + + /// Creates a heartbeat report that captures all metrics + pub fn new_heartbeat() -> Self { + MetricReport::new(MetricReportType::Heartbeat, CapturedMetrics::All) + } + + /// Creates a daily heartbeat report that captures all metrics + pub fn new_daily_heartbeat() -> Self { + MetricReport::new(MetricReportType::DailyHeartbeat, CapturedMetrics::All) + } + + fn is_captured(&self, metric_key: &MetricStringKey) -> bool { + match &self.captured_metrics { + CapturedMetrics::Metrics(metric_keys) => metric_keys.contains(metric_key), + CapturedMetrics::All => true, + } + } + + /// Adds a metric reading to the report's internal + /// metric store if the report captures that metric, + /// otherwise no-op + pub fn add_metric(&mut self, m: KeyedMetricReading) -> Result<()> { + if self.is_captured(&m.name) { + match self.metrics.entry(m.name) { + std::collections::hash_map::Entry::Occupied(mut o) => { + let state = o.get_mut(); + if let Err(e) = (*state).aggregate(&m.value) { + *state = Self::select_aggregate_for(&m.value)?; + log::warn!( + "New value for metric {} is incompatible ({}). Resetting timeseries.", + o.key(), + e + ); + } + } + std::collections::hash_map::Entry::Vacant(v) => { + let timeseries = Self::select_aggregate_for(&m.value)?; + v.insert(timeseries); + } + }; + } + Ok(()) + } + + /// Increment a counter metric by 1 + pub fn increment_counter(&mut self, name: &str) -> Result<()> { + self.add_to_counter(name, 1.0) + } + + pub fn add_to_counter(&mut self, name: &str, value: f64) -> Result<()> { + match name.parse::() { + Ok(metric_name) => self.add_metric(KeyedMetricReading::new( + metric_name, + MetricReading::Counter { + value, + timestamp: Utc::now(), + }, + )), + Err(e) => Err(eyre!("Invalid metric name: {} - {}", name, e)), + } + } + + /// Return all the metrics in memory for this report and resets its store. + pub fn take_metrics(&mut self) -> HashMap { + self.take_metric_report_snapshot().metrics + } + + fn take_metric_report_snapshot(&mut self) -> MetricReportSnapshot { + let duration = std::mem::replace(&mut self.start, Instant::now()).elapsed(); + let metrics = std::mem::take(&mut self.metrics) + .into_iter() + .map(|(name, state)| (name, state.value())) + .collect(); + + MetricReportSnapshot { duration, metrics } + } + + /// Create one metric report MAR entry with all the metrics in the store. + /// + /// All data will be timestamped with current time measured by CollectionTime::now(), effectively + /// disregarding the collectd timestamps. + pub fn prepare_metric_report( + &mut self, + mar_staging_area: &Path, + ) -> Result>> { + let snapshot = self.take_metric_report_snapshot(); + + if snapshot.metrics.is_empty() { + return Ok(None); + } + + Ok(Some(MarEntryBuilder::new(mar_staging_area)?.set_metadata( + Metadata::new_metric_report( + snapshot.metrics, + snapshot.duration, + self.report_type.clone(), + ), + ))) + } + + fn select_aggregate_for(event: &MetricReading) -> Result> { + match event { + MetricReading::Histogram { .. } => Ok(Box::new(Histogram::new(event)?)), + MetricReading::Counter { .. } => Ok(Box::new(Counter::new(event)?)), + MetricReading::Gauge { .. } => Ok(Box::new(Gauge::new(event)?)), + MetricReading::TimeWeightedAverage { .. } => { + Ok(Box::new(TimeWeightedAverage::new(event)?)) + } + MetricReading::ReportTag { .. } => Ok(Box::new(ReportTag::new(event)?)), + } + } + + pub fn report_type(&self) -> &MetricReportType { + &self.report_type + } +} + +#[cfg(test)] +mod tests { + use tempfile::TempDir; + + use std::collections::BTreeMap; + + use super::*; + use crate::test_utils::in_histograms; + use std::str::FromStr; + + use insta::assert_json_snapshot; + use rstest::rstest; + + #[rstest] + #[case(in_histograms(vec![("foo", 1.0), ("bar", 2.0), ("baz", 3.0)]), "heartbeat_report_1")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 2.0), ("foo", 3.0)]), "heartbeat_report_2")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 1.0)]), "heartbeat_report_3")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 2.0)]), "heartbeat_report_4")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 2.0), ("foo", 2.0)]), "heartbeat_report_5")] + fn test_aggregate_metrics( + #[case] metrics: impl Iterator, + #[case] test_name: &str, + ) { + let mut metric_report = MetricReport::new_heartbeat(); + + for m in metrics { + metric_report.add_metric(m).unwrap(); + } + let sorted_metrics: BTreeMap<_, _> = metric_report.take_metrics().into_iter().collect(); + assert_json_snapshot!(test_name, sorted_metrics); + } + + #[rstest] + #[case(in_histograms(vec![("foo", 1.0), ("bar", 2.0), ("baz", 3.0)]), "session_report_1")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 2.0), ("foo", 3.0)]), "session_report_2")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 1.0)]), "session_report_3")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 2.0)]), "session_report_4")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 2.0), ("baz", 2.0), ("bar", 3.0)]), "session_report_5")] + fn test_aggregate_metrics_session( + #[case] metrics: impl Iterator, + #[case] test_name: &str, + ) { + let mut metric_report = MetricReport::new( + MetricReportType::Session(SessionName::from_str("foo_only").unwrap()), + CapturedMetrics::Metrics( + [ + MetricStringKey::from_str("foo").unwrap(), + MetricStringKey::from_str("baz").unwrap(), + ] + .into_iter() + .collect(), + ), + ); + + for m in metrics { + metric_report.add_metric(m).unwrap(); + } + let sorted_metrics: BTreeMap<_, _> = metric_report.take_metrics().into_iter().collect(); + assert_json_snapshot!(test_name, sorted_metrics); + } + + /// Core metrics should always be captured by sessions even if they are not + /// configured to do so + #[rstest] + fn test_aggregate_core_metrics_session() { + let mut metric_report = MetricReport::new( + MetricReportType::Session(SessionName::from_str("foo_only").unwrap()), + CapturedMetrics::Metrics( + [ + MetricStringKey::from_str("foo").unwrap(), + MetricStringKey::from_str("baz").unwrap(), + ] + .into_iter() + .collect(), + ), + ); + + let metrics = in_histograms( + SESSION_CORE_METRICS + .map(|metric_name: &'static str| (metric_name, 100.0)) + .to_vec(), + ); + + for m in metrics { + metric_report.add_metric(m).unwrap(); + } + let sorted_metrics: BTreeMap<_, _> = metric_report.take_metrics().into_iter().collect(); + assert_json_snapshot!(sorted_metrics); + } + + #[rstest] + fn test_empty_after_write() { + let mut metric_report = MetricReport::new_heartbeat(); + for m in in_histograms(vec![("foo", 1.0), ("bar", 2.0), ("baz", 3.0)]) { + metric_report.add_metric(m).unwrap(); + } + + let tempdir = TempDir::new().unwrap(); + let _ = metric_report.prepare_metric_report(tempdir.path()); + assert_eq!(metric_report.take_metrics().len(), 0); + } +} diff --git a/memfaultd/src/metrics/metric_report_manager.rs b/memfaultd/src/metrics/metric_report_manager.rs new file mode 100644 index 0000000..79e07c9 --- /dev/null +++ b/memfaultd/src/metrics/metric_report_manager.rs @@ -0,0 +1,464 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::{eyre, Result}; +use log::{debug, error}; +use std::{ + collections::{hash_map::Entry, HashMap}, + path::Path, + sync::{Arc, Mutex}, +}; + +use super::{ + core_metrics::METRIC_OPERATIONAL_CRASHES, metric_reading::KeyedMetricReading, + metric_report::CapturedMetrics, SessionName, +}; +use crate::{ + config::SessionConfig, + mar::{MarEntryBuilder, Metadata}, + metrics::{MetricReport, MetricReportType, MetricStringKey, MetricValue}, + network::NetworkConfig, +}; + +pub struct MetricReportManager { + heartbeat: MetricReport, + daily_heartbeat: MetricReport, + sessions: HashMap, + session_configs: Vec, +} + +impl MetricReportManager { + /// Creates a MetricReportManager with no sessions + /// configured + pub fn new() -> Self { + Self { + heartbeat: MetricReport::new_heartbeat(), + daily_heartbeat: MetricReport::new_daily_heartbeat(), + sessions: HashMap::new(), + session_configs: vec![], + } + } + + pub fn new_with_session_configs(session_configs: &[SessionConfig]) -> Self { + Self { + heartbeat: MetricReport::new_heartbeat(), + daily_heartbeat: MetricReport::new_daily_heartbeat(), + sessions: HashMap::new(), + session_configs: session_configs.to_vec(), + } + } + + /// Starts a session of the specified session name. + /// Fails if the session name provided is not configured. + /// If there is already a session with that name ongoing, + /// this is a no-op + pub fn start_session(&mut self, session_name: SessionName) -> Result<()> { + let report_type = MetricReportType::Session(session_name.clone()); + let captured_metric_keys = self.captured_metric_keys_for_report(&report_type)?; + + if let Entry::Vacant(e) = self.sessions.entry(session_name) { + let session = e.insert(MetricReport::new(report_type, captured_metric_keys)); + // Make sure we always include the operational_crashes counter in every session report. + session.add_to_counter(METRIC_OPERATIONAL_CRASHES, 0.0)?; + } + Ok(()) + } + + /// Returns the metrics the provided session name is configured to capture + fn captured_metric_keys_for_report( + &self, + report_type: &MetricReportType, + ) -> Result { + match report_type { + MetricReportType::Heartbeat => Ok(CapturedMetrics::All), + MetricReportType::DailyHeartbeat => Ok(CapturedMetrics::All), + MetricReportType::Session(session_name) => self + .session_configs + .iter() + .find(|&session_config| session_config.name == *session_name) + .map(|config| { + CapturedMetrics::Metrics(config.captured_metrics.clone().into_iter().collect()) + }) + .ok_or_else(|| eyre!("No configuration for session named {} found!", session_name)), + } + } + + /// Returns an iterator over all ongoing metric reports + fn report_iter(&mut self) -> impl Iterator { + self.sessions + .values_mut() + .chain([&mut self.heartbeat, &mut self.daily_heartbeat]) + } + + /// Adds a metric reading to all ongoing metric reports + /// that capture that metric + pub fn add_metric(&mut self, m: KeyedMetricReading) -> Result<()> { + self.report_iter() + .try_for_each(|report| report.add_metric(m.clone())) + } + + /// Increment a counter metric by 1 + pub fn increment_counter(&mut self, name: &str) -> Result<()> { + self.report_iter() + .try_for_each(|report| report.increment_counter(name)) + } + + /// Increment a counter by a specified amount + pub fn add_to_counter(&mut self, name: &str, value: f64) -> Result<()> { + self.report_iter() + .try_for_each(|report| report.add_to_counter(name, value)) + } + + /// Adds a metric reading to a specific metric report + pub fn add_metric_to_report( + &mut self, + report_type: &MetricReportType, + m: KeyedMetricReading, + ) -> Result<()> { + match report_type { + MetricReportType::Heartbeat => self.heartbeat.add_metric(m), + MetricReportType::DailyHeartbeat => self.daily_heartbeat.add_metric(m), + MetricReportType::Session(session_name) => self + .sessions + .get_mut(session_name) + .ok_or_else(|| eyre!("No ongoing session with name {}", session_name)) + .and_then(|session_report| session_report.add_metric(m)), + } + } + + /// Return all the metrics in memory and resets the + /// store for the periodic heartbeat report. + pub fn take_heartbeat_metrics(&mut self) -> HashMap { + self.heartbeat.take_metrics() + } + + /// Return all the metrics in memory and resets the store + /// for a specified session. + pub fn take_session_metrics( + &mut self, + session_name: &SessionName, + ) -> Result> { + self.sessions + .get_mut(session_name) + .ok_or_else(|| eyre!("No ongoing session with name {}", session_name)) + .map(|session_report| session_report.take_metrics()) + } + + /// Dump the metrics to a MAR entry. + /// + /// This takes a &Arc> and will minimize lock time. + /// This will empty the metrics store. + /// When used with a heartbeat metric report type, the heartbeat + /// will be reset. + /// When used with a session report type, the session will end and + /// be removed from the MetricReportManager's internal sessions HashMap. + pub fn dump_report_to_mar_entry( + metric_report_manager: &Arc>, + mar_staging_area: &Path, + network_config: &NetworkConfig, + report_type: &MetricReportType, + ) -> Result<()> { + let mar_builder = match report_type { + MetricReportType::Heartbeat => metric_report_manager + .lock() + .expect("Mutex Poisoned!") + .heartbeat + .prepare_metric_report(mar_staging_area)?, + MetricReportType::DailyHeartbeat => metric_report_manager + .lock() + .expect("Mutex Poisoned!") + .daily_heartbeat + .prepare_metric_report(mar_staging_area)?, + MetricReportType::Session(session_name) => { + match metric_report_manager + .lock() + .expect("Mutex Poisoned!") + .sessions + .remove(session_name) + { + Some(mut report) => report.prepare_metric_report(mar_staging_area)?, + None => return Err(eyre!("No metric report found for {}", session_name)), + } + } + }; + + // Save to disk after releasing the lock + if let Some(mar_builder) = mar_builder { + let mar_entry = mar_builder + .save(network_config) + .map_err(|e| eyre!("Error building MAR entry: {}", e))?; + debug!( + "Generated MAR entry from metrics: {}", + mar_entry.path.display() + ); + } else { + debug!( + "Skipping generating metrics entry. No metrics in store for: {}", + report_type.as_str() + ) + } + Ok(()) + } + + fn prepare_all_metric_reports( + &mut self, + mar_staging_area: &Path, + ) -> Vec> { + self.report_iter() + .filter_map(|report| { + if let Ok(builder) = report.prepare_metric_report(mar_staging_area) { + builder.or_else(|| { + debug!( + "Skipping generating metrics entry. No metrics in store for: {}", + report.report_type().as_str() + ); + None + }) + } else { + debug!( + "Failed to prepare metric report for: {}", + report.report_type().as_str() + ); + None + } + }) + .collect() + } + + /// Ends all ongoing MetricReports and dumps them as MARs to disk. + /// + /// MetricReports with the MetricReportType specified with + /// exclude_report_types are excluded from this operation + /// entirely + pub fn dump_metric_reports( + metric_report_manager: &Arc>, + mar_staging_area: &Path, + network_config: &NetworkConfig, + ) -> Result<()> { + let mar_builders = metric_report_manager + .lock() + .expect("Mutex poisoned") + .prepare_all_metric_reports(mar_staging_area); + + for mar_builder in mar_builders { + match mar_builder.save(network_config) { + Ok(mar_entry) => debug!( + "Generated MAR entry from metrics: {}", + mar_entry.path.display() + ), + Err(e) => error!("Error building MAR entry: {}", e), + } + } + + Ok(()) + } +} + +impl Default for MetricReportManager { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use tempfile::TempDir; + + use super::*; + use crate::test_utils::in_histograms; + use insta::assert_json_snapshot; + use rstest::rstest; + use std::str::FromStr; + + #[rstest] + #[case(in_histograms(vec![("foo", 1.0), ("bar", 2.0), ("baz", 3.0)]), "heartbeat_report_1")] + #[case(in_histograms(vec![("foo", 1.0), ("foo",2.0), ("foo", 3.0)]), "heartbeat_report_2")] + #[case(in_histograms(vec![("foo", 1.0), ("foo",1.0)]), "heartbeat_report_3")] + #[case(in_histograms(vec![("foo", 1.0), ("foo",2.0)]), "heartbeat_report_4")] + #[case(in_histograms(vec![("foo", 1.0), ("foo",2.0), ("foo", 2.0)]), "heartbeat_report_5")] + fn test_heartbeat_report( + #[case] metrics: impl Iterator, + #[case] test_name: &str, + ) { + let mut metric_report_manager = MetricReportManager::new(); + for m in metrics { + metric_report_manager + .add_metric(m) + .expect("Failed to add metric reading"); + } + + let tempdir = TempDir::new().unwrap(); + let builder = metric_report_manager + .heartbeat + .prepare_metric_report(tempdir.path()) + .unwrap(); + assert_json_snapshot!(test_name, builder.unwrap().get_metadata(), {".metadata.duration_ms" => 0}); + } + + #[rstest] + fn test_unconfigured_session_name_fails() { + let mut metric_report_manager = MetricReportManager::new(); + assert!(metric_report_manager + .start_session(SessionName::from_str("test-session").unwrap()) + .is_err()) + } + + #[rstest] + #[case(in_histograms(vec![("foo", 1.0), ("bar", 2.0), ("baz", 3.0)]), "heartbeat_and_sessions_report_1")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 2.0), ("foo", 3.0)]), "heartbeat_and_sessions_report_2")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 1.0)]), "heartbeat_and_sessions_report_3")] + #[case(in_histograms(vec![("foo", 1.0), ("foo", 2.0), ("baz", 1.0), ("baz", 2.0)]), "heartbeat_and_sessions_report_4")] + #[case(in_histograms(vec![("foo", 1.0), ("bar", 2.0), ("foo", 2.0)]), "heartbeat_and_sessions_report_5")] + fn test_heartbeat_and_session_reports( + #[case] metrics: impl Iterator, + #[case] test_name: &str, + ) { + let session_a_name = SessionName::from_str("test-session-some-metrics").unwrap(); + let session_b_name = SessionName::from_str("test-session-all-metrics").unwrap(); + let session_configs = vec![ + SessionConfig { + name: session_a_name.clone(), + captured_metrics: vec![ + MetricStringKey::from_str("foo").unwrap(), + MetricStringKey::from_str("bar").unwrap(), + ], + }, + SessionConfig { + name: session_b_name.clone(), + captured_metrics: vec![ + MetricStringKey::from_str("foo").unwrap(), + MetricStringKey::from_str("bar").unwrap(), + MetricStringKey::from_str("baz").unwrap(), + ], + }, + ]; + + let mut metric_report_manager = + MetricReportManager::new_with_session_configs(&session_configs); + + assert!(metric_report_manager.start_session(session_a_name).is_ok()); + assert!(metric_report_manager.start_session(session_b_name).is_ok()); + + for m in metrics { + metric_report_manager + .add_metric(m) + .expect("Failed to add metric reading"); + } + + let tempdir = TempDir::new().unwrap(); + // Verify heartbeat report + let snapshot_name = format!("{}.{}", test_name, "heartbeat"); + assert_report_snapshot( + &mut metric_report_manager.heartbeat, + &snapshot_name, + &tempdir, + ); + + // Verify daily heartbeat report + let snapshot_name = format!("{}.{}", test_name, "daily_heartbeat"); + assert_report_snapshot( + &mut metric_report_manager.daily_heartbeat, + &snapshot_name, + &tempdir, + ); + + for (session_name, mut metric_report) in metric_report_manager.sessions { + let snapshot_name = format!("{}.{}", test_name, session_name); + assert_report_snapshot(&mut metric_report, &snapshot_name, &tempdir); + } + } + + #[rstest] + fn test_start_session_twice() { + let session_name = SessionName::from_str("test-session-start-twice").unwrap(); + let session_configs = vec![SessionConfig { + name: session_name.clone(), + captured_metrics: vec![ + MetricStringKey::from_str("foo").unwrap(), + MetricStringKey::from_str("bar").unwrap(), + ], + }]; + + let mut metric_report_manager = + MetricReportManager::new_with_session_configs(&session_configs); + + let metrics_a = in_histograms(vec![("foo", 1.0), ("bar", 2.0)]); + assert!(metric_report_manager + .start_session(session_name.clone()) + .is_ok()); + for m in metrics_a { + metric_report_manager + .add_metric(m) + .expect("Failed to add metric reading"); + } + + // Final metric report should aggregate both metrics_a and + // metrics_b as the session should not be restarted + // by the second start_session + let metrics_b = in_histograms(vec![("foo", 9.0), ("bar", 5.0)]); + assert!(metric_report_manager + .start_session(session_name.clone()) + .is_ok()); + for m in metrics_b { + metric_report_manager + .add_metric(m) + .expect("Failed to add metric reading"); + } + + let tempdir = TempDir::new().unwrap(); + let builder = metric_report_manager + .sessions + .get_mut(&session_name) + .unwrap() + .prepare_metric_report(tempdir.path()) + .unwrap(); + + assert_json_snapshot!(builder.unwrap().get_metadata(), {".metadata.duration_ms" => 0}); + } + + #[rstest] + fn test_prepare_all_prepares_sessions() { + let session_name = SessionName::from_str("test-session").unwrap(); + let session_configs = vec![SessionConfig { + name: session_name.clone(), + captured_metrics: vec![ + MetricStringKey::from_str("foo").unwrap(), + MetricStringKey::from_str("bar").unwrap(), + ], + }]; + + let mut metric_report_manager = + MetricReportManager::new_with_session_configs(&session_configs); + + let metrics = in_histograms(vec![("foo", 5.0), ("bar", 3.5)]); + assert!(metric_report_manager.start_session(session_name).is_ok()); + for m in metrics { + metric_report_manager + .add_metric(m) + .expect("Failed to add metric reading"); + } + + let tempdir = TempDir::new().unwrap(); + let builders = metric_report_manager.prepare_all_metric_reports(tempdir.path()); + + // 3 MAR builders should be created for "heartbeat", "daily-heartbeat", and "test-session" + // Note this only works because report_iter() with only 1 session is deterministic + for builder in builders { + match builder.get_metadata() { + Metadata::LinuxMetricReport { report_type, .. } => { + assert_json_snapshot!(report_type.as_str(), builder.get_metadata(), {".metadata.duration_ms" => 0}) + } + _ => panic!("Invalid MAR builder"), + } + } + } + + fn assert_report_snapshot( + metric_report: &mut MetricReport, + snapshot_name: &str, + tempdir: &TempDir, + ) { + let builder = metric_report.prepare_metric_report(tempdir.path()).unwrap(); + assert_json_snapshot!(snapshot_name, builder.unwrap().get_metadata(), {".metadata.duration_ms" => 0}); + } +} diff --git a/memfaultd/src/metrics/metric_string_key.rs b/memfaultd/src/metrics/metric_string_key.rs new file mode 100644 index 0000000..9d48030 --- /dev/null +++ b/memfaultd/src/metrics/metric_string_key.rs @@ -0,0 +1,115 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use nom::{ + combinator::map_res, + error::ParseError, + {AsChar, IResult, InputTakeAtPosition}, +}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::fmt::{Debug, Formatter}; +use std::str::FromStr; +use std::{borrow::Cow, fmt::Display}; + +/// Struct containing a valid metric / attribute key. +#[derive(Clone, PartialEq, Eq, Hash)] +#[repr(transparent)] +pub struct MetricStringKey { + inner: String, +} + +impl MetricStringKey { + pub fn as_str(&self) -> &str { + &self.inner + } + + pub fn metric_string_key_parser>(input: T) -> IResult + where + T: InputTakeAtPosition, + ::Item: AsChar, + { + input.split_at_position_complete(|item| { + let c = item.as_char(); + !(c.is_alphanumeric() || c == '_' || c == '/' || c == '.') + }) + } + + pub fn parse(input: &str) -> IResult<&str, Self> { + map_res(Self::metric_string_key_parser, Self::from_str)(input) + } +} + +impl Debug for MetricStringKey { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + std::fmt::Debug::fmt(&self.inner, f) + } +} + +impl FromStr for MetricStringKey { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + if !(1..=128).contains(&s.len()) { + return Err("Invalid key: must be between 1 and 128 characters"); + } + if !s.is_ascii() { + return Err("Invalid key: must be ASCII"); + } + Ok(Self { + inner: s.to_string(), + }) + } +} + +impl Display for MetricStringKey { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(&self.inner, f) + } +} + +impl Ord for MetricStringKey { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.inner.cmp(&other.inner) + } +} +impl PartialOrd for MetricStringKey { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.inner.cmp(&other.inner)) + } +} + +impl Serialize for MetricStringKey { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> Deserialize<'de> for MetricStringKey { + fn deserialize>(deserializer: D) -> Result { + let s: Cow = Deserialize::deserialize(deserializer)?; + let key: MetricStringKey = str::parse(&s).map_err(serde::de::Error::custom)?; + Ok(key) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::*; + + #[rstest] + #[case("", "Invalid key: must be between 1 and 128 characters")] + #[case("\u{1F4A9}", "Invalid key: must be ASCII")] + fn validation_errors(#[case] input: &str, #[case] expected: &str) { + let result: Result = str::parse(input); + assert_eq!(result.err().unwrap(), expected); + } + + #[rstest] + #[case("foo")] + #[case("weird valid.key-123$")] + fn parsed_ok(#[case] input: &str) { + let result: Result = str::parse(input); + assert_eq!(result.ok().unwrap().as_str(), input); + } +} diff --git a/memfaultd/src/metrics/metric_value.rs b/memfaultd/src/metrics/metric_value.rs new file mode 100644 index 0000000..ceb4724 --- /dev/null +++ b/memfaultd/src/metrics/metric_value.rs @@ -0,0 +1,24 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::Result; +use serde::{Deserialize, Serialize, Serializer}; + +#[derive(Clone, Debug, PartialEq, Deserialize)] +#[serde(untagged)] +pub enum MetricValue { + Number(f64), + String(String), +} + +impl Serialize for MetricValue { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self { + MetricValue::Number(v) => serializer.serialize_f64(*v), + MetricValue::String(v) => serializer.serialize_str(v.as_str()), + } + } +} diff --git a/memfaultd/src/metrics/mod.rs b/memfaultd/src/metrics/mod.rs new file mode 100644 index 0000000..8261daa --- /dev/null +++ b/memfaultd/src/metrics/mod.rs @@ -0,0 +1,55 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use chrono::{DateTime, Utc}; + +mod battery; +pub use battery::BatteryMonitor; +pub use battery::BatteryMonitorReading; +pub use battery::BatteryReadingHandler; + +mod connectivity; +pub use connectivity::ConnectivityMonitor; +pub use connectivity::ReportSyncEventHandler; + +mod metric_string_key; +pub use metric_string_key::MetricStringKey; + +mod metric_report; +pub use metric_report::MetricReport; +pub use metric_report::MetricReportType; + +mod metric_report_manager; +pub use metric_report_manager::MetricReportManager; + +mod metric_reading; +pub use metric_reading::KeyedMetricReading; +pub use metric_reading::MetricReading; + +mod timeseries; + +mod metric_value; +pub use metric_value::MetricValue; + +pub type MetricTimestamp = DateTime; + +mod periodic_metric_report; +pub use periodic_metric_report::PeriodicMetricReportDumper; + +mod crashfree_interval; +pub use crashfree_interval::CrashFreeIntervalTracker; + +mod session_name; +pub use session_name::SessionName; + +mod session_event_handler; +pub use session_event_handler::SessionEventHandler; + +pub mod core_metrics; + +pub mod statsd_server; +pub use statsd_server::StatsDServer; + +mod system_metrics; +pub use system_metrics::SystemMetricsCollector; +pub use system_metrics::BUILTIN_SYSTEM_METRIC_NAMESPACES; diff --git a/memfaultd/src/metrics/periodic_metric_report.rs b/memfaultd/src/metrics/periodic_metric_report.rs new file mode 100644 index 0000000..3707ba1 --- /dev/null +++ b/memfaultd/src/metrics/periodic_metric_report.rs @@ -0,0 +1,131 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + path::PathBuf, + sync::{Arc, Mutex}, + thread::sleep, + time::{Duration, Instant}, +}; + +use log::warn; + +use crate::network::NetworkConfig; + +use super::{MetricReportManager, MetricReportType}; + +/// A runner that periodically dumps metrics to a MAR entry +/// +/// This runner will periodically dump metrics of a given type to a MAR entry +/// in the background. The runner will sleep between dumps. +pub struct PeriodicMetricReportDumper { + metric_report_manager: Arc>, + net_config: NetworkConfig, + report_interval: Duration, + mar_staging_path: PathBuf, + report_type: MetricReportType, +} + +impl PeriodicMetricReportDumper { + pub fn new( + mar_staging_path: PathBuf, + net_config: NetworkConfig, + metric_report_manager: Arc>, + report_interval: Duration, + report_type: MetricReportType, + ) -> Self { + Self { + metric_report_manager, + net_config, + report_interval, + mar_staging_path, + report_type, + } + } + + pub fn start(&self) { + let mut next_report = Instant::now() + self.report_interval; + loop { + while Instant::now() < next_report { + sleep(next_report - Instant::now()); + } + + self.run_once(&mut next_report); + } + } + + fn run_once(&self, next_report: &mut Instant) { + *next_report += self.report_interval; + if let Err(e) = MetricReportManager::dump_report_to_mar_entry( + &self.metric_report_manager, + &self.mar_staging_path, + &self.net_config, + &self.report_type, + ) { + warn!("Unable to dump metrics: {}", e); + } + } +} + +#[cfg(test)] +mod test { + use tempfile::tempdir; + + use super::*; + + use crate::mar::manifest::Metadata; + use crate::test_utils::in_histograms; + + #[test] + fn test_happy_path_metric_report() { + let report_manager = Arc::new(Mutex::new(MetricReportManager::new())); + let tempdir = tempdir().unwrap(); + let mar_staging_path = tempdir.path().to_owned(); + let readings = in_histograms(vec![("hello", 10.0), ("mad", 20.0)]).collect::>(); + + { + let mut report_manager = report_manager.lock().unwrap(); + + for reading in &readings { + report_manager.add_metric(reading.clone()).unwrap(); + } + } + + let report_interval = Duration::from_secs(1); + let runner = PeriodicMetricReportDumper::new( + mar_staging_path.clone(), + NetworkConfig::test_fixture(), + report_manager, + report_interval, + MetricReportType::Heartbeat, + ); + let mut next_report = Instant::now(); + let report_start = next_report; + runner.run_once(&mut next_report); + + let entries = crate::mar::MarEntry::iterate_from_container(&mar_staging_path) + .unwrap() + .map(Result::unwrap) + .collect::>(); + + assert_eq!(next_report.duration_since(report_start), report_interval); + assert_eq!(entries.len(), 1); + let metadata = &entries[0].manifest.metadata; + match metadata { + Metadata::LinuxMetricReport { + metrics, + report_type, + .. + } => { + assert_eq!(metrics.len(), 2); + + matches!(report_type, &MetricReportType::Heartbeat); + + for reading in readings { + assert!(metrics.contains_key(&reading.name)); + } + } + _ => panic!("Unexpected metadata"), + } + } +} diff --git a/memfaultd/src/metrics/session_event_handler.rs b/memfaultd/src/metrics/session_event_handler.rs new file mode 100644 index 0000000..26b2756 --- /dev/null +++ b/memfaultd/src/metrics/session_event_handler.rs @@ -0,0 +1,447 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + io::Read, + path::PathBuf, + str::{from_utf8, FromStr}, + sync::{Arc, Mutex}, +}; + +use eyre::{eyre, Result}; +use log::warn; +use tiny_http::{Method, Request, Response}; + +use crate::{ + http_server::{HttpHandler, HttpHandlerResult, SessionRequest}, + metrics::{MetricReportManager, MetricReportType, SessionName}, + network::NetworkConfig, +}; + +use super::KeyedMetricReading; + +/// A server that listens for session management requests +#[derive(Clone)] +pub struct SessionEventHandler { + data_collection_enabled: bool, + metrics_store: Arc>, + mar_staging_path: PathBuf, + network_config: NetworkConfig, +} + +impl SessionEventHandler { + pub fn new( + data_collection_enabled: bool, + metrics_store: Arc>, + mar_staging_path: PathBuf, + network_config: NetworkConfig, + ) -> Self { + Self { + data_collection_enabled, + metrics_store, + mar_staging_path, + network_config, + } + } + + fn parse_request(stream: &mut dyn Read) -> Result { + let mut buf = vec![]; + stream.read_to_end(&mut buf)?; + let body = from_utf8(&buf)?; + match serde_json::from_str(body) { + Ok(request_body) => Ok(request_body), + // Fall back to legacy API, SessionName as raw str in body (no JSON) + Err(e) => { + // If the request doesn't match either the JSON API or legacy API, + // include JSON API parse error in response (as that is currently + // the standard API) + Ok(SessionRequest::new_without_readings( + SessionName::from_str(body) + .map_err(|_| eyre!("Couldn't parse request: {}", e))?, + )) + } + } + } + + fn add_metric_readings_to_session( + session_name: &SessionName, + metric_reports: Arc>, + metric_readings: Vec, + ) -> Result<()> { + let mut metric_reports = metric_reports.lock().expect("Mutex poisoned!"); + for metric_reading in metric_readings { + metric_reports.add_metric_to_report( + &MetricReportType::Session(session_name.clone()), + metric_reading, + )? + } + + Ok(()) + } +} + +impl HttpHandler for SessionEventHandler { + fn handle_request(&self, request: &mut Request) -> HttpHandlerResult { + if (request.url() != "/v1/session/start" && request.url() != "/v1/session/end") + || *request.method() != Method::Post + { + return HttpHandlerResult::NotHandled; + } + + if self.data_collection_enabled { + match Self::parse_request(request.as_reader()) { + Ok(SessionRequest { + session_name, + readings, + }) => { + if request.url() == "/v1/session/start" { + if let Err(e) = self + .metrics_store + .lock() + .expect("Mutex poisoned") + .start_session(session_name.clone()) + { + return HttpHandlerResult::Error(format!( + "Failed to start session: {:?}", + e + )); + } + } + + // Add additional metric readings after the session has started + // (if starting) but before it has ended (if ending) + if !readings.is_empty() { + if let Err(e) = Self::add_metric_readings_to_session( + &session_name, + self.metrics_store.clone(), + readings, + ) { + warn!("Failed to add metrics to session report: {}", e); + } + } + + if request.url() == "/v1/session/end" { + if let Err(e) = MetricReportManager::dump_report_to_mar_entry( + &self.metrics_store, + &self.mar_staging_path, + &self.network_config, + &MetricReportType::Session(session_name), + ) { + return HttpHandlerResult::Error(format!( + "Failed to end session: {:?}", + e + )); + } + } + } + Err(e) => { + warn!("Failed to parse session request: {:?}", e); + return HttpHandlerResult::Error(format!( + "Failed to parse session request: {:?}", + e + )); + } + } + } + HttpHandlerResult::Response(Response::empty(200).boxed()) + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::BTreeMap, + path::Path, + str::FromStr, + sync::{Arc, Mutex}, + }; + + use insta::assert_json_snapshot; + use rstest::{fixture, rstest}; + use tempfile::TempDir; + use tiny_http::{Method, TestRequest}; + + use crate::{ + config::SessionConfig, + http_server::{HttpHandler, HttpHandlerResult}, + mar::manifest::{Manifest, Metadata}, + metrics::{MetricReportManager, MetricStringKey, SessionName}, + network::NetworkConfig, + test_utils::in_histograms, + }; + + use super::*; + use crate::test_utils::setup_logger; + + #[rstest] + fn test_start_without_stop_session(handler: SessionEventHandler) { + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/session/start") + .with_body("test-session"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + let mut metric_report_manager = handler.metrics_store.lock().unwrap(); + let readings = in_histograms(vec![("foo", 1.0), ("bar", 2.0), ("not-captured", 3.0)]); + + for reading in readings { + metric_report_manager + .add_metric(reading) + .expect("Failed to add metric reading"); + } + + let metrics: BTreeMap<_, _> = metric_report_manager + .take_session_metrics(&SessionName::from_str("test-session").unwrap()) + .unwrap() + .into_iter() + .collect(); + + assert_json_snapshot!(metrics); + } + + #[rstest] + fn test_start_with_metrics(_setup_logger: (), handler: SessionEventHandler) { + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/session/start") + .with_body("{\"session_name\": \"test-session\", + \"readings\": + [ + {\"name\": \"foo\", \"value\": {\"Gauge\": {\"value\": 1.0, \"timestamp\": \"2024-01-01 00:00:00 UTC\"}}}, + {\"name\": \"bar\", \"value\": {\"Gauge\": {\"value\": 4.0, \"timestamp\": \"2024-01-01 00:00:00 UTC\"}}}, + {\"name\": \"baz\", \"value\": {\"ReportTag\": {\"value\": \"test-tag\", \"timestamp\": \"2024-01-01 00:00:00 UTC\"}}} + ] + }"); + let response = handler.handle_request(&mut r.into()); + assert!(matches!(response, HttpHandlerResult::Response(_))); + + let mut metric_report_manager = handler.metrics_store.lock().unwrap(); + let metrics: BTreeMap<_, _> = metric_report_manager + .take_session_metrics(&SessionName::from_str("test-session").unwrap()) + .unwrap() + .into_iter() + .collect(); + + assert_json_snapshot!(metrics); + } + + #[rstest] + fn test_end_with_metrics(_setup_logger: ()) { + let session_config = SessionConfig { + name: SessionName::from_str("test-session").unwrap(), + captured_metrics: vec![ + MetricStringKey::from_str("foo").unwrap(), + MetricStringKey::from_str("bar").unwrap(), + ], + }; + + let tempdir = TempDir::new().unwrap(); + let handler = SessionEventHandler::new( + true, + Arc::new(Mutex::new(MetricReportManager::new_with_session_configs( + &[session_config], + ))), + tempdir.path().to_path_buf(), + NetworkConfig::test_fixture(), + ); + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/session/start") + .with_body("test-session"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/session/end") + .with_body("{\"session_name\": \"test-session\", + \"readings\": + [ + {\"name\": \"foo\", \"value\": {\"Gauge\": {\"value\": 1.0, \"timestamp\": \"2024-01-01 00:00:00 UTC\"}}}, + {\"name\": \"bar\", \"value\": {\"Gauge\": {\"value\": 3.0, \"timestamp\": \"2024-01-01 00:00:00 UTC\"}}} + ] + }"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + verify_dumped_metric_report(&handler.mar_staging_path, "end_with_metrics") + } + + #[rstest] + fn test_start_twice_without_stop_session(_setup_logger: (), handler: SessionEventHandler) { + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/session/start") + .with_body("test-session"); + + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + { + let mut metric_report_manager = handler.metrics_store.lock().unwrap(); + let readings = + in_histograms(vec![("foo", 10.0), ("bar", 20.0), ("not-captured", 30.0)]); + + for reading in readings { + metric_report_manager + .add_metric(reading) + .expect("Failed to add metric reading"); + } + } + + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/session/start") + .with_body("test-session"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + let mut metric_report_manager = handler.metrics_store.lock().unwrap(); + let readings = in_histograms(vec![("foo", 1.0), ("bar", 2.0), ("not-captured", 3.0)]); + + for reading in readings { + metric_report_manager + .add_metric(reading) + .expect("Failed to add metric reading"); + } + + let metrics: BTreeMap<_, _> = metric_report_manager + .take_session_metrics(&SessionName::from_str("test-session").unwrap()) + .unwrap() + .into_iter() + .collect(); + + assert_json_snapshot!(metrics); + } + + #[rstest] + fn test_start_then_stop_session(_setup_logger: ()) { + let session_config = SessionConfig { + name: SessionName::from_str("test-session").unwrap(), + captured_metrics: vec![ + MetricStringKey::from_str("foo").unwrap(), + MetricStringKey::from_str("bar").unwrap(), + MetricStringKey::from_str("baz").unwrap(), + ], + }; + + let tempdir = TempDir::new().unwrap(); + let handler = SessionEventHandler::new( + true, + Arc::new(Mutex::new(MetricReportManager::new_with_session_configs( + &[session_config], + ))), + tempdir.path().to_path_buf(), + NetworkConfig::test_fixture(), + ); + + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/session/start") + .with_body("{\"session_name\": \"test-session\", \"readings\": []}"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + { + let mut metric_report_manager = handler.metrics_store.lock().unwrap(); + let readings = in_histograms(vec![("bar", 20.0), ("not-captured", 30.0)]); + + for reading in readings { + metric_report_manager + .add_metric(reading) + .expect("Failed to add metric reading"); + } + } + + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/session/end") + .with_body("{\"session_name\": \"test-session\", + \"readings\": + [ + {\"name\": \"foo\", \"value\": {\"Gauge\": {\"value\": 100, \"timestamp\": \"2024-01-01 00:00:00 UTC\"}}}, + {\"name\": \"baz\", \"value\": {\"ReportTag\": {\"value\": \"test-tag\", \"timestamp\": \"2024-01-01 00:00:00 UTC\"}}} + ] + }"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Response(_) + )); + + verify_dumped_metric_report(&handler.mar_staging_path, "start_then_stop"); + let mut metric_report_manager = handler.metrics_store.lock().unwrap(); + + // Should error as session should have been removed from MetricReportManager + // after it was ended + assert!(metric_report_manager + .take_session_metrics(&SessionName::from_str("test-session").unwrap()) + .is_err()); + } + + #[rstest] + fn test_stop_without_start_session(handler: SessionEventHandler) { + let r = TestRequest::new() + .with_method(Method::Post) + .with_path("/v1/session/end") + .with_body("test-session"); + assert!(matches!( + handler.handle_request(&mut r.into()), + HttpHandlerResult::Error(_) + )); + } + + /// Creates a SessionEventHandler whose metric store is configured with + /// a "test-session" session that captures the "foo" and "bar" metrics + #[fixture] + fn handler() -> SessionEventHandler { + let session_config = SessionConfig { + name: SessionName::from_str("test-session").unwrap(), + captured_metrics: vec![ + MetricStringKey::from_str("foo").unwrap(), + MetricStringKey::from_str("bar").unwrap(), + MetricStringKey::from_str("baz").unwrap(), + ], + }; + + SessionEventHandler::new( + true, + Arc::new(Mutex::new(MetricReportManager::new_with_session_configs( + &[session_config], + ))), + TempDir::new().unwrap().path().to_path_buf(), + NetworkConfig::test_fixture(), + ) + } + + fn verify_dumped_metric_report(mar_staging_path: &Path, test_name: &str) { + let mar_dir = std::fs::read_dir(mar_staging_path) + .expect("Failed to read temp dir") + .filter_map(|entry| entry.ok()) + .collect::>(); + + // There should only be an entry for the reboot reason + assert_eq!(mar_dir.len(), 1); + + let mar_manifest = mar_dir[0].path().join("manifest.json"); + let manifest_string = std::fs::read_to_string(mar_manifest).unwrap(); + let manifest: Manifest = serde_json::from_str(&manifest_string).unwrap(); + + if let Metadata::LinuxMetricReport { .. } = manifest.metadata { + assert_json_snapshot!(test_name, manifest.metadata); + } else { + panic!("Unexpected metadata type"); + } + } +} diff --git a/memfaultd/src/metrics/session_name.rs b/memfaultd/src/metrics/session_name.rs new file mode 100644 index 0000000..5e6465a --- /dev/null +++ b/memfaultd/src/metrics/session_name.rs @@ -0,0 +1,94 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::fmt::{Debug, Formatter}; +use std::str::FromStr; +use std::{borrow::Cow, fmt::Display}; + +use eyre::{eyre, ErrReport, Result}; + +use crate::metrics::metric_report::{DAILY_HEARTBEAT_REPORT_TYPE, HEARTBEAT_REPORT_TYPE}; +use crate::util::patterns::alphanum_slug_is_valid_and_starts_alpha; +const RESERVED_SESSION_NAMES: &[&str; 2] = &[HEARTBEAT_REPORT_TYPE, DAILY_HEARTBEAT_REPORT_TYPE]; + +/// Struct containing a valid session name +#[derive(Clone, PartialEq, Eq, Hash, Ord, PartialOrd)] +#[repr(transparent)] +pub struct SessionName { + inner: String, +} + +impl SessionName { + pub fn as_str(&self) -> &str { + &self.inner + } +} + +impl Debug for SessionName { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + std::fmt::Debug::fmt(&self.inner, f) + } +} + +impl FromStr for SessionName { + type Err = ErrReport; + + fn from_str(s: &str) -> Result { + match alphanum_slug_is_valid_and_starts_alpha(s, 64) { + Ok(()) => { + if RESERVED_SESSION_NAMES.contains(&s) { + Err(eyre!("Cannot use reserved session name: {}", s)) + } else { + Ok(Self { + inner: s.to_string(), + }) + } + } + Err(e) => Err(eyre!("Invalid session name {}: {}", s, e)), + } + } +} + +impl Display for SessionName { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(&self.inner, f) + } +} + +impl Serialize for SessionName { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(self.as_str()) + } +} + +impl<'de> Deserialize<'de> for SessionName { + fn deserialize>(deserializer: D) -> Result { + let s: Cow = Deserialize::deserialize(deserializer)?; + let name: SessionName = str::parse(&s).map_err(serde::de::Error::custom)?; + Ok(name) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::*; + + #[rstest] + #[case("")] + #[case("\u{1F4A9}")] + #[case("Wi-fi Connected")] + #[case("heartbeat")] + #[case("daily-heartbeat")] + fn validation_errors(#[case] input: &str) { + assert!(SessionName::from_str(input).is_err()) + } + + #[rstest] + #[case("foo")] + #[case("valid_session-name")] + fn parsed_ok(#[case] input: &str) { + assert!(SessionName::from_str(input).is_ok()) + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__aggregate_core_metrics_session.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__aggregate_core_metrics_session.snap new file mode 100644 index 0000000..6095daf --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__aggregate_core_metrics_session.snap @@ -0,0 +1,15 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "battery_discharge_duration_ms": 100.0, + "battery_soc_pct_drop": 100.0, + "connectivity_connected_time_ms": 100.0, + "connectivity_expected_time_ms": 100.0, + "operational_crashes": 100.0, + "sync_failure": 100.0, + "sync_memfault_failure": 100.0, + "sync_memfault_successful": 100.0, + "sync_successful": 100.0 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_1.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_1.snap new file mode 100644 index 0000000..b66b75d --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_1.snap @@ -0,0 +1,9 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "bar": 2.0, + "baz": 3.0, + "foo": 1.0 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_2.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_2.snap new file mode 100644 index 0000000..4420475 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_2.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "foo": 2.0 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_3.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_3.snap new file mode 100644 index 0000000..9a3a9ca --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_3.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "foo": 1.0 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_4.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_4.snap new file mode 100644 index 0000000..de0e20b --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_4.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "foo": 1.5 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_5.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_5.snap new file mode 100644 index 0000000..74f3f37 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__heartbeat_report_5.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "foo": 1.6666666666666667 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_1.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_1.snap new file mode 100644 index 0000000..089dbde --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_1.snap @@ -0,0 +1,8 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "baz": 3.0, + "foo": 1.0 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_2.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_2.snap new file mode 100644 index 0000000..4420475 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_2.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "foo": 2.0 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_3.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_3.snap new file mode 100644 index 0000000..9a3a9ca --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_3.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "foo": 1.0 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_4.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_4.snap new file mode 100644 index 0000000..de0e20b --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_4.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "foo": 1.5 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_5.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_5.snap new file mode 100644 index 0000000..468b5d0 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report__tests__session_report_5.snap @@ -0,0 +1,8 @@ +--- +source: memfaultd/src/metrics/metric_report.rs +expression: sorted_metrics +--- +{ + "baz": 2.0, + "foo": 1.5 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__daily-heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__daily-heartbeat.snap new file mode 100644 index 0000000..2dba7e8 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__daily-heartbeat.snap @@ -0,0 +1,15 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 3.5, + "foo": 5.0 + }, + "duration_ms": 0, + "report_type": "daily-heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat.snap new file mode 100644 index 0000000..49f4490 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat.snap @@ -0,0 +1,15 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 3.5, + "foo": 5.0 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.daily_heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.daily_heartbeat.snap new file mode 100644 index 0000000..7d20519 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.daily_heartbeat.snap @@ -0,0 +1,16 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 2.0, + "baz": 3.0, + "foo": 1.0 + }, + "duration_ms": 0, + "report_type": "daily-heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.heartbeat.snap new file mode 100644 index 0000000..c5d86b3 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.heartbeat.snap @@ -0,0 +1,16 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 2.0, + "baz": 3.0, + "foo": 1.0 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.test-session-all-metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.test-session-all-metrics.snap new file mode 100644 index 0000000..249b685 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.test-session-all-metrics.snap @@ -0,0 +1,19 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 2.0, + "baz": 3.0, + "foo": 1.0, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-all-metrics" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.test-session-some-metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.test-session-some-metrics.snap new file mode 100644 index 0000000..3dfcdca --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_1.test-session-some-metrics.snap @@ -0,0 +1,18 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 2.0, + "foo": 1.0, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-some-metrics" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.daily_heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.daily_heartbeat.snap new file mode 100644 index 0000000..c3c95fc --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.daily_heartbeat.snap @@ -0,0 +1,14 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 2.0 + }, + "duration_ms": 0, + "report_type": "daily-heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.heartbeat.snap new file mode 100644 index 0000000..5dca47d --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.heartbeat.snap @@ -0,0 +1,14 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 2.0 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.test-session-all-metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.test-session-all-metrics.snap new file mode 100644 index 0000000..a18ecb4 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.test-session-all-metrics.snap @@ -0,0 +1,17 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 2.0, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-all-metrics" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.test-session-some-metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.test-session-some-metrics.snap new file mode 100644 index 0000000..8d5b783 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_2.test-session-some-metrics.snap @@ -0,0 +1,17 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 2.0, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-some-metrics" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.daily_heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.daily_heartbeat.snap new file mode 100644 index 0000000..dfbc85a --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.daily_heartbeat.snap @@ -0,0 +1,14 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 1.0 + }, + "duration_ms": 0, + "report_type": "daily-heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.heartbeat.snap new file mode 100644 index 0000000..4499c05 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.heartbeat.snap @@ -0,0 +1,14 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 1.0 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.test-session-all-metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.test-session-all-metrics.snap new file mode 100644 index 0000000..86dab3c --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.test-session-all-metrics.snap @@ -0,0 +1,17 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 1.0, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-all-metrics" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.test-session-some-metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.test-session-some-metrics.snap new file mode 100644 index 0000000..f7fb507 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_3.test-session-some-metrics.snap @@ -0,0 +1,17 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 1.0, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-some-metrics" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.daily_heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.daily_heartbeat.snap new file mode 100644 index 0000000..07efa1e --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.daily_heartbeat.snap @@ -0,0 +1,15 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "baz": 1.5, + "foo": 1.5 + }, + "duration_ms": 0, + "report_type": "daily-heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.heartbeat.snap new file mode 100644 index 0000000..31599bd --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.heartbeat.snap @@ -0,0 +1,15 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "baz": 1.5, + "foo": 1.5 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.test-session-all-metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.test-session-all-metrics.snap new file mode 100644 index 0000000..4843681 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.test-session-all-metrics.snap @@ -0,0 +1,18 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "baz": 1.5, + "foo": 1.5, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-all-metrics" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.test-session-some-metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.test-session-some-metrics.snap new file mode 100644 index 0000000..5d9d0fd --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_4.test-session-some-metrics.snap @@ -0,0 +1,17 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 1.5, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-some-metrics" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.daily_heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.daily_heartbeat.snap new file mode 100644 index 0000000..5c86285 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.daily_heartbeat.snap @@ -0,0 +1,15 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 2.0, + "foo": 1.5 + }, + "duration_ms": 0, + "report_type": "daily-heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.heartbeat.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.heartbeat.snap new file mode 100644 index 0000000..e690567 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.heartbeat.snap @@ -0,0 +1,15 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 2.0, + "foo": 1.5 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.test-session-all-metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.test-session-all-metrics.snap new file mode 100644 index 0000000..4229d9e --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.test-session-all-metrics.snap @@ -0,0 +1,18 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 2.0, + "foo": 1.5, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-all-metrics" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.test-session-some-metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.test-session-some-metrics.snap new file mode 100644 index 0000000..b29a3ce --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_and_sessions_report_5.test-session-some-metrics.snap @@ -0,0 +1,18 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 2.0, + "foo": 1.5, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-some-metrics" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_1.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_1.snap new file mode 100644 index 0000000..c5d86b3 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_1.snap @@ -0,0 +1,16 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 2.0, + "baz": 3.0, + "foo": 1.0 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_2.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_2.snap new file mode 100644 index 0000000..5dca47d --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_2.snap @@ -0,0 +1,14 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 2.0 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_3.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_3.snap new file mode 100644 index 0000000..4499c05 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_3.snap @@ -0,0 +1,14 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 1.0 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_4.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_4.snap new file mode 100644 index 0000000..be2df83 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_4.snap @@ -0,0 +1,14 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 1.5 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_5.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_5.snap new file mode 100644 index 0000000..b783ecc --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__heartbeat_report_5.snap @@ -0,0 +1,14 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "foo": 1.6666666666666667 + }, + "duration_ms": 0, + "report_type": "heartbeat" + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__start_session_twice.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__start_session_twice.snap new file mode 100644 index 0000000..caf83fd --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__start_session_twice.snap @@ -0,0 +1,18 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.unwrap().get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 3.5, + "foo": 5.0, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session-start-twice" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__test-session.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__test-session.snap new file mode 100644 index 0000000..bb5d7b8 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__metric_report_manager__tests__test-session.snap @@ -0,0 +1,18 @@ +--- +source: memfaultd/src/metrics/metric_report_manager.rs +expression: builder.get_metadata() +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 3.5, + "foo": 5.0, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__end_with_metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__end_with_metrics.snap new file mode 100644 index 0000000..58f2ccd --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__end_with_metrics.snap @@ -0,0 +1,18 @@ +--- +source: memfaultd/src/metrics/session_event_handler.rs +expression: manifest.metadata +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 3.0, + "foo": 1.0, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_then_stop.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_then_stop.snap new file mode 100644 index 0000000..266ebc6 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_then_stop.snap @@ -0,0 +1,19 @@ +--- +source: memfaultd/src/metrics/session_event_handler.rs +expression: manifest.metadata +--- +{ + "type": "linux-metric-report", + "metadata": { + "metrics": { + "bar": 20.0, + "baz": "test-tag", + "foo": 100.0, + "operational_crashes": 0.0 + }, + "duration_ms": 0, + "report_type": { + "session": "test-session" + } + } +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_twice_without_stop_session.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_twice_without_stop_session.snap new file mode 100644 index 0000000..bb3f755 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_twice_without_stop_session.snap @@ -0,0 +1,9 @@ +--- +source: memfaultd/src/metrics/session_event_handler.rs +expression: metrics +--- +{ + "bar": 11.0, + "foo": 5.5, + "operational_crashes": 0.0 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_with_metrics.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_with_metrics.snap new file mode 100644 index 0000000..3e8e135 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_with_metrics.snap @@ -0,0 +1,10 @@ +--- +source: memfaultd/src/metrics/session_event_handler.rs +expression: metrics +--- +{ + "bar": 4.0, + "baz": "test-tag", + "foo": 1.0, + "operational_crashes": 0.0 +} diff --git a/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_without_stop_session.snap b/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_without_stop_session.snap new file mode 100644 index 0000000..0536e61 --- /dev/null +++ b/memfaultd/src/metrics/snapshots/memfaultd__metrics__session_event_handler__tests__start_without_stop_session.snap @@ -0,0 +1,9 @@ +--- +source: memfaultd/src/metrics/session_event_handler.rs +expression: metrics +--- +{ + "bar": 2.0, + "foo": 1.0, + "operational_crashes": 0.0 +} diff --git a/memfaultd/src/metrics/statsd_server/mod.rs b/memfaultd/src/metrics/statsd_server/mod.rs new file mode 100644 index 0000000..198cb36 --- /dev/null +++ b/memfaultd/src/metrics/statsd_server/mod.rs @@ -0,0 +1,126 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::net::SocketAddr; +use std::net::UdpSocket; +use std::sync::{Arc, Mutex}; +use std::thread::spawn; + +use eyre::Result; +use log::warn; + +use crate::metrics::{KeyedMetricReading, MetricReportManager}; + +pub struct StatsDServer {} + +impl StatsDServer { + pub fn new() -> StatsDServer { + StatsDServer {} + } + + pub fn start( + &self, + listening_address: SocketAddr, + metric_report_manager: Arc>, + ) -> Result<()> { + let socket = UdpSocket::bind(listening_address)?; + spawn(move || { + loop { + // This means that packets with > 1432 bytes are NOT supported + // Clients must enforce a maximum message size of 1432 bytes or less + let mut buf = [0; 1432]; + match socket.recv(&mut buf) { + Ok(amt) => { + let message = String::from_utf8_lossy(&buf[..amt]); + Self::process_statsd_message(&message, &metric_report_manager) + } + Err(e) => warn!("Statsd server socket error: {}", e), + } + } + }); + Ok(()) + } + + fn process_statsd_message( + message: &str, + metric_report_manager: &Arc>, + ) { + // https://github.com/statsd/statsd/blob/master/docs/server.md + // From statsd spec: + // Multiple metrics can be received in a single packet if separated by the \n character. + let metric_readings = message + .trim() + .lines() + .map(KeyedMetricReading::from_statsd_str) + // Drop strings that couldn't be parsed as a KeyedMetricReading + .filter_map(|res| { + if let Err(e) = &res { + warn!("{}", e) + }; + res.ok() + }); + + for metric_reading in metric_readings { + if let Err(e) = metric_report_manager + .lock() + .expect("Mutex poisoned!") + .add_metric(metric_reading) + { + warn!("Error adding metric sent to StatsD server: {}", e); + } + } + } +} + +impl Default for StatsDServer { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod test { + use std::collections::BTreeMap; + + use insta::assert_json_snapshot; + use rstest::rstest; + + use super::*; + + #[rstest] + #[case("test_counter:1|c", "test_gauge:2.0|g", "test_simple")] + #[case("test_counter:1|c", "test_counter:1|c", "test_counter_aggregation")] + #[case( + "test_counter:1|c\ntest_gauge:2.0|g", + "test_counter:1|c\ntest_gauge:10.0|g", + "test_counter_and_gauge_aggregation" + )] + #[case( + "test_histo:100|h\ntest_another_histo:20.0|h", + "test_one_more_histo:35|h\ntest_another_histo:1000.0|h", + "test_histogram_aggregation" + )] + fn test_process_statsd_message( + #[case] statsd_message_a: &str, + #[case] statsd_message_b: &str, + #[case] test_name: &str, + ) { + let metric_report_manager = Arc::new(Mutex::new(MetricReportManager::new())); + + // Process first StatsD test message + StatsDServer::process_statsd_message(statsd_message_a, &metric_report_manager); + + // Process second StatsD test message + StatsDServer::process_statsd_message(statsd_message_b, &metric_report_manager); + + // Verify resulting metric report + let metrics: BTreeMap<_, _> = metric_report_manager + .lock() + .unwrap() + .take_heartbeat_metrics() + .into_iter() + .collect(); + + assert_json_snapshot!(test_name, metrics); + } +} diff --git a/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_counter_aggregation.snap b/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_counter_aggregation.snap new file mode 100644 index 0000000..8ebff42 --- /dev/null +++ b/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_counter_aggregation.snap @@ -0,0 +1,7 @@ +--- +source: memfaultd/src/metrics/statsd_server/mod.rs +expression: metrics +--- +{ + "test_counter": 2.0 +} diff --git a/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_counter_and_gauge_aggregation.snap b/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_counter_and_gauge_aggregation.snap new file mode 100644 index 0000000..194f9d0 --- /dev/null +++ b/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_counter_and_gauge_aggregation.snap @@ -0,0 +1,8 @@ +--- +source: memfaultd/src/metrics/statsd_server/mod.rs +expression: metrics +--- +{ + "test_counter": 2.0, + "test_gauge": 10.0 +} diff --git a/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_histogram_aggregation.snap b/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_histogram_aggregation.snap new file mode 100644 index 0000000..731e6e6 --- /dev/null +++ b/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_histogram_aggregation.snap @@ -0,0 +1,9 @@ +--- +source: memfaultd/src/metrics/statsd_server/mod.rs +expression: metrics +--- +{ + "test_another_histo": 510.0, + "test_histo": 100.0, + "test_one_more_histo": 35.0 +} diff --git a/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_simple.snap b/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_simple.snap new file mode 100644 index 0000000..157abc1 --- /dev/null +++ b/memfaultd/src/metrics/statsd_server/snapshots/memfaultd__metrics__statsd_server__test__test_simple.snap @@ -0,0 +1,8 @@ +--- +source: memfaultd/src/metrics/statsd_server/mod.rs +expression: metrics +--- +{ + "test_counter": 1.0, + "test_gauge": 2.0 +} diff --git a/memfaultd/src/metrics/system_metrics/cpu.rs b/memfaultd/src/metrics/system_metrics/cpu.rs new file mode 100644 index 0000000..be95095 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/cpu.rs @@ -0,0 +1,304 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Collect CPU metric readings from /proc/stat +//! +//! This module parses CPU statistics from /proc/stat and +//! constructs KeyedMetricReadings based on those statistics. +//! Because the /proc/stat values are accumulations since boot, +//! a "previous reading" (stored in CpuMetricCollector) is +//! required to calculate the time each CPU core +//! has spent in each state since the last reading. +//! +//! Example /proc/stat contents: +//! cpu 326218 0 178980 36612114 6054 0 11961 0 0 0 +//! cpu0 77186 0 73689 9126238 1353 0 6352 0 0 0 +//! cpu1 83902 0 35260 9161039 1524 0 1865 0 0 0 +//! cpu2 83599 0 35323 9161010 1676 0 1875 0 0 0 +//! cpu3 81530 0 34707 9163825 1500 0 1867 0 0 0 +//! intr 95400676 0 9795 1436573 0 0 0 0 0 0 0 0 93204555 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 77883 0 530 0 0 1523 0 0 468762 0 0 97412 103573 0 70 0 0 0 0 0 +//! ctxt 9591503 +//! btime 1714309294 +//! processes 9416 +//! procs_running 1 +//! procs_blocked 0 +//! softirq 47765068 15 3173702 0 541726 82192 0 1979 41497887 0 2467567 +//! +//! Only the lines that start with "cpu" are currently +//! processed into metric readings by this module - the rest are discarded. +//! +//! See additional Linux kernel documentation on /proc/stat here: +//! https://docs.kernel.org/filesystems/proc.html#miscellaneous-kernel-statistics-in-proc-stat +use std::collections::HashMap; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::iter::zip; +use std::ops::Add; +use std::path::Path; +use std::str::FromStr; + +use chrono::Utc; +use nom::{ + bytes::complete::tag, + character::complete::{digit0, space1}, + multi::count, + number::complete::double, + sequence::{pair, preceded}, + IResult, +}; + +use crate::metrics::{ + system_metrics::SystemMetricFamilyCollector, KeyedMetricReading, MetricReading, MetricStringKey, +}; +use eyre::{eyre, ErrReport, Result}; + +const PROC_STAT_PATH: &str = "/proc/stat"; +pub const CPU_METRIC_NAMESPACE: &str = "cpu"; + +pub struct CpuMetricCollector { + last_reading_by_cpu: HashMap>, +} + +impl CpuMetricCollector { + pub fn new() -> Self { + Self { + last_reading_by_cpu: HashMap::new(), + } + } + + pub fn get_cpu_metrics(&mut self) -> Result> { + // Track if any lines in /proc/stat are parse-able + // so we can alert user if none are + let mut no_parseable_lines = true; + + let path = Path::new(PROC_STAT_PATH); + + let file = File::open(path)?; + let reader = BufReader::new(file); + + let mut cpu_metric_readings = vec![]; + + for line in reader.lines() { + // Discard errors - the assumption here is that we are only parsing + // lines that follow the specified format and expect other lines in the file to error + if let Ok((cpu_id, cpu_stats)) = Self::parse_proc_stat_line_cpu(line?.trim()) { + no_parseable_lines = false; + if let Ok(Some(mut readings)) = self.delta_since_last_reading(cpu_id, cpu_stats) { + cpu_metric_readings.append(&mut readings); + } + } + } + + // Check if we were able to parse at least one CPU metric reading + if !no_parseable_lines { + Ok(cpu_metric_readings) + } else { + Err(eyre!( + "No CPU metrics were collected from {} - is it a properly formatted /proc/stat file?", + PROC_STAT_PATH + )) + } + } + + /// Parse a cpu ID from a line of /proc/stat + /// + /// A cpu ID is the digit following the 3 character string "cpu" + /// No ID being present implies these stats are for the total + /// of all cores on the CPU + fn parse_cpu_id(input: &str) -> IResult<&str, &str> { + preceded(tag("cpu"), digit0)(input) + } + + /// Parse the CPU stats from the suffix of a /proc/stat line following the cpu ID + /// + /// 7 or more space delimited integers are expected. Values after the 7th are discarded. + fn parse_cpu_stats(input: &str) -> IResult<&str, Vec> { + count(preceded(space1, double), 7)(input) + } + + /// Parse the output of a line of /proc/stat, returning + /// a pair of the cpu ID that the parsed line corresponds + /// to and the first 7 floats listed for it + /// + /// The 7 floats represent how much time since boot the cpu has + /// spent in the "user", "nice", "system", "idle", "iowait", "irq", + /// "softirq", in that order + /// Example of a valid parse-able line: + /// cpu2 36675 176 11216 1552961 689 0 54 + fn parse_proc_stat_line_cpu(line: &str) -> Result<(String, Vec)> { + let (_remaining, (cpu_id, cpu_stats)) = + pair(Self::parse_cpu_id, Self::parse_cpu_stats)(line) + .map_err(|_e| eyre!("Failed to parse CPU stats line: {}", line))?; + Ok(("cpu".to_string().add(cpu_id), cpu_stats)) + } + + /// Calculate the time spent in each state for the + /// provided CPU core since the last reading collected + /// by the CpuMetricCollector + /// + /// Returns an Ok(None) if there is no prior reading + /// to calculate a delta from. + fn delta_since_last_reading( + &mut self, + cpu_id: String, + cpu_stats: Vec, + ) -> Result>> { + // Check to make sure there was a previous reading to calculate a delta with + if let Some(last_stats) = self + .last_reading_by_cpu + .insert(cpu_id.clone(), cpu_stats.clone()) + { + let delta = cpu_stats + .iter() + .zip(last_stats) + .map(|(current, previous)| current - previous); + + let cpu_states_with_ticks = zip( + ["user", "nice", "system", "idle", "iowait", "irq", "softirq"], + delta, + ) + .collect::>(); + + let sum: f64 = cpu_states_with_ticks.iter().map(|(_k, v)| v).sum(); + let timestamp = Utc::now(); + + let readings = cpu_states_with_ticks + .iter() + .map(|(key, value)| -> Result { + Ok(KeyedMetricReading::new( + MetricStringKey::from_str(&format!( + "{}/{}/percent/{}", + CPU_METRIC_NAMESPACE, cpu_id, key + )) + .map_err(|e| eyre!(e))?, + MetricReading::Histogram { + // Transform raw tick value to a percentage + value: 100.0 * value / sum, + timestamp, + }, + )) + }) + .collect::>>(); + match readings { + Ok(readings) => Ok(Some(readings)), + Err(e) => Err(e), + } + } else { + Ok(None) + } + } +} + +impl SystemMetricFamilyCollector for CpuMetricCollector { + fn family_name(&self) -> &'static str { + CPU_METRIC_NAMESPACE + } + + fn collect_metrics(&mut self) -> Result> { + self.get_cpu_metrics() + } +} + +#[cfg(test)] +mod test { + + use insta::{assert_json_snapshot, rounded_redaction, with_settings}; + use rstest::rstest; + + use super::*; + + #[rstest] + #[case("cpu 1000 5 0 0 2 0 0", "test_basic_line")] + #[case("cpu0 1000 5 0 0 2 0 0 0 0 0", "test_basic_line_with_extra")] + fn test_process_valid_proc_stat_line(#[case] proc_stat_line: &str, #[case] test_name: &str) { + assert_json_snapshot!(test_name, + CpuMetricCollector::parse_proc_stat_line_cpu(proc_stat_line).unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + } + + #[rstest] + #[case("cpu 1000 5 0 0 2")] + #[case("1000 5 0 0 2 0 0 0 0 0")] + #[case("processor0 1000 5 0 0 2 0 0 0 0 0")] + #[case("softirq 403453672 10204651 21667771 199 12328940 529390 0 3519783 161759969 147995 193294974")] + fn test_fails_on_invalid_proc_stat_line(#[case] proc_stat_line: &str) { + assert!(CpuMetricCollector::parse_proc_stat_line_cpu(proc_stat_line).is_err()) + } + + #[rstest] + #[case( + "cpu 1000 5 0 0 2 0 0", + "cpu 1500 20 4 1 2 0 0", + "cpu 1550 200 40 3 3 0 0", + "basic_delta" + )] + fn test_cpu_metric_collector_calcs( + #[case] proc_stat_line_a: &str, + #[case] proc_stat_line_b: &str, + #[case] proc_stat_line_c: &str, + #[case] test_name: &str, + ) { + let mut cpu_metric_collector = CpuMetricCollector::new(); + + let (cpu, stats) = CpuMetricCollector::parse_proc_stat_line_cpu(proc_stat_line_a).unwrap(); + let result_a = cpu_metric_collector.delta_since_last_reading(cpu, stats); + matches!(result_a, Ok(None)); + + let (cpu, stats) = CpuMetricCollector::parse_proc_stat_line_cpu(proc_stat_line_b).unwrap(); + let result_b = cpu_metric_collector.delta_since_last_reading(cpu, stats); + + assert!(result_b.is_ok()); + + with_settings!({sort_maps => true}, { + assert_json_snapshot!(format!("{}_{}", test_name, "a_b_metrics"), + result_b.unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + }); + + let (cpu, stats) = CpuMetricCollector::parse_proc_stat_line_cpu(proc_stat_line_c).unwrap(); + let result_c = cpu_metric_collector.delta_since_last_reading(cpu, stats); + + assert!(result_c.is_ok()); + + with_settings!({sort_maps => true}, { + assert_json_snapshot!(format!("{}_{}", test_name, "b_c_metrics"), + result_c.unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + }); + } + + #[rstest] + #[case( + "cpu1 40 20 30 10 0 0 0", + "cpu0 1500 20 4 1 2 0 0", + "cpu1 110 30 40 12 5 3 0", + "different_cores" + )] + fn test_cpu_metric_collector_different_cores( + #[case] proc_stat_line_a: &str, + #[case] proc_stat_line_b: &str, + #[case] proc_stat_line_c: &str, + #[case] test_name: &str, + ) { + let mut cpu_metric_collector = CpuMetricCollector::new(); + + let (cpu, stats) = CpuMetricCollector::parse_proc_stat_line_cpu(proc_stat_line_a).unwrap(); + let result_a = cpu_metric_collector.delta_since_last_reading(cpu, stats); + matches!(result_a, Ok(None)); + + let (cpu, stats) = CpuMetricCollector::parse_proc_stat_line_cpu(proc_stat_line_b).unwrap(); + let result_b = cpu_metric_collector.delta_since_last_reading(cpu, stats); + matches!(result_b, Ok(None)); + + let (cpu, stats) = CpuMetricCollector::parse_proc_stat_line_cpu(proc_stat_line_c).unwrap(); + let result_c = cpu_metric_collector.delta_since_last_reading(cpu, stats); + + assert!(result_c.is_ok()); + + with_settings!({sort_maps => true}, { + assert_json_snapshot!(format!("{}_{}", test_name, "a_c_metrics"), + result_c.unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + }); + } +} diff --git a/memfaultd/src/metrics/system_metrics/disk_space.rs b/memfaultd/src/metrics/system_metrics/disk_space.rs new file mode 100644 index 0000000..0110c50 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/disk_space.rs @@ -0,0 +1,341 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Collect disk space metric readings for devices listed in +//! /proc/mounts +//! +//! This module parses mounted devices and their mount points +//! from /proc/mounts and calculates how many bytes are free +//! and used on the device. +//! +use std::{ + collections::HashSet, + fs::File, + io::{BufRead, BufReader}, + path::{Path, PathBuf}, +}; + +use eyre::{eyre, Result}; +use log::warn; +use nix::sys::statvfs::statvfs; +use nom::{ + bytes::complete::take_while, + character::complete::multispace1, + sequence::{pair, preceded}, + IResult, +}; + +use serde::Serialize; + +use crate::metrics::{system_metrics::SystemMetricFamilyCollector, KeyedMetricReading}; + +pub const DISKSPACE_METRIC_NAMESPACE_LEGACY: &str = "df"; +pub const DISKSPACE_METRIC_NAMESPACE: &str = "disk_space"; +pub const PROC_MOUNTS_PATH: &str = "/proc/mounts"; + +pub struct DiskSpaceInfo { + block_size: u64, + blocks: u64, + blocks_free: u64, +} + +#[cfg_attr(test, mockall::automock)] +pub trait DiskSpaceInfoForPath { + fn disk_space_info_for_path(&self, p: &Path) -> Result; +} + +pub struct NixStatvfs {} + +impl NixStatvfs { + pub fn new() -> Self { + Self {} + } +} + +impl DiskSpaceInfoForPath for NixStatvfs { + fn disk_space_info_for_path(&self, p: &Path) -> Result { + let statfs = statvfs(p) + .map_err(|e| eyre!("Failed to get statfs info for {}: {}", p.display(), e))?; + + Ok(DiskSpaceInfo { + // Ignore unnecessary cast for these + // as it is needed on 32-bit systems. + #[allow(clippy::unnecessary_cast)] + block_size: statfs.block_size() as u64, + #[allow(clippy::unnecessary_cast)] + blocks: statfs.blocks() as u64, + #[allow(clippy::unnecessary_cast)] + blocks_free: statfs.blocks_free() as u64, + }) + } +} + +#[derive(Serialize)] +struct Mount { + device: PathBuf, + mount_point: PathBuf, +} + +pub enum DiskSpaceMetricsConfig { + Auto, + Disks(HashSet), +} + +pub struct DiskSpaceMetricCollector +where + T: DiskSpaceInfoForPath, +{ + config: DiskSpaceMetricsConfig, + mounts: Vec, + disk_space_impl: T, +} + +impl DiskSpaceMetricCollector +where + T: DiskSpaceInfoForPath, +{ + pub fn new(disk_space_impl: T, config: DiskSpaceMetricsConfig) -> Self { + Self { + config, + mounts: Vec::new(), + disk_space_impl, + } + } + fn disk_is_monitored(&self, disk: &str) -> bool { + match &self.config { + DiskSpaceMetricsConfig::Auto => disk.starts_with("/dev"), + DiskSpaceMetricsConfig::Disks(configured_disks) => configured_disks.contains(disk), + } + } + + /// Parses a line of /proc/mounts for the name of + /// the device the line corresponds to + /// + /// Example input: + /// "/dev/sda2 / ext4 rw,noatime 0 0" + /// Example output: + /// "/dev/sda2" + fn parse_proc_mounts_device(proc_mounts_line: &str) -> IResult<&str, &str> { + take_while(|c: char| !c.is_whitespace())(proc_mounts_line) + } + + /// Parses a line of /proc/mounts for the + /// mount point the line corresponds to + /// Parses /proc/mounts for a list of devices with active + /// mount points in the system + /// Example input: + /// " / ext4 rw,noatime 0 0" + /// Example output: + /// "/" + fn parse_proc_mounts_mount_point(proc_mounts_line: &str) -> IResult<&str, &str> { + preceded(multispace1, take_while(|c: char| !c.is_whitespace()))(proc_mounts_line) + } + + /// Parse a line of /proc/mounts + /// Example input: + /// "/dev/sda2 / ext4 rw,noatime 0 0" + /// Example output: + /// Mount { device: "/dev/sda2", "mount_point": "/" } + fn parse_proc_mounts_line(line: &str) -> Result { + let (_remaining, (device, mount_point)) = pair( + Self::parse_proc_mounts_device, + Self::parse_proc_mounts_mount_point, + )(line) + .map_err(|e| eyre!("Failed to parse /proc/mounts line: {}", e))?; + Ok(Mount { + device: Path::new(device).to_path_buf(), + mount_point: Path::new(mount_point).to_path_buf(), + }) + } + + /// Initialize the list of mounted devices and their mount points based + /// on the contents of /proc/mounts + pub fn initialize_mounts(&mut self, proc_mounts_path: &Path) -> Result<()> { + let file = File::open(proc_mounts_path)?; + let reader = BufReader::new(file); + + for line in reader.lines() { + // Discard errors - the assumption here is that we are only parsing + // lines that follow the specified format and expect other lines in the file to error + if let Ok(mount) = Self::parse_proc_mounts_line(line?.trim()) { + if self.disk_is_monitored(&mount.device.to_string_lossy()) { + self.mounts.push(mount); + } + } + } + Ok(()) + } + + /// For a given mounted device, construct metric readings + /// for how many bytes are used and free on the device + fn get_metrics_for_mount(&self, mount: &Mount) -> Result> { + let mount_stats = self + .disk_space_impl + .disk_space_info_for_path(mount.mount_point.as_path())?; + + let block_size = mount_stats.block_size; + let bytes_free = mount_stats.blocks_free * block_size; + let bytes_used = (mount_stats.blocks * block_size) - bytes_free; + + let disk_id = mount + .device + .file_name() + .ok_or_else(|| eyre!("Couldn't extract basename"))? + .to_string_lossy(); + + let bytes_free_reading = KeyedMetricReading::new_histogram( + format!("disk_space/{}/free_bytes", disk_id) + .as_str() + .parse() + .map_err(|e| eyre!("Couldn't parse metric key for bytes free: {}", e))?, + bytes_free as f64, + ); + + let bytes_used_reading = KeyedMetricReading::new_histogram( + format!("disk_space/{}/used_bytes", disk_id) + .as_str() + .parse() + .map_err(|e| eyre!("Couldn't parse metric key for bytes used: {}", e))?, + bytes_used as f64, + ); + + Ok(vec![bytes_free_reading, bytes_used_reading]) + } + + pub fn get_disk_space_metrics(&mut self) -> Result> { + if self.mounts.is_empty() { + self.initialize_mounts(Path::new(PROC_MOUNTS_PATH))?; + } + + let mut disk_space_readings = Vec::new(); + for mount in self.mounts.iter() { + match self.get_metrics_for_mount(mount) { + Ok(readings) => disk_space_readings.extend(readings), + Err(e) => warn!( + "Failed to calculate disk space readings for {} mounted at {}: {}", + mount.device.display(), + mount.mount_point.display(), + e + ), + } + } + + Ok(disk_space_readings) + } +} + +impl SystemMetricFamilyCollector for DiskSpaceMetricCollector +where + T: DiskSpaceInfoForPath, +{ + fn family_name(&self) -> &'static str { + DISKSPACE_METRIC_NAMESPACE + } + + fn collect_metrics(&mut self) -> Result> { + self.get_disk_space_metrics() + } +} + +#[cfg(test)] +mod test { + use std::fs::File; + use std::io::Write; + + use insta::{assert_json_snapshot, rounded_redaction}; + use rstest::rstest; + use tempfile::tempdir; + + use super::*; + + #[rstest] + fn test_process_valid_proc_mounts_line() { + let line = "/dev/sda2 /media ext4 rw,noatime 0 0"; + let mount = + DiskSpaceMetricCollector::::parse_proc_mounts_line(line) + .unwrap(); + + assert_eq!(mount.device.as_os_str().to_string_lossy(), "/dev/sda2"); + assert_eq!(mount.mount_point.as_os_str().to_string_lossy(), "/media"); + } + + #[rstest] + fn test_initialize_and_calc_disk_space_for_mounts() { + let mut mock_statfs = MockDiskSpaceInfoForPath::new(); + + mock_statfs + .expect_disk_space_info_for_path() + .times(2) + .returning(|_p| { + Ok(DiskSpaceInfo { + block_size: 4096, + blocks: 1024, + blocks_free: 286, + }) + }); + + let mut disk_space_collector = + DiskSpaceMetricCollector::new(mock_statfs, DiskSpaceMetricsConfig::Auto); + + let line = "/dev/sda2 /media ext4 rw,noatime 0 0"; + let line2 = "/dev/sda1 / ext4 rw,noatime 0 0"; + + let dir = tempdir().unwrap(); + + let mounts_file_path = dir.path().join("mounts"); + let mut mounts_file = File::create(mounts_file_path.clone()).unwrap(); + + writeln!(mounts_file, "{}", line).unwrap(); + writeln!(mounts_file, "{}", line2).unwrap(); + + assert!(disk_space_collector + .initialize_mounts(&mounts_file_path) + .is_ok()); + + disk_space_collector + .mounts + .sort_by(|a, b| a.device.cmp(&b.device)); + + assert_json_snapshot!(disk_space_collector.mounts); + + let metrics = disk_space_collector.collect_metrics().unwrap(); + + assert_json_snapshot!(metrics, + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)} + ); + + dir.close().unwrap(); + } + #[rstest] + fn test_unmonitored_disks_not_initialized() { + let mock_statfs = MockDiskSpaceInfoForPath::new(); + + let mut disk_space_collector = DiskSpaceMetricCollector::new( + mock_statfs, + DiskSpaceMetricsConfig::Disks(HashSet::from_iter(["/dev/sdc1".to_string()])), + ); + + let line = "/dev/sda2 /media ext4 rw,noatime 0 0"; + let line2 = "/dev/sda1 / ext4 rw,noatime 0 0"; + + let dir = tempdir().unwrap(); + + let mounts_file_path = dir.path().join("mounts"); + let mut mounts_file = File::create(mounts_file_path.clone()).unwrap(); + + writeln!(mounts_file, "{}", line).unwrap(); + writeln!(mounts_file, "{}", line2).unwrap(); + + assert!(disk_space_collector + .initialize_mounts(&mounts_file_path) + .is_ok()); + + disk_space_collector + .mounts + .sort_by(|a, b| a.device.cmp(&b.device)); + + assert!(disk_space_collector.mounts.is_empty()); + + dir.close().unwrap(); + } +} diff --git a/memfaultd/src/metrics/system_metrics/memory.rs b/memfaultd/src/metrics/system_metrics/memory.rs new file mode 100644 index 0000000..8642a1f --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/memory.rs @@ -0,0 +1,291 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Collect memory metric readings from /proc/meminfo +//! +//! This module parses memory statistics from /proc/meminfo and +//! constructs KeyedMetricReadings based on those statistics. +//! +//! Example /proc/meminfo contents: +//! MemTotal: 365916 kB +//! MemFree: 242276 kB +//! MemAvailable: 292088 kB +//! Buffers: 4544 kB +//! Cached: 52128 kB +//! SwapCached: 0 kB +//! Active: 21668 kB +//! Inactive: 51404 kB +//! Active(anon): 2312 kB +//! Inactive(anon): 25364 kB +//! Active(file): 19356 kB +//! Inactive(file): 26040 kB +//! Unevictable: 3072 kB +//! Mlocked: 0 kB +//! SwapTotal: 0 kB +//! SwapFree: 0 kB +//! Dirty: 0 kB +//! Writeback: 0 kB +//! AnonPages: 19488 kB +//! Mapped: 29668 kB +//! Shmem: 11264 kB +//! KReclaimable: 14028 kB +//! Slab: 32636 kB +//! SReclaimable: 14028 kB +//! +//! Only the following lines are currently processed: +//! MemTotal, MemFree, and optionally MemAvailable +//! +//! These lines are used by this module to calculate +//! free and used memory. MemFree is used in place of +//! MemAvailable if the latter is not present. +//! +//! +//! See additional Linux kernel documentation on /proc/meminfo here: +//! https://www.kernel.org/doc/html/latest/filesystems/proc.html#meminfo +use std::fs::read_to_string; +use std::path::Path; +use std::{collections::HashMap, str::FromStr}; + +use eyre::{eyre, Result}; +use nom::{ + bytes::complete::tag, + character::complete::{alpha1, multispace1}, + number::complete::double, + sequence::{delimited, terminated}, + IResult, +}; + +use crate::metrics::{ + system_metrics::SystemMetricFamilyCollector, KeyedMetricReading, MetricStringKey, +}; + +const PROC_MEMINFO_PATH: &str = "/proc/meminfo"; +pub const MEMORY_METRIC_NAMESPACE: &str = "memory"; + +pub struct MemoryMetricsCollector; + +impl MemoryMetricsCollector { + pub fn new() -> Self { + MemoryMetricsCollector {} + } + + pub fn get_memory_metrics(&self) -> Result> { + let path = Path::new(PROC_MEMINFO_PATH); + // Need to read all of /proc/meminfo at once + // as we derive used memory based on a calculation + // using multiple lines + let contents = read_to_string(path)?; + Self::parse_meminfo_memory_stats(&contents) + } + + /// Parses the key in a /proc/meminfo line + /// + /// A key is the string terminated by the `:` character + /// In the following line, "MemTotal" would be parsed as the key + /// MemTotal: 365916 kB + fn parse_meminfo_key(meminfo_line: &str) -> IResult<&str, &str> { + terminated(alpha1, tag(":"))(meminfo_line) + } + + /// Parses the kilobyte value in a /proc/meminfo line + /// + /// This value is the kilobytes used by the corresponding + /// meminfo key. The value is a number terminated by " kB". + /// In the following line, 365916.0 would be parsed by + /// this function as the kB used by "MemTotal" + /// MemTotal: 365916 kB + fn parse_meminfo_kb(meminfo_line_suffix: &str) -> IResult<&str, f64> { + delimited(multispace1, double, tag(" kB"))(meminfo_line_suffix) + } + + /// Parses a full /proc/meminfo contents and returns + /// a vector of KeyedMetricReadings + fn parse_meminfo_memory_stats(meminfo: &str) -> Result> { + let mut stats = meminfo + .trim() + .lines() + .map(|line| -> Result<(&str, f64), String> { + let (suffix, key) = Self::parse_meminfo_key(line).map_err(|e| e.to_string())?; + let (_, kb) = Self::parse_meminfo_kb(suffix).map_err(|e| e.to_string())?; + // Use bytes as unit instead of KB + Ok((key, kb * 1024.0)) + }) + .filter_map(|result| result.ok()) + .collect::>(); + + // Use the same methodology as `free` to calculate used memory. + // + // For kernels 3.14 and greater: + // MemUsed = MemTotal - MemAvailable + // + // For kernels less than 3.14 (no MemAvailable): + // MemUsed = MemTotal - MemFree + // + // See below man page for more info: + // https://man7.org/linux/man-pages/man1/free.1.html + let total = stats + .remove("MemTotal") + .ok_or_else(|| eyre!("{} is missing required value MemTotal", PROC_MEMINFO_PATH))?; + let free = stats + .remove("MemFree") + .ok_or_else(|| eyre!("{} is missing required value MemFree", PROC_MEMINFO_PATH))?; + let available = stats.remove("MemAvailable").unwrap_or(free); + + let used = total - available; + + let used_key = MetricStringKey::from_str("memory/memory/used") + .map_err(|e| eyre!("Failed to construct MetricStringKey for used memory: {}", e))?; + let free_key = MetricStringKey::from_str("memory/memory/free") + .map_err(|e| eyre!("Failed to construct MetricStringKey for used memory: {}", e))?; + + Ok(vec![ + KeyedMetricReading::new_histogram(free_key, free), + KeyedMetricReading::new_histogram(used_key, used), + ]) + } +} + +impl SystemMetricFamilyCollector for MemoryMetricsCollector { + fn collect_metrics(&mut self) -> Result> { + self.get_memory_metrics() + } + + fn family_name(&self) -> &'static str { + MEMORY_METRIC_NAMESPACE + } +} + +#[cfg(test)] +mod test { + + use insta::{assert_json_snapshot, rounded_redaction, with_settings}; + use rstest::rstest; + + use super::MemoryMetricsCollector; + + #[rstest] + #[case("MemTotal: 365916 kB", "MemTotal", 365916.0)] + #[case("MemFree: 242276 kB", "MemFree", 242276.0)] + #[case("MemAvailable: 292088 kB", "MemAvailable", 292088.0)] + #[case("Buffers: 4544 kB", "Buffers", 4544.0)] + #[case("Cached: 52128 kB", "Cached", 52128.0)] + fn test_parse_meminfo_line( + #[case] proc_meminfo_line: &str, + #[case] expected_key: &str, + #[case] expected_value: f64, + ) { + let (suffix, key) = MemoryMetricsCollector::parse_meminfo_key(proc_meminfo_line).unwrap(); + let (_, kb) = MemoryMetricsCollector::parse_meminfo_kb(suffix).unwrap(); + + assert_eq!(key, expected_key); + assert_eq!(kb, expected_value); + } + + #[rstest] + fn test_get_memory_metrics() { + let meminfo = "MemTotal: 365916 kB +MemFree: 242276 kB +MemAvailable: 292088 kB +Buffers: 4544 kB +Cached: 52128 kB +SwapCached: 0 kB +Active: 21668 kB +Inactive: 51404 kB +Active(anon): 2312 kB +Inactive(anon): 25364 kB +Active(file): 19356 kB +Inactive(file): 26040 kB +Unevictable: 3072 kB +Mlocked: 0 kB +SwapTotal: 0 kB +SwapFree: 0 kB +Dirty: 0 kB +Writeback: 0 kB +AnonPages: 19488 kB +Mapped: 29668 kB +Shmem: 11264 kB +KReclaimable: 14028 kB + "; + + with_settings!({sort_maps => true}, { + assert_json_snapshot!( + MemoryMetricsCollector::parse_meminfo_memory_stats(meminfo).unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + }); + } + + #[rstest] + fn test_get_memory_metrics_no_memavailable() { + let meminfo = "MemTotal: 365916 kB +MemFree: 242276 kB +Buffers: 4544 kB +Cached: 52128 kB +SwapCached: 0 kB +Active: 21668 kB +Inactive: 51404 kB +Active(anon): 2312 kB +Inactive(anon): 25364 kB +Active(file): 19356 kB +Inactive(file): 26040 kB +Unevictable: 3072 kB +Mlocked: 0 kB +SwapTotal: 0 kB +SwapFree: 0 kB +Dirty: 0 kB +Writeback: 0 kB +AnonPages: 19488 kB +Mapped: 29668 kB +Shmem: 11264 kB +KReclaimable: 14028 kB + "; + + with_settings!({sort_maps => true}, { + assert_json_snapshot!( + MemoryMetricsCollector::parse_meminfo_memory_stats(meminfo).unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + }); + } + + #[rstest] + fn test_fail_to_parse_bad_meminfo_line() { + assert!(MemoryMetricsCollector::parse_meminfo_key("MemFree=1080kB").is_err()); + assert!(MemoryMetricsCollector::parse_meminfo_kb("1080 mB").is_err()); + } + + #[rstest] + fn test_fail_get_metrics_with_missing_required_lines() { + // MemFree is missing + let meminfo = "MemTotal: 365916 kB +MemAvailable: 292088 kB +Buffers: 4544 kB +Cached: 52128 kB +SwapCached: 0 kB +Active: 21668 kB +Inactive: 51404 kB +Active(anon): 2312 kB +Inactive(anon): 25364 kB +Active(file): 19356 kB +Inactive(file): 26040 kB +Unevictable: 3072 kB +Mlocked: 0 kB +SwapTotal: 0 kB +SwapFree: 0 kB +Dirty: 0 kB +Writeback: 0 kB +AnonPages: 19488 kB +Mapped: 29668 kB +Shmem: 11264 kB +KReclaimable: 14028 kB + "; + assert!(MemoryMetricsCollector::parse_meminfo_memory_stats(meminfo).is_err()); + } + + #[rstest] + fn test_fail_get_metrics_with_bad_fmt() { + // Not properly formatted with newlines between each key / kB pair + let meminfo = "MemTotal: 365916 kB MemFree: 242276 kB +Buffers: 4544 kB Cached: 52128 kB Shmem: 11264 kB + "; + assert!(MemoryMetricsCollector::parse_meminfo_memory_stats(meminfo).is_err()); + } +} diff --git a/memfaultd/src/metrics/system_metrics/mod.rs b/memfaultd/src/metrics/system_metrics/mod.rs new file mode 100644 index 0000000..c616928 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/mod.rs @@ -0,0 +1,180 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + collections::HashSet, + sync::{Arc, Mutex}, + thread::sleep, + time::{Duration, Instant}, +}; + +use eyre::Result; +use log::warn; + +use crate::{ + config::SystemMetricConfig, + metrics::{KeyedMetricReading, MetricReportManager}, + util::system::{bytes_per_page, clock_ticks_per_second}, +}; + +mod cpu; +use crate::metrics::system_metrics::cpu::{CpuMetricCollector, CPU_METRIC_NAMESPACE}; + +mod thermal; +use crate::metrics::system_metrics::thermal::{ThermalMetricsCollector, THERMAL_METRIC_NAMESPACE}; + +mod memory; +use crate::metrics::system_metrics::memory::{MemoryMetricsCollector, MEMORY_METRIC_NAMESPACE}; + +mod network_interfaces; +use crate::metrics::system_metrics::network_interfaces::{ + NetworkInterfaceMetricCollector, NetworkInterfaceMetricsConfig, + NETWORK_INTERFACE_METRIC_NAMESPACE, +}; + +mod processes; +use processes::{ProcessMetricsCollector, PROCESSES_METRIC_NAMESPACE}; + +mod disk_space; +use disk_space::{ + DiskSpaceMetricCollector, DiskSpaceMetricsConfig, NixStatvfs, DISKSPACE_METRIC_NAMESPACE, + DISKSPACE_METRIC_NAMESPACE_LEGACY, +}; + +use self::processes::ProcessMetricsConfig; + +pub const BUILTIN_SYSTEM_METRIC_NAMESPACES: &[&str; 7] = &[ + CPU_METRIC_NAMESPACE, + MEMORY_METRIC_NAMESPACE, + THERMAL_METRIC_NAMESPACE, + NETWORK_INTERFACE_METRIC_NAMESPACE, + PROCESSES_METRIC_NAMESPACE, + DISKSPACE_METRIC_NAMESPACE, + // Include in list of namespaces so that + // legacy collectd from the "df" plugin + // are still filtered out + DISKSPACE_METRIC_NAMESPACE_LEGACY, +]; + +pub trait SystemMetricFamilyCollector { + fn collect_metrics(&mut self) -> Result>; + fn family_name(&self) -> &'static str; +} + +pub struct SystemMetricsCollector { + metric_family_collectors: Vec>, +} + +impl SystemMetricsCollector { + pub fn new(system_metric_config: SystemMetricConfig) -> Self { + // CPU, Memory, and Thermal metrics are captured by default + let mut metric_family_collectors: Vec> = vec![ + Box::new(CpuMetricCollector::new()), + Box::new(MemoryMetricsCollector::new()), + Box::new(ThermalMetricsCollector::new()), + ]; + + // Check if process metrics have been manually configured + match system_metric_config.processes { + Some(processes) if !processes.is_empty() => { + metric_family_collectors.push(Box::new(ProcessMetricsCollector::::new( + ProcessMetricsConfig::Processes(processes), + clock_ticks_per_second() as f64 / 1000.0, + bytes_per_page() as f64, + ))) + } + // Monitoring no processes means this collector is disabled + Some(_empty_set) => {} + None => { + metric_family_collectors.push(Box::new(ProcessMetricsCollector::::new( + ProcessMetricsConfig::Auto, + clock_ticks_per_second() as f64 / 1000.0, + bytes_per_page() as f64, + ))) + } + }; + + // Check if disk space metrics have been manually configured + match system_metric_config.disk_space { + Some(disks) if !disks.is_empty() => { + metric_family_collectors.push(Box::new(DiskSpaceMetricCollector::new( + NixStatvfs::new(), + DiskSpaceMetricsConfig::Disks(disks), + ))) + } + // Monitoring no disks means this collector is disabled + Some(_empty_set) => {} + None => metric_family_collectors.push(Box::new(DiskSpaceMetricCollector::new( + NixStatvfs::new(), + DiskSpaceMetricsConfig::Auto, + ))), + }; + + // Check if network interface metrics have been manually configured + match system_metric_config.network_interfaces { + Some(interfaces) if !interfaces.is_empty() => metric_family_collectors.push(Box::new( + NetworkInterfaceMetricCollector::::new( + NetworkInterfaceMetricsConfig::Interfaces(interfaces), + ), + )), + // Monitoring no interfaces means this collector is disabled + Some(_empty_set) => {} + None => metric_family_collectors.push(Box::new(NetworkInterfaceMetricCollector::< + Instant, + >::new( + NetworkInterfaceMetricsConfig::Auto + ))), + }; + + Self { + metric_family_collectors, + } + } + + pub fn run( + &mut self, + metric_poll_duration: Duration, + metric_report_manager: Arc>, + ) { + loop { + for collector in self.metric_family_collectors.iter_mut() { + match collector.collect_metrics() { + Ok(readings) => { + for metric_reading in readings { + if let Err(e) = metric_report_manager + .lock() + .expect("Mutex poisoned") + .add_metric(metric_reading) + { + warn!( + "Couldn't add metric reading for family \"{}\": {:?}", + collector.family_name(), + e + ) + } + } + } + Err(e) => warn!( + "Failed to collect readings for family \"{}\": {}", + collector.family_name(), + e + ), + } + } + + sleep(metric_poll_duration); + } + } +} + +impl Default for SystemMetricsCollector { + fn default() -> Self { + Self::new(SystemMetricConfig { + enable: true, + poll_interval_seconds: Duration::from_secs(10), + processes: Some(HashSet::from_iter(["memfaultd".to_string()])), + disk_space: None, + network_interfaces: None, + }) + } +} diff --git a/memfaultd/src/metrics/system_metrics/network_interfaces.rs b/memfaultd/src/metrics/system_metrics/network_interfaces.rs new file mode 100644 index 0000000..77ace6b --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/network_interfaces.rs @@ -0,0 +1,489 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Collect Network Interface metric readings from /proc/net/dev +//! +//! Example /proc/net/dev output: +//! Inter-| Receive | Transmit +//! face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed +//! lo: 2707 25 0 0 0 0 0 0 2707 25 0 0 0 0 0 0 +//! eth0: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +//! wlan0: 10919408 8592 0 0 0 0 0 0 543095 4066 0 0 0 0 0 0 +//! +//! Kernel docs: +//! https://docs.kernel.org/filesystems/proc.html#networking-info-in-proc-net +use std::collections::{HashMap, HashSet}; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::iter::zip; +use std::path::Path; +use std::str::FromStr; +use std::time::Duration; + +use chrono::Utc; +use nom::bytes::complete::tag; +use nom::character::complete::{alphanumeric1, multispace0, multispace1, u64}; +use nom::sequence::terminated; +use nom::{ + multi::count, + sequence::{pair, preceded}, + IResult, +}; + +use crate::{ + metrics::{ + system_metrics::SystemMetricFamilyCollector, KeyedMetricReading, MetricReading, + MetricStringKey, + }, + util::time_measure::TimeMeasure, +}; + +use eyre::{eyre, ErrReport, Result}; + +const PROC_NET_DEV_PATH: &str = "/proc/net/dev"; +pub const NETWORK_INTERFACE_METRIC_NAMESPACE: &str = "interface"; + +// Metric keys that are currently captured and reported +// by memfaultd. +// There is a lot of information in /proc/net/dev +// and the intention with this list is to use it +// to filter out the values read from it so that +// only the high-signal and widely-applicable metrics remain. +const NETWORK_INTERFACE_METRIC_KEYS: &[&str; 8] = &[ + "bytes_per_second/rx", + "packets_per_second/rx", + "errors_per_second/rx", + "dropped_per_second/rx", + "bytes_per_second/tx", + "packets_per_second/tx", + "errors_per_second/tx", + "dropped_per_second/tx", +]; + +pub enum NetworkInterfaceMetricsConfig { + Auto, + Interfaces(HashSet), +} + +pub struct NetworkInterfaceMetricCollector { + config: NetworkInterfaceMetricsConfig, + previous_readings_by_interface: HashMap>, +} + +#[derive(Clone)] +pub struct ProcNetDevReading { + stats: Vec, + reading_time: T, +} + +impl NetworkInterfaceMetricCollector +where + T: TimeMeasure + Copy + Ord + std::ops::Add + Send + Sync + 'static, +{ + pub fn new(config: NetworkInterfaceMetricsConfig) -> Self { + Self { + config, + previous_readings_by_interface: HashMap::new(), + } + } + + fn interface_is_monitored(&self, interface: &str) -> bool { + match &self.config { + // Ignore loopback interfaces + NetworkInterfaceMetricsConfig::Auto => !interface.starts_with("lo"), + NetworkInterfaceMetricsConfig::Interfaces(configured_interfaces) => { + configured_interfaces.contains(interface) + } + } + } + + pub fn get_network_interface_metrics(&mut self) -> Result> { + // Track if any lines in /proc/net/dev are parse-able + // so we can alert user if none are + let mut no_parseable_lines = true; + + let path = Path::new(PROC_NET_DEV_PATH); + + let file = File::open(path)?; + let reader = BufReader::new(file); + + let mut net_metric_readings = vec![]; + + for line in reader.lines() { + // Discard errors - the assumption here is that we are only parsing + // lines that follow the specified format and expect other lines in the file to error + if let Ok((interface_id, net_stats)) = Self::parse_proc_net_dev_line(line?.trim()) { + no_parseable_lines = false; + + // Ignore unmonitored interfaces + if self.interface_is_monitored(&interface_id) { + if let Ok(Some(mut readings)) = self.calculate_network_metrics( + interface_id.to_string(), + ProcNetDevReading { + stats: net_stats, + reading_time: T::now(), + }, + ) { + net_metric_readings.append(&mut readings); + } + } + } + } + + // Check if we were able to parse at least one CPU metric reading + if no_parseable_lines { + Err(eyre!( + "No network metrics were collected from {} - is it a properly formatted /proc/net/dev file?", + PROC_NET_DEV_PATH + )) + } else { + Ok(net_metric_readings) + } + } + + /// Parse a network interface name from a line of /proc/net/dev + /// The network interface may be preceded by whitespace and will + /// always be terminated with a ':' + /// in a line that is followed by 16 number values + fn parse_net_if(input: &str) -> IResult<&str, &str> { + terminated(preceded(multispace0, alphanumeric1), tag(":"))(input) + } + + /// Parse the CPU stats from the suffix of a /proc/net/dev line following the interface ID + /// + /// The first 8 values track RX traffic on the interface. The latter 8 track TX traffic. + fn parse_interface_stats(input: &str) -> IResult<&str, Vec> { + count(preceded(multispace1, u64), 16)(input) + } + + /// Parse the output of a line of /proc/net/dev, returning + /// a pair of the network interface that the parsed line corresponds + /// to and the first 7 floats listed for it + /// + /// The first 8 values track RX traffic on the interface since boot with + /// following names (in order): + /// "bytes", "packets", "errs", "drop", "fifo", "frame", "compressed" "multicast" + /// The latter 8 track TX traffic, with the following names: + /// "bytes", "packets", "errs", "drop", "fifo", "colls", "carrier", "compressed" + /// + /// Important!!: The rest of this module assumes this is the ordering of values + /// in the /proc/net/dev file + fn parse_proc_net_dev_line(line: &str) -> Result<(String, Vec)> { + let (_remaining, (interface_id, net_stats)) = + pair(Self::parse_net_if, Self::parse_interface_stats)(line) + .map_err(|e| eyre!("Failed to parse /proc/net/dev line: {}", e))?; + Ok((interface_id.to_string(), net_stats)) + } + + /// We need to account for potential rollovers in the + /// /proc/net/dev counters, handled by this function + fn counter_delta_with_overflow(current: u64, previous: u64) -> u64 { + // The only time a counter's value would be less + // that its previous value is if it rolled over + // due to overflow - drop these readings that overlap + // with an overflow + if current < previous { + // Need to detect if the counter rolled over at u32::MAX or u64::MAX + current + + ((if previous > u32::MAX as u64 { + u64::MAX + } else { + u32::MAX as u64 + }) - previous) + } else { + current - previous + } + } + + /// Calculates network metrics + fn calculate_network_metrics( + &mut self, + interface: String, + current_reading: ProcNetDevReading, + ) -> Result>> { + // Check to make sure there was a previous reading to calculate a delta with + if let Some(ProcNetDevReading { + stats: previous_net_stats, + reading_time: previous_reading_time, + }) = self + .previous_readings_by_interface + .insert(interface.clone(), current_reading.clone()) + { + let current_period_rates = + current_reading + .stats + .iter() + .zip(previous_net_stats) + .map(|(current, previous)| { + Self::counter_delta_with_overflow(*current, previous) as f64 + / (current_reading + .reading_time + .since(&previous_reading_time) + .as_secs_f64()) + }); + + let net_keys_with_stats = zip( + [ + "bytes_per_second/rx", + "packets_per_second/rx", + "errors_per_second/rx", + "dropped_per_second/rx", + "fifo/rx", + "frame/rx", + "compressed/rx", + "multicast/rx", + "bytes_per_second/tx", + "packets_per_second/tx", + "errors_per_second/tx", + "dropped_per_second/tx", + "fifo/tx", + "colls/tx", + "carrier/tx", + "compressed/tx", + ], + current_period_rates, + ) + // Filter out metrics we don't want memfaultd to include in reports like fifo and colls + .filter(|(key, _)| NETWORK_INTERFACE_METRIC_KEYS.contains(key)) + .collect::>(); + + let timestamp = Utc::now(); + let readings = net_keys_with_stats + .iter() + .map(|(key, value)| -> Result { + Ok(KeyedMetricReading::new( + MetricStringKey::from_str(&format!( + "{}/{}/{}", + NETWORK_INTERFACE_METRIC_NAMESPACE, interface, key + )) + .map_err(|e| eyre!(e))?, + MetricReading::Histogram { + value: *value, + timestamp, + }, + )) + }) + .collect::>>(); + match readings { + Ok(readings) => Ok(Some(readings)), + Err(e) => Err(e), + } + } else { + Ok(None) + } + } +} + +impl SystemMetricFamilyCollector for NetworkInterfaceMetricCollector +where + T: TimeMeasure + Copy + Ord + std::ops::Add + Send + Sync + 'static, +{ + fn family_name(&self) -> &'static str { + NETWORK_INTERFACE_METRIC_NAMESPACE + } + + fn collect_metrics(&mut self) -> Result> { + self.get_network_interface_metrics() + } +} + +#[cfg(test)] +mod test { + + use insta::{assert_json_snapshot, rounded_redaction, with_settings}; + use rstest::rstest; + + use super::*; + use crate::test_utils::TestInstant; + + #[rstest] + #[case(" eth0: 2707 25 0 0 0 0 0 0 2707 25 0 0 0 0 0 0", "eth0")] + #[case("wlan1: 2707 25 0 0 0 0 0 0 2707 25 0 0 0 0 0 0", "wlan1")] + fn test_parse_netdev_line(#[case] proc_net_dev_line: &str, #[case] test_name: &str) { + assert_json_snapshot!(test_name, + NetworkInterfaceMetricCollector::::parse_proc_net_dev_line(proc_net_dev_line).unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + } + #[rstest] + // Missing a colon after wlan0 + #[case("wlan0 2707 25 0 0 0 0 0 0 2707 25 0 0 0 0 0 0")] + // Only 15 stat values instead of 16 + #[case("wlan0: 2707 0 0 0 0 0 0 2707 25 0 0 0 0 0 0")] + fn test_fails_on_invalid_proc_net_dev_line(#[case] proc_net_dev_line: &str) { + assert!( + NetworkInterfaceMetricCollector::::parse_proc_net_dev_line( + proc_net_dev_line + ) + .is_err() + ) + } + + #[rstest] + #[case( + " eth0: 1000 25 0 0 0 0 0 0 2000 25 0 0 0 0 0 0", + " eth0: 2500 80 10 10 0 0 0 0 3000 50 0 0 0 0 0 0", + " eth0: 5000 100 15 15 0 0 0 0 5000 75 20 20 0 0 0 0", + "basic_delta" + )] + #[case( + " eth0: 4294967293 25 0 0 0 0 0 0 2000 25 0 0 0 0 0 0", + " eth0: 2498 80 10 10 0 0 0 0 3000 50 0 0 0 0 0 0", + " eth0: 5000 100 15 15 0 0 0 0 5000 75 20 20 0 0 0 0", + "with_overflow" + )] + fn test_net_if_metric_collector_calcs( + #[case] proc_net_dev_line_a: &str, + #[case] proc_net_dev_line_b: &str, + #[case] proc_net_dev_line_c: &str, + #[case] test_name: &str, + ) { + let mut net_metric_collector = NetworkInterfaceMetricCollector::::new( + NetworkInterfaceMetricsConfig::Interfaces(HashSet::from_iter(["eth0".to_string()])), + ); + + let (net_if, stats) = + NetworkInterfaceMetricCollector::::parse_proc_net_dev_line( + proc_net_dev_line_a, + ) + .unwrap(); + let reading_a = ProcNetDevReading { + stats, + reading_time: TestInstant::now(), + }; + let result_a = net_metric_collector.calculate_network_metrics(net_if, reading_a); + matches!(result_a, Ok(None)); + + TestInstant::sleep(Duration::from_secs(10)); + + let (net_if, stats) = + NetworkInterfaceMetricCollector::::parse_proc_net_dev_line( + proc_net_dev_line_b, + ) + .unwrap(); + let reading_b = ProcNetDevReading { + stats, + reading_time: TestInstant::now(), + }; + let result_b = net_metric_collector.calculate_network_metrics(net_if, reading_b); + + assert!(result_b.is_ok()); + + with_settings!({sort_maps => true}, { + assert_json_snapshot!(format!("{}_{}", test_name, "a_b_metrics"), + result_b.unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + }); + + TestInstant::sleep(Duration::from_secs(30)); + + let (net_if, stats) = + NetworkInterfaceMetricCollector::::parse_proc_net_dev_line( + proc_net_dev_line_c, + ) + .unwrap(); + let reading_c = ProcNetDevReading { + stats, + reading_time: TestInstant::now(), + }; + let result_c = net_metric_collector.calculate_network_metrics(net_if, reading_c); + + assert!(result_c.is_ok()); + + with_settings!({sort_maps => true}, { + assert_json_snapshot!(format!("{}_{}", test_name, "b_c_metrics"), + result_c.unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + }); + } + + #[rstest] + #[case( + " eth0: 1000 25 0 0 0 0 0 0 2000 25 0 0 0 0 0 0", + " eth1: 2500 80 10 10 0 0 0 0 3000 50 0 0 0 0 0 0", + " eth0: 5000 100 15 15 0 0 0 0 5000 75 20 20 0 0 0 0", + true, + "different_interfaces" + )] + fn test_net_if_metric_collector_different_if( + #[case] proc_net_dev_line_a: &str, + #[case] proc_net_dev_line_b: &str, + #[case] proc_net_dev_line_c: &str, + #[case] use_auto_config: bool, + #[case] test_name: &str, + ) { + let mut net_metric_collector = + NetworkInterfaceMetricCollector::::new(if use_auto_config { + NetworkInterfaceMetricsConfig::Auto + } else { + NetworkInterfaceMetricsConfig::Interfaces(HashSet::from_iter(["eth1".to_string()])) + }); + + let (net_if, stats) = + NetworkInterfaceMetricCollector::::parse_proc_net_dev_line( + proc_net_dev_line_a, + ) + .unwrap(); + let reading_a = ProcNetDevReading { + stats, + reading_time: TestInstant::now(), + }; + let result_a = net_metric_collector.calculate_network_metrics(net_if, reading_a); + matches!(result_a, Ok(None)); + + TestInstant::sleep(Duration::from_secs(10)); + + let (net_if, stats) = + NetworkInterfaceMetricCollector::::parse_proc_net_dev_line( + proc_net_dev_line_b, + ) + .unwrap(); + let reading_b = ProcNetDevReading { + stats, + reading_time: TestInstant::now(), + }; + let result_b = net_metric_collector.calculate_network_metrics(net_if, reading_b); + matches!(result_b, Ok(None)); + + TestInstant::sleep(Duration::from_secs(30)); + + let (net_if, stats) = + NetworkInterfaceMetricCollector::::parse_proc_net_dev_line( + proc_net_dev_line_c, + ) + .unwrap(); + let reading_c = ProcNetDevReading { + stats, + reading_time: TestInstant::now(), + }; + let result_c = net_metric_collector.calculate_network_metrics(net_if, reading_c); + + assert!(result_c.is_ok()); + + // 2 readings are required to calculate metrics (since they are rates), + // so we should only get actual metrics after processing reading_c + // (which is the second eth0 reading) + with_settings!({sort_maps => true}, { + assert_json_snapshot!(format!("{}_{}", test_name, "a_c_metrics"), + result_c.unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + }); + } + #[rstest] + #[case(vec!["eth0".to_string(), "wlan1".to_string()], "eth1", false)] + #[case(vec!["eth0".to_string(), "wlan1".to_string()], "eth0", true)] + #[case(vec!["eth0".to_string(), "wlan1".to_string()], "enp0s10", false)] + #[case(vec!["eth0".to_string(), "wlan1".to_string()], "wlan1", true)] + fn test_interface_is_monitored( + #[case] monitored_interfaces: Vec, + #[case] interface: &str, + #[case] should_be_monitored: bool, + ) { + let net_metric_collector = NetworkInterfaceMetricCollector::::new( + NetworkInterfaceMetricsConfig::Interfaces(HashSet::from_iter(monitored_interfaces)), + ); + assert_eq!( + net_metric_collector.interface_is_monitored(interface), + should_be_monitored + ) + } +} diff --git a/memfaultd/src/metrics/system_metrics/processes.rs b/memfaultd/src/metrics/system_metrics/processes.rs new file mode 100644 index 0000000..06a7eca --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/processes.rs @@ -0,0 +1,526 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Collect per-process metrics from /proc//stat +//! +//! Collects process-level metrics using the +//! /proc//stat files for processes whose +//! process name matches an item in the user-specified +//! list of processes to monitor. +//! +//! Example /proc//stat contents: +//! +//! 55270 (memfaultd) S 1 55270 55270 0 -1 4194368 825 0 0 0 155 102 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0 +//! +//! Further documentation of the /proc//stat file +//! can be found at: +//! https://man7.org/linux/man-pages/man5/proc_pid_stat.5.html +use std::{ + collections::HashMap, + collections::HashSet, + fs::{read_dir, read_to_string}, +}; + +use eyre::{eyre, Result}; +use log::warn; +use nom::character::complete::{alpha1, multispace1}; +use nom::{ + bytes::complete::{is_not, tag}, + character::complete::{space1, u64}, + multi::count, + number::complete::double, + sequence::{delimited, preceded, terminated}, + IResult, +}; + +use crate::metrics::{system_metrics::SystemMetricFamilyCollector, KeyedMetricReading}; +use crate::util::time_measure::TimeMeasure; + +const PROC_DIR: &str = "/proc/"; +pub const PROCESSES_METRIC_NAMESPACE: &str = "processes"; + +pub enum ProcessMetricsConfig { + Auto, + Processes(HashSet), +} + +#[derive(Clone, Debug)] +struct ProcessReading { + pid: u64, + name: String, + cputime_user: f64, + cputime_system: f64, + num_threads: f64, + rss: f64, + vm: f64, + pagefaults_major: f64, + pagefaults_minor: f64, + reading_time: T, +} + +pub struct ProcessMetricsCollector { + config: ProcessMetricsConfig, + processes: HashMap>, + clock_ticks_per_ms: f64, + bytes_per_page: f64, +} + +impl ProcessMetricsCollector +where + T: TimeMeasure + Copy + Send + Sync + 'static, +{ + pub fn new(config: ProcessMetricsConfig, clock_ticks_per_ms: f64, bytes_per_page: f64) -> Self { + Self { + config, + processes: HashMap::new(), + clock_ticks_per_ms, + bytes_per_page, + } + } + + fn process_is_monitored(&self, process_name: &str) -> bool { + match &self.config { + ProcessMetricsConfig::Auto => process_name == "memfaultd", + ProcessMetricsConfig::Processes(ps) => ps.contains(process_name), + } + } + + /// Parses the PID at the start of the /proc//stat + /// + /// Example input: + /// 55270 (memfaultd) S 1 55270 55270 0 -1 4194368 825 0 0 0 100 50 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + /// Example output: + /// 55270 + fn parse_pid(proc_pid_stat_line: &str) -> IResult<&str, u64> { + terminated(u64, space1)(proc_pid_stat_line) + } + + /// Parses the process name which follows the PID, delimited by ( and ), in /proc//stat + /// + /// Note this snippet from the documentation on /proc//stat: + /// + /// (2) comm %s + /// The filename of the executable, in parentheses. + /// Strings longer than TASK_COMM_LEN (16) characters + /// (including the terminating null byte) are silently + /// truncated. This is visible whether or not the + /// executable is swapped out. + /// + /// So for executables with names longer than 16 characters, the + /// name will be truncated to just the first 16 characters. + /// + /// Example input: + /// (memfaultd) S 1 55270 55270 0 -1 4194368 825 0 0 0 100 50 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + /// Example output: + /// memfaultd + fn parse_name(proc_pid_stat_line: &str) -> IResult<&str, &str> { + // This will break if there's a ')' in the process name - that seems unlikely + // enough to leave as-is for now + delimited(tag("("), is_not(")"), tag(")"))(proc_pid_stat_line) + } + + /// Parses the process state which follows the PID, delimited by ( and ) + /// + /// Example input: + /// S 1 55270 55270 0 -1 4194368 825 0 0 0 100 50 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + /// Example output: + /// S + fn parse_state(proc_pid_stat_line: &str) -> IResult<&str, &str> { + preceded(space1, alpha1)(proc_pid_stat_line) + } + + /// Parses the process state which follows the PID, delimited by ( and ) + /// + /// The following values from the resulting Vector are currently used: + /// - minfault: The number of minor faults the process has made + /// which have not required loading a memory page from + /// disk. (index 6) + /// - majfault: The number of major faults the process has made + /// which have required loading a memory page from + /// disk. (index 8) + /// - utime: Amount of time that this process has been scheduled + /// in user mode, measured in clock ticks (index 10) + /// - stime: Amount of time that this process has been scheduled + /// in kernel mode, measured in clock ticks (index 11) + /// - num_threads: Number of threads in the corresponding process (Index 16) + /// - vsize: Virtual memory size in bytes for the process (index 19) + /// - rss: number of pages the process has in real memory (index 20) + /// + /// Example input: + /// 1 55270 55270 0 -1 4194368 825 0 0 0 100 50 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + /// Example output: + /// vec![1 55270 55270 0 -1 4194368 825 0 0 0 100 50 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0] + fn parse_stats(proc_pid_stat_line: &str) -> IResult<&str, Vec> { + // There are more than 29 values in a line but we don't use any past the 29th + // (kstkesp) so don't parse past that + count(preceded(multispace1, double), 29)(proc_pid_stat_line) + } + + /// Parses the full contents of a /proc//stat file into a ProcessReading + /// + /// If the process name is not in the set of configured processes to monitor, this + /// function will stop parsing and return Ok(None) to avoid doing unnecessary work. + fn parse_process_stat(&self, proc_pid_stat_line: &str) -> Result>> { + let (after_pid, pid) = Self::parse_pid(proc_pid_stat_line) + .map_err(|_e| eyre!("Failed to parse PID for process"))?; + let (after_name, name) = + Self::parse_name(after_pid).map_err(|_e| eyre!("Failed to parse process name"))?; + + // Don't bother continuing to parse processes that aren't monitored + if self.process_is_monitored(name) { + let (after_state, _) = Self::parse_state(after_name) + .map_err(|_e| eyre!("Failed to parse process state for {}", name))?; + let (_, stats) = Self::parse_stats(after_state) + .map_err(|_e| eyre!("Failed to parse process stats for {}", name))?; + + let pagefaults_minor = *stats + .get(6) + .ok_or(eyre!("Failed to read pagefaults_minor"))?; + let pagefaults_major = *stats + .get(8) + .ok_or(eyre!("Failed to read pagefaults_major"))?; + + let cputime_user = *stats.get(10).ok_or(eyre!("Failed to read cputime_user"))?; + let cputime_system = *stats.get(11).ok_or(eyre!("Failed to read cputime_user"))?; + + let num_threads = *stats.get(16).ok_or(eyre!("Failed to read num_threads"))?; + + let vm = *stats.get(19).ok_or(eyre!("Failed to read vm"))?; + + // RSS is provided as the number of pages used by the process, we need + // to multiply by the system-specific bytes per page to get a value in bytes + let rss = *stats.get(20).ok_or(eyre!("Failed to read rss"))? * self.bytes_per_page; + + Ok(Some(ProcessReading { + pid, + name: name.to_string(), + cputime_user, + cputime_system, + num_threads, + rss, + pagefaults_major, + pagefaults_minor, + vm, + reading_time: T::now(), + })) + } else { + Ok(None) + } + } + + fn calculate_metric_readings( + &self, + previous: ProcessReading, + current: ProcessReading, + ) -> Result> { + let rss_reading = KeyedMetricReading::new_histogram( + format!("processes/{}/rss_bytes", current.name) + .as_str() + .parse() + .map_err(|e| eyre!("Couldn't parse metric key: {}", e))?, + current.rss, + ); + + let vm_reading = KeyedMetricReading::new_histogram( + format!("processes/{}/vm_bytes", current.name) + .as_str() + .parse() + .map_err(|e| eyre!("Couldn't parse metric key: {}", e))?, + current.vm, + ); + + let num_threads_reading = KeyedMetricReading::new_histogram( + format!("processes/{}/num_threads", current.name) + .as_str() + .parse() + .map_err(|e| eyre!("Couldn't parse metric key: {}", e))?, + current.num_threads, + ); + + // The values from /proc//stat are monotonic counters of jiffies since the + // process was started. We need to calculate the difference since the last reading, + // divide by the amount of jiffies in a millisecond (to get milliseconds spent in + // the given state), then divide by the total number of milliseconds since the + // previous reading in order to give us the % of time this process caused + // the CPU to spend in the user and system states. + let cputime_user_pct = (((current.cputime_user - previous.cputime_user) + / self.clock_ticks_per_ms) + / (current + .reading_time + .since(&previous.reading_time) + .as_millis() as f64)) + * 100.0; + + let cputime_sys_pct = (((current.cputime_system - previous.cputime_system) + / self.clock_ticks_per_ms) + / (current + .reading_time + .since(&previous.reading_time) + .as_millis() as f64)) + * 100.0; + + let utime_reading = KeyedMetricReading::new_histogram( + format!("processes/{}/cpu/percent/user", current.name) + .as_str() + .parse() + .map_err(|e| eyre!("Couldn't parse metric key: {}", e))?, + cputime_user_pct, + ); + let stime_reading = KeyedMetricReading::new_histogram( + format!("processes/{}/cpu/percent/system", current.name) + .as_str() + .parse() + .map_err(|e| eyre!("Couldn't parse metric key: {}", e))?, + cputime_sys_pct, + ); + + let pagefaults_minor_reading = KeyedMetricReading::new_histogram( + format!("processes/{}/pagefaults/minor", current.name) + .as_str() + .parse() + .map_err(|e| eyre!("Couldn't parse metric key: {}", e))?, + current.pagefaults_minor - previous.pagefaults_minor, + ); + let pagefaults_major_reading = KeyedMetricReading::new_histogram( + format!("processes/{}/pagefaults/major", current.name) + .as_str() + .parse() + .map_err(|e| eyre!("Couldn't parse metric key: {}", e))?, + current.pagefaults_major - previous.pagefaults_major, + ); + + Ok(vec![ + rss_reading, + vm_reading, + num_threads_reading, + stime_reading, + utime_reading, + pagefaults_minor_reading, + pagefaults_major_reading, + ]) + } + + // To facilitate unit testing, make the process directory path an arg + fn read_process_metrics_from_dir(&mut self, proc_dir: &str) -> Result> { + let process_readings: Vec<_> = read_dir(proc_dir)? + .filter_map(|entry| entry.map(|e| e.path()).ok()) + // Filter out non-numeric directories (since these won't be PIDs) + .filter(|path| match path.file_name() { + Some(p) => p.to_string_lossy().chars().all(|c| c.is_numeric()), + None => false, + }) + // Append "/stat" to the path since this is the file we want to read + // for a given PID's directory + .filter_map(|path| read_to_string(path.join("stat")).ok()) + .filter_map(|proc_pid_stat_contents| { + self.parse_process_stat(&proc_pid_stat_contents).ok() + }) + .flatten() + .collect(); + + let mut process_metric_readings = vec![]; + for current_reading in process_readings { + // A previous reading is required to calculate CPU time %s, as + // /proc//stat only has monotonic counters that track + // time spent in CPU states. Without a delta over a known period + // of time, we can't know if the contents of the counter are relevant + // to the current sampling window or not. + // + // For simplicity, only return any metrics for a PID when there + // is a previous reading for that PID in the `processes` map. + if let Some(previous_reading) = self + .processes + .insert(current_reading.pid, current_reading.clone()) + { + match self.calculate_metric_readings(previous_reading, current_reading.clone()) { + Ok(metric_readings) => process_metric_readings.extend(metric_readings), + Err(e) => warn!( + "Couldn't calculate metric readings for process {} (PID {}): {}", + current_reading.name, current_reading.pid, e + ), + } + } + } + + Ok(process_metric_readings) + } + + pub fn get_process_metrics(&mut self) -> Result> { + self.read_process_metrics_from_dir(PROC_DIR) + } +} + +impl SystemMetricFamilyCollector for ProcessMetricsCollector +where + T: TimeMeasure + Copy + Send + Sync + 'static, +{ + fn family_name(&self) -> &'static str { + PROCESSES_METRIC_NAMESPACE + } + + fn collect_metrics(&mut self) -> Result> { + self.get_process_metrics() + } +} + +#[cfg(test)] +mod tests { + use std::{ + fs::{create_dir, remove_file, File}, + io::Write, + time::Duration, + }; + use tempfile::tempdir; + + use insta::{assert_json_snapshot, rounded_redaction, with_settings}; + use rstest::rstest; + + use super::*; + use crate::test_utils::TestInstant; + + #[rstest] + fn test_parse_single_line() { + let collector = ProcessMetricsCollector::::new( + ProcessMetricsConfig::Processes(HashSet::from_iter(["memfaultd".to_string()])), + 100.0, + 4096.0, + ); + + let line = "55270 (memfaultd) S 1 55270 55270 0 -1 4194368 825 0 0 0 155 102 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0"; + assert!( + ProcessMetricsCollector::::parse_process_stat(&collector, line).is_ok() + ); + } + + #[rstest] + #[case( + "55270 (memfaultd) S 1 55270 55270 0 -1 4194368 825 0 0 0 100 50 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + "55270 (memfaultd) S 1 55270 55270 0 -1 4194368 875 0 10 0 1100 550 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + "simple_cpu_delta", + )] + fn test_collect_metrics(#[case] line1: &str, #[case] line2: &str, #[case] test_name: &str) { + let collector = ProcessMetricsCollector::::new( + ProcessMetricsConfig::Processes(HashSet::from_iter(["memfaultd".to_string()])), + 100.0, + 4096.0, + ); + + let first_reading = + ProcessMetricsCollector::::parse_process_stat(&collector, line1) + .unwrap() + .unwrap(); + + TestInstant::sleep(Duration::from_secs(10)); + + let second_reading = + ProcessMetricsCollector::::parse_process_stat(&collector, line2) + .unwrap() + .unwrap(); + + let process_metric_readings = + collector.calculate_metric_readings(first_reading, second_reading); + with_settings!({sort_maps => true}, { + assert_json_snapshot!(format!("{}_{}", test_name, "metrics"), + process_metric_readings.unwrap(), + {"[].value.**.timestamp" => "[timestamp]", "[].value.**.value" => rounded_redaction(5)}) + }); + } + + #[rstest] + #[case( + "55270 (memfaultd) S 1 55270 55270 0 -1 4194368 825 0 0 0 100 50 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + "24071 (systemd) S 1 24071 24071 0 -1 4194560 1580 2275 0 0 12 2 0 1 20 0 1 0 1465472 19828736 2784 18446744073709551615 1 1 0 0 0 0 671173123 4096 0 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + "55270 (memfaultd) S 1 55270 55270 0 -1 4194368 845 0 16 0 1100 550 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + "24071 (systemd) S 1 24071 24071 0 -1 4194560 1580 2275 0 0 100 30 0 1 20 0 1 0 1465472 19828736 2784 18446744073709551615 1 1 0 0 0 0 671173123 4096 0 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + false, + )] + #[case( + "55270 (memfaultd) S 1 55270 55270 0 -1 4194368 825 0 0 0 100 50 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + "24071 (systemd) S 1 24071 24071 0 -1 4194560 1580 2275 0 0 12 2 0 1 20 0 1 0 1465472 19828736 2784 18446744073709551615 1 1 0 0 0 0 671173123 4096 0 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + "55270 (memfaultd) S 1 55270 55270 0 -1 4194368 825 0 0 0 1100 550 0 0 20 0 19 0 18548522 1411293184 4397 18446744073709551615 1 1 0 0 0 0 0 4096 17987 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + "24071 (systemd) S 1 24071 24071 0 -1 4194560 1580 2275 0 0 100 30 0 1 20 0 1 0 1465472 19828736 2784 18446744073709551615 1 1 0 0 0 0 671173123 4096 0 0 0 0 17 7 0 0 0 0 0 0 0 0 0 0 0 0 0", + true, + )] + fn test_process_stats_from_proc( + #[case] process_a_sample_1: &str, + #[case] process_b_sample_1: &str, + #[case] process_a_sample_2: &str, + #[case] process_b_sample_2: &str, + #[case] use_auto: bool, + ) { + let mut collector = if use_auto { + ProcessMetricsCollector::::new(ProcessMetricsConfig::Auto, 100.0, 4096.0) + } else { + // If auto is not used, the configuration should capture metrics from both processes + ProcessMetricsCollector::::new( + ProcessMetricsConfig::Processes(HashSet::from_iter([ + "memfaultd".to_string(), + "systemd".to_string(), + ])), + 100.0, + 4096.0, + ) + }; + + // Create a temporary directory. + let dir = tempdir().unwrap(); + + let temp_proc_dir = dir.path().join("temp_proc"); + create_dir(&temp_proc_dir).unwrap(); + + // Set up /proc//stat files for first sample for both + // memfaultd and systemd + let process_a_dir = temp_proc_dir.join("55270"); + create_dir(&process_a_dir).unwrap(); + + let process_a_path = process_a_dir.join("stat"); + let mut process_a_file = File::create(process_a_path.clone()).unwrap(); + + let process_b_dir = temp_proc_dir.join("24071"); + create_dir(&process_b_dir).unwrap(); + let process_b_path = process_b_dir.join("stat"); + let mut process_b_file = File::create(process_b_path.clone()).unwrap(); + + writeln!(process_a_file, "{}", process_a_sample_1).unwrap(); + writeln!(process_b_file, "{}", process_b_sample_1).unwrap(); + + // Read /proc//stat files for second sample for both + // memfaultd and systemd + let process_a_path = temp_proc_dir.join("55270").join("stat"); + let mut process_a_file = File::create(process_a_path).unwrap(); + + let process_b_path = temp_proc_dir.join("24071").join("stat"); + let mut process_b_file = File::create(process_b_path).unwrap(); + + writeln!(process_a_file, "{}", process_a_sample_2).unwrap(); + writeln!(process_b_file, "{}", process_b_sample_2).unwrap(); + + // Read /proc/ "[timestamp]", "[].value.**.value" => rounded_redaction(5)}); + + // Delete the temporary directory. + dir.close().unwrap(); + } +} diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__basic_delta_a_b_metrics.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__basic_delta_a_b_metrics.snap new file mode 100644 index 0000000..0cda315 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__basic_delta_a_b_metrics.snap @@ -0,0 +1,69 @@ +--- +source: memfaultd/src/metrics/system_metrics/cpu.rs +expression: result_b.unwrap() +--- +[ + { + "name": "cpu/cpu/percent/user", + "value": { + "Histogram": { + "value": 96.15385, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/nice", + "value": { + "Histogram": { + "value": 2.88462, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/system", + "value": { + "Histogram": { + "value": 0.76923, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/idle", + "value": { + "Histogram": { + "value": 0.19231, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/iowait", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/irq", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/softirq", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__basic_delta_b_c_metrics.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__basic_delta_b_c_metrics.snap new file mode 100644 index 0000000..9e65a0a --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__basic_delta_b_c_metrics.snap @@ -0,0 +1,69 @@ +--- +source: memfaultd/src/metrics/system_metrics/cpu.rs +expression: result_c.unwrap() +--- +[ + { + "name": "cpu/cpu/percent/user", + "value": { + "Histogram": { + "value": 18.58736, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/nice", + "value": { + "Histogram": { + "value": 66.9145, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/system", + "value": { + "Histogram": { + "value": 13.3829, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/idle", + "value": { + "Histogram": { + "value": 0.74349, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/iowait", + "value": { + "Histogram": { + "value": 0.37175, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/irq", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu/percent/softirq", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__different_cores_a_c_metrics.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__different_cores_a_c_metrics.snap new file mode 100644 index 0000000..13dd9ed --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__different_cores_a_c_metrics.snap @@ -0,0 +1,69 @@ +--- +source: memfaultd/src/metrics/system_metrics/cpu.rs +expression: result_c.unwrap() +--- +[ + { + "name": "cpu/cpu1/percent/user", + "value": { + "Histogram": { + "value": 70.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu1/percent/nice", + "value": { + "Histogram": { + "value": 10.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu1/percent/system", + "value": { + "Histogram": { + "value": 10.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu1/percent/idle", + "value": { + "Histogram": { + "value": 2.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu1/percent/iowait", + "value": { + "Histogram": { + "value": 5.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu1/percent/irq", + "value": { + "Histogram": { + "value": 3.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "cpu/cpu1/percent/softirq", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__test_basic_line.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__test_basic_line.snap new file mode 100644 index 0000000..bf5cca9 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__test_basic_line.snap @@ -0,0 +1,16 @@ +--- +source: memfaultd/src/metrics/system_metrics/cpu.rs +expression: "CpuMetricCollector::parse_proc_stat_line_cpu(proc_stat_line).unwrap()" +--- +[ + "cpu", + [ + 1000.0, + 5.0, + 0.0, + 0.0, + 2.0, + 0.0, + 0.0 + ] +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__test_basic_line_with_extra.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__test_basic_line_with_extra.snap new file mode 100644 index 0000000..11e8b35 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__cpu__test__test_basic_line_with_extra.snap @@ -0,0 +1,16 @@ +--- +source: memfaultd/src/metrics/system_metrics/cpu.rs +expression: "CpuMetricCollector::parse_proc_stat_line_cpu(proc_stat_line).unwrap()" +--- +[ + "cpu0", + [ + 1000.0, + 5.0, + 0.0, + 0.0, + 2.0, + 0.0, + 0.0 + ] +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__disk_space__test__initialize_and_calc_disk_space_for_mounts-2.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__disk_space__test__initialize_and_calc_disk_space_for_mounts-2.snap new file mode 100644 index 0000000..c3dbaf0 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__disk_space__test__initialize_and_calc_disk_space_for_mounts-2.snap @@ -0,0 +1,42 @@ +--- +source: memfaultd/src/metrics/system_metrics/disk_space.rs +expression: metrics +--- +[ + { + "name": "disk_space/sda1/free_bytes", + "value": { + "Histogram": { + "value": 1171456.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "disk_space/sda1/used_bytes", + "value": { + "Histogram": { + "value": 3022848.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "disk_space/sda2/free_bytes", + "value": { + "Histogram": { + "value": 1171456.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "disk_space/sda2/used_bytes", + "value": { + "Histogram": { + "value": 3022848.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__disk_space__test__initialize_and_calc_disk_space_for_mounts.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__disk_space__test__initialize_and_calc_disk_space_for_mounts.snap new file mode 100644 index 0000000..720db1a --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__disk_space__test__initialize_and_calc_disk_space_for_mounts.snap @@ -0,0 +1,14 @@ +--- +source: memfaultd/src/metrics/system_metrics/disk_space.rs +expression: disk_space_collector.mounts +--- +[ + { + "device": "/dev/sda1", + "mount_point": "/" + }, + { + "device": "/dev/sda2", + "mount_point": "/media" + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__disk_space__test__unmonitored_disks_not_initialized.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__disk_space__test__unmonitored_disks_not_initialized.snap new file mode 100644 index 0000000..f41c0ed --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__disk_space__test__unmonitored_disks_not_initialized.snap @@ -0,0 +1,5 @@ +--- +source: memfaultd/src/metrics/system_metrics/disk_space.rs +expression: disk_space_collector.mounts +--- +[] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__memory__test__get_memory_metrics.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__memory__test__get_memory_metrics.snap new file mode 100644 index 0000000..9964a31 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__memory__test__get_memory_metrics.snap @@ -0,0 +1,24 @@ +--- +source: memfaultd/src/metrics/system_metrics/memory.rs +expression: "MemoryMetricsCollector::parse_meminfo_memory_stats(meminfo).unwrap()" +--- +[ + { + "name": "memory/memory/free", + "value": { + "Histogram": { + "value": 248090624.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "memory/memory/used", + "value": { + "Histogram": { + "value": 75599872.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__memory__test__get_memory_metrics_no_memavailable.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__memory__test__get_memory_metrics_no_memavailable.snap new file mode 100644 index 0000000..171db3e --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__memory__test__get_memory_metrics_no_memavailable.snap @@ -0,0 +1,24 @@ +--- +source: memfaultd/src/metrics/system_metrics/memory.rs +expression: "MemoryMetricsCollector::parse_meminfo_memory_stats(meminfo).unwrap()" +--- +[ + { + "name": "memory/memory/free", + "value": { + "Histogram": { + "value": 248090624.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "memory/memory/used", + "value": { + "Histogram": { + "value": 126607360.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__basic_delta_a_b_metrics.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__basic_delta_a_b_metrics.snap new file mode 100644 index 0000000..b075d2c --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__basic_delta_a_b_metrics.snap @@ -0,0 +1,78 @@ +--- +source: memfaultd/src/metrics/system_metrics/network_interfaces.rs +expression: result_b.unwrap() +--- +[ + { + "name": "interface/eth0/bytes_per_second/rx", + "value": { + "Histogram": { + "value": 150.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/packets_per_second/rx", + "value": { + "Histogram": { + "value": 5.5, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/errors_per_second/rx", + "value": { + "Histogram": { + "value": 1.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/dropped_per_second/rx", + "value": { + "Histogram": { + "value": 1.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/bytes_per_second/tx", + "value": { + "Histogram": { + "value": 100.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/packets_per_second/tx", + "value": { + "Histogram": { + "value": 2.5, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/errors_per_second/tx", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/dropped_per_second/tx", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__basic_delta_b_c_metrics.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__basic_delta_b_c_metrics.snap new file mode 100644 index 0000000..747c537 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__basic_delta_b_c_metrics.snap @@ -0,0 +1,78 @@ +--- +source: memfaultd/src/metrics/system_metrics/network_interfaces.rs +expression: result_c.unwrap() +--- +[ + { + "name": "interface/eth0/bytes_per_second/rx", + "value": { + "Histogram": { + "value": 83.33333, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/packets_per_second/rx", + "value": { + "Histogram": { + "value": 0.66667, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/errors_per_second/rx", + "value": { + "Histogram": { + "value": 0.16667, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/dropped_per_second/rx", + "value": { + "Histogram": { + "value": 0.16667, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/bytes_per_second/tx", + "value": { + "Histogram": { + "value": 66.66667, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/packets_per_second/tx", + "value": { + "Histogram": { + "value": 0.83333, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/errors_per_second/tx", + "value": { + "Histogram": { + "value": 0.66667, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/dropped_per_second/tx", + "value": { + "Histogram": { + "value": 0.66667, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__different_interfaces_a_c_metrics.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__different_interfaces_a_c_metrics.snap new file mode 100644 index 0000000..dcab1f9 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__different_interfaces_a_c_metrics.snap @@ -0,0 +1,78 @@ +--- +source: memfaultd/src/metrics/system_metrics/network_interfaces.rs +expression: result_c.unwrap() +--- +[ + { + "name": "interface/eth0/bytes_per_second/rx", + "value": { + "Histogram": { + "value": 100.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/packets_per_second/rx", + "value": { + "Histogram": { + "value": 1.875, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/errors_per_second/rx", + "value": { + "Histogram": { + "value": 0.375, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/dropped_per_second/rx", + "value": { + "Histogram": { + "value": 0.375, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/bytes_per_second/tx", + "value": { + "Histogram": { + "value": 75.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/packets_per_second/tx", + "value": { + "Histogram": { + "value": 1.25, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/errors_per_second/tx", + "value": { + "Histogram": { + "value": 0.5, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/dropped_per_second/tx", + "value": { + "Histogram": { + "value": 0.5, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__eth0.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__eth0.snap new file mode 100644 index 0000000..8488d1b --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__eth0.snap @@ -0,0 +1,25 @@ +--- +source: memfaultd/src/metrics/system_metrics/network_interfaces.rs +expression: "NetworkInterfaceMetricCollector::::parse_proc_net_dev_line(proc_net_dev_line).unwrap()" +--- +[ + "eth0", + [ + 2707, + 25, + 0, + 0, + 0, + 0, + 0, + 0, + 2707, + 25, + 0, + 0, + 0, + 0, + 0, + 0 + ] +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__with_overflow_a_b_metrics.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__with_overflow_a_b_metrics.snap new file mode 100644 index 0000000..6aaac07 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__with_overflow_a_b_metrics.snap @@ -0,0 +1,78 @@ +--- +source: memfaultd/src/metrics/system_metrics/network_interfaces.rs +expression: result_b.unwrap() +--- +[ + { + "name": "interface/eth0/bytes_per_second/rx", + "value": { + "Histogram": { + "value": 250.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/packets_per_second/rx", + "value": { + "Histogram": { + "value": 5.5, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/errors_per_second/rx", + "value": { + "Histogram": { + "value": 1.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/dropped_per_second/rx", + "value": { + "Histogram": { + "value": 1.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/bytes_per_second/tx", + "value": { + "Histogram": { + "value": 100.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/packets_per_second/tx", + "value": { + "Histogram": { + "value": 2.5, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/errors_per_second/tx", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/dropped_per_second/tx", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__with_overflow_b_c_metrics.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__with_overflow_b_c_metrics.snap new file mode 100644 index 0000000..fb6382a --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__with_overflow_b_c_metrics.snap @@ -0,0 +1,78 @@ +--- +source: memfaultd/src/metrics/system_metrics/network_interfaces.rs +expression: result_c.unwrap() +--- +[ + { + "name": "interface/eth0/bytes_per_second/rx", + "value": { + "Histogram": { + "value": 83.4, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/packets_per_second/rx", + "value": { + "Histogram": { + "value": 0.66667, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/errors_per_second/rx", + "value": { + "Histogram": { + "value": 0.16667, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/dropped_per_second/rx", + "value": { + "Histogram": { + "value": 0.16667, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/bytes_per_second/tx", + "value": { + "Histogram": { + "value": 66.66667, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/packets_per_second/tx", + "value": { + "Histogram": { + "value": 0.83333, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/errors_per_second/tx", + "value": { + "Histogram": { + "value": 0.66667, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "interface/eth0/dropped_per_second/tx", + "value": { + "Histogram": { + "value": 0.66667, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__wlan1.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__wlan1.snap new file mode 100644 index 0000000..0476b78 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__network_interfaces__test__wlan1.snap @@ -0,0 +1,25 @@ +--- +source: memfaultd/src/metrics/system_metrics/network_interfaces.rs +expression: "NetworkInterfaceMetricCollector::::parse_proc_net_dev_line(proc_net_dev_line).unwrap()" +--- +[ + "wlan1", + [ + 2707, + 25, + 0, + 0, + 0, + 0, + 0, + 0, + 2707, + 25, + 0, + 0, + 0, + 0, + 0, + 0 + ] +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__processes__tests__process_metrics_auto_false.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__processes__tests__process_metrics_auto_false.snap new file mode 100644 index 0000000..3371196 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__processes__tests__process_metrics_auto_false.snap @@ -0,0 +1,132 @@ +--- +source: memfaultd/src/metrics/system_metrics/processes.rs +expression: process_metric_readings_2 +--- +[ + { + "name": "processes/memfaultd/cpu/percent/system", + "value": { + "Histogram": { + "value": 0.05, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/cpu/percent/user", + "value": { + "Histogram": { + "value": 0.1, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/num_threads", + "value": { + "Histogram": { + "value": 19.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/pagefaults/major", + "value": { + "Histogram": { + "value": 16.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/pagefaults/minor", + "value": { + "Histogram": { + "value": 20.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/rss_bytes", + "value": { + "Histogram": { + "value": 18010112.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/vm_bytes", + "value": { + "Histogram": { + "value": 1411293184.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/systemd/cpu/percent/system", + "value": { + "Histogram": { + "value": 0.0028, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/systemd/cpu/percent/user", + "value": { + "Histogram": { + "value": 0.0088, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/systemd/num_threads", + "value": { + "Histogram": { + "value": 1.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/systemd/pagefaults/major", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/systemd/pagefaults/minor", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/systemd/rss_bytes", + "value": { + "Histogram": { + "value": 11403264.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/systemd/vm_bytes", + "value": { + "Histogram": { + "value": 19828736.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__processes__tests__process_metrics_auto_true.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__processes__tests__process_metrics_auto_true.snap new file mode 100644 index 0000000..8380217 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__processes__tests__process_metrics_auto_true.snap @@ -0,0 +1,69 @@ +--- +source: memfaultd/src/metrics/system_metrics/processes.rs +expression: process_metric_readings_2 +--- +[ + { + "name": "processes/memfaultd/cpu/percent/system", + "value": { + "Histogram": { + "value": 0.05, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/cpu/percent/user", + "value": { + "Histogram": { + "value": 0.1, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/num_threads", + "value": { + "Histogram": { + "value": 19.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/pagefaults/major", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/pagefaults/minor", + "value": { + "Histogram": { + "value": 0.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/rss_bytes", + "value": { + "Histogram": { + "value": 18010112.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/vm_bytes", + "value": { + "Histogram": { + "value": 1411293184.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__processes__tests__simple_cpu_delta_metrics.snap b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__processes__tests__simple_cpu_delta_metrics.snap new file mode 100644 index 0000000..3c733cc --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/snapshots/memfaultd__metrics__system_metrics__processes__tests__simple_cpu_delta_metrics.snap @@ -0,0 +1,69 @@ +--- +source: memfaultd/src/metrics/system_metrics/processes.rs +expression: process_metric_readings.unwrap() +--- +[ + { + "name": "processes/memfaultd/rss_bytes", + "value": { + "Histogram": { + "value": 18010112.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/vm_bytes", + "value": { + "Histogram": { + "value": 1411293184.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/num_threads", + "value": { + "Histogram": { + "value": 19.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/cpu/percent/system", + "value": { + "Histogram": { + "value": 0.05, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/cpu/percent/user", + "value": { + "Histogram": { + "value": 0.1, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/pagefaults/minor", + "value": { + "Histogram": { + "value": 50.0, + "timestamp": "[timestamp]" + } + } + }, + { + "name": "processes/memfaultd/pagefaults/major", + "value": { + "Histogram": { + "value": 10.0, + "timestamp": "[timestamp]" + } + } + } +] diff --git a/memfaultd/src/metrics/system_metrics/thermal.rs b/memfaultd/src/metrics/system_metrics/thermal.rs new file mode 100644 index 0000000..d251f32 --- /dev/null +++ b/memfaultd/src/metrics/system_metrics/thermal.rs @@ -0,0 +1,138 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Collect temperature readings from /sys/class/thermal +//! +//! This module parses thermal readings from /sys/class/thermal and constructs +//! KeyedMetricReadings based on those statistics. +//! +//! Example /sys/class/thermal contents: +//! +//! /sys/class/thermal/ +//! ├── cooling_device0 -> ../../devices/virtual/thermal/cooling_device0 +//! ├── cooling_device1 -> ../../devices/virtual/thermal/cooling_device1 +//! ├── thermal_zone0 -> ../../devices/virtual/thermal/thermal_zone0 +//! └── thermal_zone1 -> ../../devices/virtual/thermal/thermal_zone1 +//! +//! Example /sys/class/thermal/thermal_zone[0-*] contents: +//! +//! /sys/class/thermal/thermal_zone0 +//! ├── ... +//! └── temp // this is the property we're interested in +//! +//! See additional Linux kernel documentation on /sys/class/thermal here: +//! https://www.kernel.org/doc/Documentation/thermal/sysfs-api.txt + +use std::str::FromStr; + +use crate::metrics::{ + system_metrics::SystemMetricFamilyCollector, KeyedMetricReading, MetricStringKey, +}; +use eyre::{eyre, Result}; + +const SYS_CLASS_THERMAL_PATH: &str = "/sys/class/thermal"; +pub const THERMAL_METRIC_NAMESPACE: &str = "thermal"; + +pub struct ThermalMetricsCollector; + +impl ThermalMetricsCollector { + pub fn new() -> Self { + ThermalMetricsCollector {} + } + + fn read_thermal_zone_temp(zone_name: &str, root_dir: &str) -> Result { + let temp_file = &format!("{}/{}/temp", root_dir, zone_name); + // The readings are in millidegrees Celsius, so we divide by 1000 to get + // the temperature in degrees Celsius. + let temp_in_celsius = std::fs::read_to_string(temp_file)?.trim().parse::()? / 1000.0; + + Ok(KeyedMetricReading::new_gauge( + MetricStringKey::from_str(zone_name).map_err(|e| { + eyre!( + "Failed to construct MetricStringKey for thermal zone: {}", + e + ) + })?, + temp_in_celsius, + )) + } + + // To facilitate unit testing, make the thermal directory path an arg + fn read_thermal_metrics_from_dir(dir: &str) -> Result> { + // The /sys/class/thermal/ directory will contain symlinks to + // pseudo-files named "thermal_zone0" etc, depending on the number of + // thermal zones in the system. The file we read for the temperature + // reading is for example /sys/class/thermal/thermal_zone0/temp, + // containing an integer value in millidegrees Celsius, ex: "53000" + let metrics: Vec<_> = std::fs::read_dir(dir)? + .filter_map(|entry| entry.ok()) + .map(|entry| entry.path()) + .filter_map(|path| Some(path.file_name()?.to_str()?.to_string())) + .filter(|name| name.starts_with("thermal_zone")) + .filter_map(|name| Self::read_thermal_zone_temp(&name, dir).ok()) + .collect(); + + Ok(metrics) + } + + pub fn get_thermal_metrics() -> Result> { + Self::read_thermal_metrics_from_dir(SYS_CLASS_THERMAL_PATH) + } +} + +impl SystemMetricFamilyCollector for ThermalMetricsCollector { + fn family_name(&self) -> &'static str { + THERMAL_METRIC_NAMESPACE + } + + fn collect_metrics(&mut self) -> Result> { + Self::get_thermal_metrics() + } +} + +#[cfg(test)] +// The floating point literal pattern is allowed in this test module because +// the input and output values are known. +mod tests { + use super::*; + use crate::metrics::MetricReading; + use std::fs::File; + use std::io::Write; + use tempfile::tempdir; + + #[test] + fn test_read_thermal_zone_temp() { + // Create a temporary directory. + let dir = tempdir().unwrap(); + + // Create a "thermal_zone0" directory inside the temporary directory. + let thermal_zone_dir = dir.path().join("thermal_zone0"); + std::fs::create_dir(&thermal_zone_dir).unwrap(); + + // Create a "temp" file inside the "thermal_zone0" directory. + let temp_file_path = thermal_zone_dir.join("temp"); + let mut temp_file = File::create(temp_file_path).unwrap(); + + // Write the temperature (in millidegrees Celsius) to the "temp" file. + writeln!(temp_file, "50000").unwrap(); + + // Call the function and check the result. + let result = ThermalMetricsCollector::read_thermal_zone_temp( + "thermal_zone0", + dir.path().to_str().unwrap(), + ) + .unwrap(); + // The temperature should be 50.0 degrees Celsius. + assert!(matches!( + result.value, + MetricReading::Gauge { + #[allow(illegal_floating_point_literal_pattern)] + value: 50.0, + .. + } + )); + + // Delete the temporary directory. + dir.close().unwrap(); + } +} diff --git a/memfaultd/src/metrics/timeseries/mod.rs b/memfaultd/src/metrics/timeseries/mod.rs new file mode 100644 index 0000000..836feae --- /dev/null +++ b/memfaultd/src/metrics/timeseries/mod.rs @@ -0,0 +1,447 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use chrono::{DateTime, Utc}; +use eyre::{eyre, Result}; +use std::cmp; + +use super::{MetricReading, MetricValue}; + +const FINITENESS_ERROR: &str = "Metric values must be finite."; + +/// A trait for the storage of multiple metric events aggregated together. +/// This (roughly) maps to a time series in OpenTelemetry data model: +/// https://opentelemetry.io/docs/specs/otel/metrics/data-model/#timeseries-model +pub trait TimeSeries { + fn aggregate(&mut self, newer: &MetricReading) -> Result<()>; + fn value(&self) -> MetricValue; +} + +pub struct Histogram { + sum: f64, + count: u64, + start: DateTime, + end: DateTime, + min: f64, + max: f64, +} + +impl Histogram { + pub fn new(reading: &MetricReading) -> Result { + match *reading { + MetricReading::Histogram { value, timestamp } => { + if !value.is_finite() { + return Err(eyre!(FINITENESS_ERROR)); + } + Ok(Self { + sum: value, + count: 1, + start: timestamp, + end: timestamp, + min: value, + max: value, + }) + } + _ => Err(eyre!("Cannot create a histogram from a non-gauge metric")), + } + } +} + +impl TimeSeries for Histogram { + fn aggregate(&mut self, newer: &MetricReading) -> Result<()> { + match newer { + MetricReading::Histogram { value, timestamp } => { + if !value.is_finite() { + return Err(eyre!(FINITENESS_ERROR)); + } + self.sum += value; + self.count += 1; + self.start = cmp::min(self.start, *timestamp); + self.end = cmp::max(self.end, *timestamp); + self.min = f64::min(self.min, *value); + self.max = f64::max(self.max, *value); + Ok(()) + } + _ => Err(eyre!( + "Cannot aggregate a histogram with a non-gauge metric" + )), + } + } + + fn value(&self) -> MetricValue { + if self.count > 0 { + MetricValue::Number(self.sum / self.count as f64) + } else { + MetricValue::Number(f64::NAN) + } + } +} + +/// An aggregation that calculates the sum of all values received. This assumes that all readings will be positive numbers. +/// Monotonic counter in OpenTelemetry data model. +pub struct Counter { + sum: f64, + end: DateTime, +} + +impl Counter { + pub fn new(reading: &MetricReading) -> Result { + match *reading { + MetricReading::Counter { value, timestamp } => { + if !value.is_finite() { + return Err(eyre!(FINITENESS_ERROR)); + } + Ok(Self { + sum: value, + end: timestamp, + }) + } + _ => Err(eyre!("Cannot create a sum from a non-counter metric")), + } + } +} + +impl TimeSeries for Counter { + fn aggregate(&mut self, newer: &MetricReading) -> Result<()> { + match newer { + MetricReading::Counter { value, timestamp } => { + if !value.is_finite() { + return Err(eyre!(FINITENESS_ERROR)); + } + self.sum += value; + self.end = *timestamp; + Ok(()) + } + _ => Err(eyre!("Cannot aggregate a sum with a non-counter metric")), + } + } + + fn value(&self) -> MetricValue { + MetricValue::Number(self.sum) + } +} + +/// A time-weighted sum of all values received. This is useful to maintain an accurate average measurement when the interval between readings is not constant. +pub struct TimeWeightedAverage { + weighted_sum: f64, + duration: u64, + end: DateTime, +} + +impl TimeWeightedAverage { + pub fn new(reading: &MetricReading) -> Result { + match *reading { + MetricReading::TimeWeightedAverage { + value, + timestamp, + interval, + } => { + if !value.is_finite() { + return Err(eyre!(FINITENESS_ERROR)); + } + Ok(Self { + weighted_sum: value * interval.num_milliseconds() as f64, + duration: interval.num_milliseconds() as u64, + end: timestamp, + }) + } + _ => Err(eyre!( + "Mismatch between Time Weighted Average aggregation and {:?} reading", + reading + )), + } + } +} + +impl TimeSeries for TimeWeightedAverage { + fn aggregate(&mut self, newer: &MetricReading) -> Result<()> { + match newer { + MetricReading::TimeWeightedAverage { + value, timestamp, .. + } => { + if !value.is_finite() { + return Err(eyre!(FINITENESS_ERROR)); + } + if timestamp < &self.end { + return Err(eyre!( + "Cannot aggregate a time-weighted average with an older timestamp" + )); + } + let duration = (*timestamp - self.end).num_milliseconds() as u64; + self.weighted_sum += value * duration as f64; + self.duration += duration; + self.end = *timestamp; + Ok(()) + } + _ => Err(eyre!( + "Cannot aggregate a time-weighted average with a non-gauge metric" + )), + } + } + + fn value(&self) -> MetricValue { + if self.duration > 0 { + MetricValue::Number(self.weighted_sum / self.duration as f64) + } else { + MetricValue::Number(f64::NAN) + } + } +} + +pub struct Gauge { + value: f64, + end: DateTime, +} + +impl Gauge { + pub fn new(reading: &MetricReading) -> Result { + match *reading { + MetricReading::Gauge { value, timestamp } => { + if !value.is_finite() { + return Err(eyre!(FINITENESS_ERROR)); + } + Ok(Self { + value, + end: timestamp, + }) + } + _ => Err(eyre!( + "Cannot create a gauge aggregation from a non-gauge metric" + )), + } + } +} + +impl TimeSeries for Gauge { + fn aggregate(&mut self, newer: &MetricReading) -> Result<()> { + match newer { + MetricReading::Gauge { value, timestamp } => { + if !value.is_finite() { + return Err(eyre!(FINITENESS_ERROR)); + } + if *timestamp > self.end { + self.value = *value; + self.end = *timestamp; + } + Ok(()) + } + _ => Err(eyre!( + "Cannot aggregate a histogram with a non-gauge metric" + )), + } + } + + fn value(&self) -> MetricValue { + MetricValue::Number(self.value) + } +} + +/// An aggregation stores the most recently received String +/// associated with a key as a tag on the report +pub struct ReportTag { + value: String, + end: DateTime, +} + +impl ReportTag { + pub fn new(reading: &MetricReading) -> Result { + match reading { + MetricReading::ReportTag { value, timestamp } => Ok(Self { + value: value.clone(), + end: *timestamp, + }), + _ => Err(eyre!( + "Cannot create a report tag from a non-report tag reading" + )), + } + } +} + +impl TimeSeries for ReportTag { + fn aggregate(&mut self, newer: &MetricReading) -> Result<()> { + match newer { + MetricReading::ReportTag { value, timestamp } => { + if *timestamp > self.end { + self.value = value.clone(); + self.end = *timestamp; + } + Ok(()) + } + _ => Err(eyre!( + "Cannot aggregate a report tag with a non-report-tag reading" + )), + } + } + + fn value(&self) -> MetricValue { + MetricValue::String(self.value.clone()) + } +} +#[cfg(test)] +mod tests { + use chrono::Duration; + use rstest::rstest; + + use crate::metrics::{MetricReading, MetricTimestamp, MetricValue}; + use std::{f64::INFINITY, f64::NAN, f64::NEG_INFINITY, str::FromStr}; + + use super::TimeSeries; + use super::{Counter, Gauge, Histogram, TimeWeightedAverage}; + + #[rstest] + #[case(1.0, 1000, 2.0, 1.5, 1000)] + #[case(10.0, 10_000, 10.0, 10.0, 10_000)] + #[case(1.0, 9_000, 0.0, 0.5, 9_000)] + #[case(1.0, 0, 2.0, 1.5, 0)] + #[case(1.0, 1000, 2.0, 1.5, 1000)] + fn test_histogram_aggregation( + #[case] a: f64, + #[case] duration_between_ms: i64, + #[case] b: f64, + #[case] expected: f64, + #[case] expected_ms: i64, + ) { + let t0 = MetricTimestamp::from_str("2021-01-01T00:00:00Z").unwrap(); + + let a = MetricReading::Histogram { + value: a, + timestamp: t0, + }; + let b = MetricReading::Histogram { + value: b, + timestamp: t0 + Duration::milliseconds(duration_between_ms), + }; + + let mut h = Histogram::new(&a).unwrap(); + h.aggregate(&b).unwrap(); + + assert_eq!(h.start, t0); + assert_eq!(h.end, t0 + Duration::milliseconds(duration_between_ms)); + assert_eq!((h.end - h.start).num_milliseconds(), expected_ms); + assert_eq!(h.value(), MetricValue::Number(expected)); + } + + #[rstest] + #[case(1.0, 1000, 2.0, 1000, 1.5, 2000)] + #[case(10.0, 10000, 10.0, 1000, 10.0, 11000)] + #[case(1.0, 9_000, 0.0, 1_000, 0.9, 10_000)] + #[case(1.0, 0, 2.0, 1, 2.0, 1)] + #[case(1.0, 1000, 2.0, 0, 1.0, 1000)] + fn test_time_weighted_aggregation( + #[case] a: f64, + #[case] a_ms: i64, + #[case] b: f64, + #[case] b_ms: i64, + #[case] expected: f64, + #[case] expected_ms: u64, + ) { + let t0 = MetricTimestamp::from_str("2021-01-01T00:00:00Z").unwrap(); + + let a = MetricReading::TimeWeightedAverage { + value: a, + timestamp: t0 + Duration::milliseconds(a_ms), + interval: Duration::milliseconds(a_ms), + }; + let b = MetricReading::TimeWeightedAverage { + value: b, + timestamp: t0 + Duration::milliseconds(a_ms + b_ms), + interval: Duration::milliseconds(b_ms), + }; + + let mut h = TimeWeightedAverage::new(&a).unwrap(); + h.aggregate(&b).unwrap(); + + assert_eq!(h.end, t0 + Duration::milliseconds(a_ms + b_ms)); + assert_eq!(h.duration, expected_ms); + assert_eq!(h.value(), MetricValue::Number(expected)); + } + + #[rstest] + fn test_incompatible_metric_type_on_histogram() { + let timestamp = MetricTimestamp::from_str("2021-01-01T00:00:00Z").unwrap(); + + let a = MetricReading::Histogram { + value: 1.0, + timestamp, + }; + let b = MetricReading::Counter { + value: 2.0, + timestamp, + }; + + assert!(Histogram::new(&a).unwrap().aggregate(&b).is_err()); + } + + #[rstest] + #[case(INFINITY)] + #[case(NEG_INFINITY)] + #[case(NAN)] + fn test_edge_values_new(#[case] edge_value: f64) { + let timestamp = MetricTimestamp::from_str("2021-01-01T00:00:00Z").unwrap(); + let a = MetricReading::Histogram { + value: edge_value, + timestamp, + }; + assert!(Histogram::new(&a).is_err()); + } + + #[rstest] + #[case(INFINITY)] + #[case(NEG_INFINITY)] + #[case(NAN)] + fn test_edge_values_aggregate(#[case] edge_value: f64) { + let timestamp = MetricTimestamp::from_str("2021-01-01T00:00:00Z").unwrap(); + let a = MetricReading::Histogram { + value: 0.0, + timestamp, + }; + let b = MetricReading::Histogram { + value: edge_value, + timestamp, + }; + assert!(Histogram::new(&a).unwrap().aggregate(&b).is_err()); + } + + #[rstest] + #[case(1.0, 2.0, 3.0)] + fn test_counter_aggregation(#[case] a: f64, #[case] b: f64, #[case] expected: f64) { + let timestamp = MetricTimestamp::from_str("2021-01-01T00:00:00Z").unwrap(); + let timestamp2 = MetricTimestamp::from_str("2021-01-01T00:00:43Z").unwrap(); + + let a = MetricReading::Counter { + value: a, + timestamp, + }; + let b = MetricReading::Counter { + value: b, + timestamp: timestamp2, + }; + + let mut sum = Counter::new(&a).unwrap(); + sum.aggregate(&b).unwrap(); + assert_eq!(sum.end, timestamp2); + assert_eq!(sum.sum, expected); + } + + #[rstest] + #[case(1.0, 2.0, 2.0)] + #[case(10.0, 1.0, 1.0)] + fn test_gauge_aggregation(#[case] a: f64, #[case] b: f64, #[case] expected: f64) { + let timestamp = MetricTimestamp::from_str("2021-01-01T00:00:00Z").unwrap(); + let timestamp2 = MetricTimestamp::from_str("2021-01-01T00:00:43Z").unwrap(); + + let a = MetricReading::Gauge { + value: a, + timestamp, + }; + let b = MetricReading::Gauge { + value: b, + timestamp: timestamp2, + }; + + let mut gauge = Gauge::new(&a).unwrap(); + gauge.aggregate(&b).unwrap(); + assert_eq!(gauge.end, timestamp2); + assert_eq!(gauge.value, expected); + } +} diff --git a/memfaultd/src/network/client.rs b/memfaultd/src/network/client.rs new file mode 100644 index 0000000..e726e33 --- /dev/null +++ b/memfaultd/src/network/client.rs @@ -0,0 +1,238 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::fs::File; +use std::io::Read; +use std::str; + +use eyre::eyre; +use eyre::Context; +use eyre::Result; +use log::{debug, trace}; +use reqwest::blocking; +use reqwest::blocking::Body; +use reqwest::header; + +use crate::retriable_error::RetriableError; +use crate::util::io::StreamLen; +use crate::util::string::Ellipsis; + +use super::requests::DeviceConfigResponse; +use super::requests::UploadPrepareRequest; +use super::requests::UploadPrepareResponse; +use super::requests::{DeviceConfigRequest, MarUploadMetadata}; +use super::NetworkClient; +use super::NetworkConfig; + +/// Memfault Network client +pub struct NetworkClientImpl { + client: blocking::Client, + /// A separate client for upload to file storage + file_upload_client: blocking::Client, + config: NetworkConfig, +} + +#[allow(clippy::upper_case_acronyms)] +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Method { + POST, +} + +impl NetworkClientImpl { + pub fn new(config: NetworkConfig) -> Result { + let headers = [ + ( + header::ACCEPT, + header::HeaderValue::from_static("application/json"), + ), + ( + header::CONTENT_TYPE, + header::HeaderValue::from_static("application/json"), + ), + ( + header::HeaderName::from_static("memfault-project-key"), + header::HeaderValue::from_str(&config.project_key)?, + ), + ( + header::CONTENT_ENCODING, + header::HeaderValue::from_static("utf-8"), + ), + ] + .into_iter() + .collect(); + + let client = blocking::ClientBuilder::new() + .default_headers(headers) + .build()?; + + Ok(NetworkClientImpl { + client, + file_upload_client: blocking::Client::new(), + config, + }) + } + + fn good_response_or_error(response: blocking::Response) -> Result { + // Map status code to an error + let status = response.status(); + match status.as_u16() { + 200..=299 => Ok(response), + // Server errors are expected to be temporary and will be retried later + 500..=599 => Err(RetriableError::ServerError { + status_code: status.as_u16(), + } + .into()), + // Any other error (404, etc) will be considered fatal and will not be retried. + // HTTP client errors (4xx) are not expected to happen in normal operation, but can + // occur due to misconfiguration/integration issues. Log the first 1KB of the response + // body to help with debugging: + _ => { + let mut response_text = response.text().unwrap_or_else(|_| "???".into()); + // Limit the size of the response text to avoid filling up the log: + response_text.truncate_with_ellipsis(1024); + Err(eyre!( + "Unexpected server response: {} {}", + status.as_u16(), + response_text + )) + } + } + } + + /// Send a request to Memfault backend + fn fetch(&self, method: Method, endpoint: &str, payload: &str) -> Result { + let url = format!("{}{}", self.config.base_url, endpoint); + debug!( + "{:?} {} - Payload {} bytes\n{:?}", + method, + url, + payload.len(), + payload + ); + let response = self + .client + .request( + match method { + Method::POST => reqwest::Method::POST, + }, + url, + ) + .body(payload.to_owned()) + .send() + // "send(): This method fails if there was an error while sending request, redirect loop was detected or redirect limit was exhausted." + // All kinds of errors here are considered "recoverable" and will be retried. + .map_err(|e| RetriableError::NetworkError { source: e })?; + debug!( + " Response status {} - Size {:?}", + response.status(), + response.content_length(), + ); + Self::good_response_or_error(response) + } + + /// Upload a file to S3 and return the file token + fn prepare_and_upload( + &self, + file: BodyAdapter, + gzipped: bool, + ) -> Result { + let prepare_request = + UploadPrepareRequest::prepare(&self.config, file.size as usize, gzipped); + + let prepare_response = self + .fetch( + Method::POST, + "/api/v0/upload", + &serde_json::to_string(&prepare_request)?, + )? + .json::() + .wrap_err("Prepare upload error")?; + + trace!("Upload prepare response: {:?}", prepare_response); + + self.put_file( + &prepare_response.data.upload_url, + file, + if gzipped { Some("gzip") } else { None }, + ) + .wrap_err("Storage upload error")?; + debug!("Successfully transmitted file"); + + Ok(prepare_response.data.token) + } + + fn put_file( + &self, + url: &str, + file: BodyAdapter, + content_encoding: Option<&str>, + ) -> Result<()> { + let mut req = self.file_upload_client.put(url); + + if let Some(content_encoding) = content_encoding { + trace!("Adding content-encoding header"); + req = req.header(header::CONTENT_ENCODING, content_encoding); + } + + trace!("Uploading file to {}", url); + let body: Body = file.into(); + let r = req.body(body).send()?; + Self::good_response_or_error(r).and(Ok(())) + } +} + +impl NetworkClient for NetworkClientImpl { + fn upload_mar_file(&self, file: F) -> Result<()> { + let token = self.prepare_and_upload(file.into(), false)?; + + let mar_upload = MarUploadMetadata::prepare(&self.config, &token); + self.fetch( + Method::POST, + "/api/v0/upload/mar", + &serde_json::to_string(&mar_upload)?, + ) + .wrap_err("MAR Upload Error") + .and(Ok(())) + } + + fn fetch_device_config(&self) -> Result { + let request = DeviceConfigRequest::from(&self.config); + self.fetch( + Method::POST, + "/api/v0/device-config", + &serde_json::to_string(&request)?, + )? + .json::() + .wrap_err("Fetch device-config error") + } +} + +/// Small helper to adapt a Read/File into a Body. +/// Note it's not possible to directly write: impl From for Body { ... } +/// because of orphan rules. See https://doc.rust-lang.org/error_codes/E0210.html +struct BodyAdapter { + reader: R, + size: u64, +} + +impl From for BodyAdapter { + fn from(reader: R) -> Self { + let size = reader.stream_len(); + Self { reader, size } + } +} + +impl TryFrom for BodyAdapter { + type Error = std::io::Error; + + fn try_from(file: File) -> Result { + let size = file.metadata()?.len(); + Ok(Self { reader: file, size }) + } +} + +impl From> for Body { + fn from(wrapper: BodyAdapter) -> Self { + Body::sized(wrapper.reader, wrapper.size) + } +} diff --git a/memfaultd/src/network/mod.rs b/memfaultd/src/network/mod.rs new file mode 100644 index 0000000..f8a41f0 --- /dev/null +++ b/memfaultd/src/network/mod.rs @@ -0,0 +1,64 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::Read; + +use eyre::Result; +#[cfg(test)] +use mockall::automock; + +use crate::config::Config; +use crate::util::io::StreamLen; + +mod client; +pub use client::NetworkClientImpl; + +mod requests; +pub use requests::*; + +#[cfg_attr(test, automock)] +pub trait NetworkClient { + /// Upload a MAR file to Memfault + fn upload_mar_file(&self, file: F) -> Result<()>; + + /// Fetch DeviceConfig from Memfault. + fn fetch_device_config(&self) -> Result; +} + +/// Internal representation of what is needed to talk to the backend. +#[derive(Clone)] +pub struct NetworkConfig { + pub project_key: String, + pub base_url: String, + pub device_id: String, + pub hardware_version: String, + pub software_version: String, + pub software_type: String, +} + +impl From<&Config> for NetworkConfig { + fn from(config: &Config) -> Self { + NetworkConfig { + project_key: config.config_file.project_key.clone(), + device_id: config.device_info.device_id.clone(), + base_url: config.config_file.base_url.clone(), + hardware_version: config.device_info.hardware_version.clone(), + software_type: config.software_type().to_string(), + software_version: config.software_version().to_string(), + } + } +} + +#[cfg(test)] +impl NetworkConfig { + pub fn test_fixture() -> Self { + NetworkConfig { + project_key: "abcd".to_owned(), + base_url: "https://devices.memfault.com/".to_owned(), + device_id: "001".to_owned(), + hardware_version: "DVT".to_owned(), + software_version: "1.0.0".to_owned(), + software_type: "test".to_owned(), + } + } +} diff --git a/memfaultd/src/network/requests.rs b/memfaultd/src/network/requests.rs new file mode 100644 index 0000000..444c0ba --- /dev/null +++ b/memfaultd/src/network/requests.rs @@ -0,0 +1,158 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use super::NetworkConfig; +use serde::{Deserialize, Serialize}; + +/// Device metadata required to prepare and commit uploads. +#[derive(Serialize, Deserialize, Debug)] +pub struct UploadDeviceMetadata<'a> { + device_serial: &'a str, + hardware_version: &'a str, + software_version: &'a str, + software_type: &'a str, +} + +impl<'a> From<&'a NetworkConfig> for UploadDeviceMetadata<'a> { + fn from(config: &'a NetworkConfig) -> Self { + UploadDeviceMetadata { + device_serial: config.device_id.as_str(), + hardware_version: config.hardware_version.as_str(), + software_type: config.software_type.as_str(), + software_version: config.software_version.as_str(), + } + } +} + +/// Request body to prepare an upload +#[derive(Serialize, Debug)] +pub struct UploadPrepareRequest<'a> { + content_encoding: Option<&'static str>, + size: usize, + device: UploadDeviceMetadata<'a>, +} + +impl<'a> UploadPrepareRequest<'a> { + pub fn prepare( + config: &'a NetworkConfig, + filesize: usize, + gzipped: bool, + ) -> UploadPrepareRequest<'a> { + UploadPrepareRequest { + content_encoding: if gzipped { Some("gzip") } else { None }, + size: filesize, + device: UploadDeviceMetadata::from(config), + } + } +} + +/// Response for prepare-upload request +#[derive(Serialize, Deserialize, Debug)] +pub struct UploadPrepareResponse { + pub data: UploadPrepareResponseData, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct UploadPrepareResponseData { + pub upload_url: String, + pub token: String, +} + +#[derive(Serialize, Debug)] +pub struct PreparedFile<'a> { + token: &'a str, +} + +#[derive(Serialize, Debug)] +pub struct MarUploadMetadata<'a> { + device_serial: &'a str, + file: PreparedFile<'a>, + hardware_version: &'a str, + software_type: &'a str, + software_version: &'a str, +} + +impl<'a> MarUploadMetadata<'a> { + pub fn prepare(config: &'a NetworkConfig, token: &'a str) -> Self { + Self { + device_serial: &config.device_id, + hardware_version: &config.hardware_version, + software_type: &config.software_type, + software_version: &config.software_version, + file: PreparedFile { token }, + } + } +} + +#[derive(Serialize, Deserialize, Debug)] +struct DeviceConfigDeviceInfo<'a> { + device_serial: &'a str, + hardware_version: &'a str, + software_version: &'a str, + software_type: &'a str, +} + +/// Device metadata required to prepare and commit uploads. +#[derive(Serialize, Deserialize, Debug)] +pub struct DeviceConfigRequest<'a> { + #[serde(borrow)] + device: DeviceConfigDeviceInfo<'a>, +} + +impl<'a> From<&'a NetworkConfig> for DeviceConfigRequest<'a> { + fn from(config: &'a NetworkConfig) -> Self { + DeviceConfigRequest { + device: DeviceConfigDeviceInfo { + device_serial: config.device_id.as_str(), + hardware_version: config.hardware_version.as_str(), + software_type: config.software_type.as_str(), + software_version: config.software_version.as_str(), + }, + } + } +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct DeviceConfigResponse { + pub data: DeviceConfigResponseData, +} + +pub type DeviceConfigRevision = u32; + +#[derive(Serialize, Deserialize, Debug)] +pub struct DeviceConfigResponseData { + pub config: DeviceConfigResponseConfig, + pub revision: DeviceConfigRevision, + pub completed: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct DeviceConfigResponseConfig { + pub memfault: DeviceConfigResponseMemfault, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct DeviceConfigResponseMemfault { + pub sampling: DeviceConfigResponseSampling, +} +#[derive(Serialize, Deserialize, Debug)] +pub struct DeviceConfigResponseSampling { + #[serde(rename = "debugging.resolution")] + pub debugging_resolution: DeviceConfigResponseResolution, + #[serde(rename = "logging.resolution")] + pub logging_resolution: DeviceConfigResponseResolution, + #[serde(rename = "monitoring.resolution")] + pub monitoring_resolution: DeviceConfigResponseResolution, +} + +#[derive(Serialize, Deserialize, Debug)] +pub enum DeviceConfigResponseResolution { + #[serde(rename = "off")] + Off, + #[serde(rename = "low")] + Low, + #[serde(rename = "normal")] + Normal, + #[serde(rename = "high")] + High, +} diff --git a/memfaultd/src/reboot/mod.rs b/memfaultd/src/reboot/mod.rs new file mode 100644 index 0000000..c0743fb --- /dev/null +++ b/memfaultd/src/reboot/mod.rs @@ -0,0 +1,465 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +mod reason; +pub use reason::RebootReason; +mod reason_codes; +pub use reason_codes::RebootReasonCode; + +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::{fs::write, fs::File, process::Command}; + +use eyre::{eyre, Context, Result}; +use log::{debug, error, info, warn}; +use uuid::Uuid; + +use crate::util::system::read_system_boot_id; +use crate::{config::Config, service_manager::ServiceManagerStatus}; +use crate::{mar::MarEntryBuilder, network::NetworkConfig}; +use crate::{mar::Metadata, service_manager::MemfaultdServiceManager}; + +const PSTORE_DIR: &str = "/sys/fs/pstore"; + +/// Manages reboot reasons and writes them to the MAR file if untracked. +/// +/// This tracker is responsible for tracking the boot_id and reboot reason +/// across reboots. It will write the reboot reason to the MAR file if it +/// has not been tracked yet. +pub struct RebootReasonTracker<'a> { + config: &'a Config, + sources: Vec, + service_manager: &'a dyn MemfaultdServiceManager, +} + +impl<'a> RebootReasonTracker<'a> { + pub fn new(config: &'a Config, service_manager: &'a dyn MemfaultdServiceManager) -> Self { + let sources = vec![ + RebootReasonSource { + name: "pstore", + func: read_reboot_reason_and_clear_file_pstore, + }, + RebootReasonSource { + name: "custom", + func: read_reboot_reason_and_clear_file_customer, + }, + RebootReasonSource { + name: "internal", + func: read_reboot_reason_and_clear_file_internal, + }, + ]; + + Self { + config, + sources, + service_manager, + } + } + + pub fn track_reboot(&self) -> Result<()> { + let boot_id = read_system_boot_id()?; + + if !self.config.config_file.enable_data_collection { + // Clear boot id since we haven't enabled data collection yet. + self.check_boot_id_is_tracked(&boot_id); + + process_pstore_files(PSTORE_DIR); + + return Ok(()); + } + + if !self.check_boot_id_is_tracked(&boot_id) { + let reboot_reason = self.resolve_reboot_reason(&boot_id)?; + + let mar_builder = MarEntryBuilder::new(&self.config.mar_staging_path())? + .set_metadata(Metadata::new_reboot(reboot_reason)); + + let network_config = NetworkConfig::from(self.config); + mar_builder.save(&network_config)?; + } + + Ok(()) + } + + fn check_boot_id_is_tracked(&self, boot_id: &Uuid) -> bool { + let tmp_filename = self + .config + .config_file + .generate_tmp_filename("last_tracked_boot_id"); + + let last_boot_id = std::fs::read_to_string(&tmp_filename) + .ok() + .and_then(|boot_id| Uuid::from_str(boot_id.trim()).ok()); + if last_boot_id.is_none() { + warn!("No last tracked boot_id found"); + } + + if let Err(e) = std::fs::write(tmp_filename, boot_id.to_string()) { + error!("Failed to write last tracked boot_id: {}", e); + } + + match last_boot_id { + Some(last_boot_id) => { + let is_tracked = &last_boot_id == boot_id; + if is_tracked { + info!("boot_id already tracked {}!", boot_id); + } + + is_tracked + } + None => false, + } + } + + fn resolve_reboot_reason(&self, boot_id: &Uuid) -> Result { + let mut reboot_reason = None; + for reason_source in &self.sources { + if let Some(new_reboot_reason) = (reason_source.func)(self.config) { + if reboot_reason.is_some() { + info!( + "Discarded reboot reason {:?} from {} source for boot_id {}", + new_reboot_reason, reason_source.name, boot_id + ); + } else { + info!( + "Using reboot reason {:?} from {} source for boot_id {}", + new_reboot_reason, reason_source.name, boot_id + ); + reboot_reason = Some(new_reboot_reason); + } + } + } + + Ok(reboot_reason.unwrap_or_else(|| RebootReason::from(RebootReasonCode::Unknown))) + } +} + +impl<'a> Drop for RebootReasonTracker<'a> { + fn drop(&mut self) { + let status = self.service_manager.service_manager_status(); + + match status { + Ok(ServiceManagerStatus::Stopping) => { + // Only write the user reset reason if the service manager is stopping + let reboot_file = self + .config + .config_file + .generate_persist_filename("lastrebootreason"); + + // Helper function to combine errors and handle them easier + fn inner_write(mut file: File) -> Result<()> { + // Ensure file is written as the process could exit before it is done. + let reason_int = RebootReasonCode::UserReset as u32; + + file.write_all(reason_int.to_string().as_bytes())?; + file.sync_all()?; + + Ok(()) + } + + info!("Writing reboot reason to {:?}", reboot_file); + match File::create(reboot_file) { + Ok(file) => { + if let Err(e) = inner_write(file) { + error!("Failed to write reboot reason: {}", e); + } + } + Err(e) => { + error!("Failed to create reboot reason file: {}", e); + } + } + } + Ok(status) => { + debug!( + "Service manager in state {:?} while closing. Not writing reboot reason", + status + ); + } + Err(e) => error!("Failed to get service manager status: {}", e), + } + } +} + +struct RebootReasonSource { + name: &'static str, + func: fn(&Config) -> Option, +} + +const PSTORE_DMESG_FILE: &str = "/sys/fs/pstore/dmesg-ramoops-0"; + +fn read_reboot_reason_and_clear_file_pstore(_config: &Config) -> Option { + if Path::new(PSTORE_DMESG_FILE).exists() { + process_pstore_files(PSTORE_DIR); + + Some(RebootReason::from(RebootReasonCode::KernelPanic)) + } else { + None + } +} + +fn read_reboot_reason_and_clear_file_internal(config: &Config) -> Option { + let reboot_file = config + .config_file + .generate_persist_filename("lastrebootreason"); + + read_reboot_reason_and_clear_file(&reboot_file) +} + +fn read_reboot_reason_and_clear_file_customer(config: &Config) -> Option { + let file_name = &config.config_file.reboot.last_reboot_reason_file; + + read_reboot_reason_and_clear_file(file_name) +} + +fn read_reboot_reason_and_clear_file(file_name: &PathBuf) -> Option { + let reboot_reason_string = match std::fs::read_to_string(file_name) { + Ok(reboot_reason) => reboot_reason, + Err(e) => { + debug!("Failed to open {:?}: {}", file_name, e.kind()); + return None; + } + }; + + let reboot_reason = RebootReason::from_str(reboot_reason_string.trim()).unwrap_or_else(|_| { + warn!("Couldn't parse reboot reason: {}", reboot_reason_string); + RebootReason::Code(RebootReasonCode::Unknown) + }); + + if let Err(e) = std::fs::remove_file(file_name) { + error!("Failed to remove {:?}: {}", file_name, e.kind()); + } + + Some(reboot_reason) +} + +fn process_pstore_files(pstore_dir: &str) { + // TODO: MFLT-7805 Process last kmsg/console logs + debug!("Cleaning up pstore..."); + + fn inner_process_pstore(pstore_dir: &str) -> Result<()> { + for entry in std::fs::read_dir(pstore_dir)? { + let path = entry?.path(); + + if path.is_file() || path.is_symlink() { + debug!("Cleaning pstore - Removing {}...", path.display()); + std::fs::remove_file(path)?; + } + } + + Ok(()) + } + + if let Err(e) = inner_process_pstore(pstore_dir) { + error!("Failed to process pstore files: {}", e); + } +} + +pub fn write_reboot_reason_and_reboot( + last_reboot_reason_file: &Path, + reason: RebootReason, +) -> Result<()> { + write(last_reboot_reason_file, format!("{}", reason)).wrap_err_with(|| { + format!( + "Unable to write reboot reason (path: {}).", + last_reboot_reason_file.display() + ) + })?; + if !Command::new("reboot").status()?.success() { + return Err(eyre!("Failed to reboot")); + } + Ok(()) +} + +#[cfg(test)] +mod test { + use super::*; + + use crate::mar::manifest::{Manifest, Metadata}; + use crate::service_manager::{MockMemfaultdServiceManager, ServiceManagerStatus}; + use crate::test_utils::setup_logger; + use crate::util::path::AbsolutePath; + + use rstest::rstest; + use tempfile::tempdir; + + impl<'a> RebootReasonTracker<'a> { + fn new_with_sources( + config: &'a Config, + sources: Vec, + service_manager: &'a dyn MemfaultdServiceManager, + ) -> Self { + Self { + config, + sources, + service_manager, + } + } + } + + const TEST_BOOT_ID: &str = "32c45579-8881-4a43-b7d1-f1df8invalid"; + + #[rstest] + fn test_reboot_reason_source_ordering(_setup_logger: ()) { + let mut config = Config::test_fixture(); + config.config_file.enable_data_collection = true; + + let persist_dir = tempdir().unwrap(); + config.config_file.persist_dir = + AbsolutePath::try_from(persist_dir.path().to_path_buf()).unwrap(); + + let main_source = RebootReasonSource { + name: "main", + func: |_: &Config| Some(RebootReason::from(RebootReasonCode::HardFault)), + }; + let secondary_source = RebootReasonSource { + name: "secondary", + func: |_: &Config| Some(RebootReason::from(RebootReasonCode::UserReset)), + }; + + let mut service_manager = MockMemfaultdServiceManager::new(); + service_manager + .expect_service_manager_status() + .once() + .returning(|| Ok(ServiceManagerStatus::Stopping)); + + let mar_staging_path = config.mar_staging_path(); + std::fs::create_dir_all(&mar_staging_path).expect("Failed to create mar staging dir"); + + let tracker = RebootReasonTracker::new_with_sources( + &config, + vec![main_source, secondary_source], + &service_manager, + ); + tracker + .track_reboot() + .expect("Failed to init reboot tracker"); + + // Verify that the first reboot reason source is used + verify_mar_reboot_reason( + RebootReason::from(RebootReasonCode::HardFault), + &mar_staging_path, + ); + } + + #[rstest] + fn test_reboot_reason_parsing(_setup_logger: ()) { + let mut config = Config::test_fixture(); + config.config_file.enable_data_collection = true; + + let persist_dir = tempdir().unwrap(); + config.config_file.persist_dir = + AbsolutePath::try_from(persist_dir.path().to_path_buf()).unwrap(); + + let reboot_reason = RebootReason::from(RebootReasonCode::HardFault); + + // Write values to last boot id and last reboot reason files + let last_reboot_file = persist_dir.path().join("lastrebootreason"); + std::fs::write(&last_reboot_file, reboot_reason.to_string()) + .expect("Failed to write last reboot file"); + let last_boot_id_file = persist_dir.path().join("last_tracked_boot_id"); + std::fs::write(last_boot_id_file, TEST_BOOT_ID).expect("Failed to write last boot id file"); + + let source = RebootReasonSource { + name: "test", + func: read_reboot_reason_and_clear_file_internal, + }; + + let mut service_manager = MockMemfaultdServiceManager::new(); + service_manager + .expect_service_manager_status() + .once() + .returning(|| Ok(ServiceManagerStatus::Stopping)); + + let mar_staging_path = config.mar_staging_path(); + + // Create mar staging dir + std::fs::create_dir_all(&mar_staging_path).expect("Failed to create mar staging dir"); + + let tracker = + RebootReasonTracker::new_with_sources(&config, vec![source], &service_manager); + tracker + .track_reboot() + .expect("Failed to init reboot tracker"); + + verify_mar_reboot_reason(reboot_reason, &mar_staging_path); + + // Drop tracker and ensure new reboot reason is written to file + drop(tracker); + let reboot_reason = std::fs::read_to_string(&last_reboot_file) + .expect("Failed to read last reboot file") + .parse::() + .expect("Failed to parse reboot reason"); + let reboot_reason = RebootReason::from( + RebootReasonCode::from_repr(reboot_reason).expect("Invalid reboot reason"), + ); + + assert_eq!( + reboot_reason, + RebootReason::from(RebootReasonCode::UserReset) + ); + } + + #[rstest] + fn test_custom_reboot_reason_parsing(_setup_logger: ()) { + let mut config = Config::test_fixture(); + config.config_file.enable_data_collection = true; + + let persist_dir = tempdir().unwrap(); + config.config_file.persist_dir = + AbsolutePath::try_from(persist_dir.path().to_path_buf()).unwrap(); + + let reboot_reason = RebootReason::from_str("CustomRebootReason").unwrap(); + + // Write values to last boot id and last reboot reason files + let last_reboot_file = persist_dir.path().join("lastrebootreason"); + std::fs::write(last_reboot_file, "CustomRebootReason") + .expect("Failed to write last reboot file"); + let last_boot_id_file = persist_dir.path().join("last_tracked_boot_id"); + std::fs::write(last_boot_id_file, TEST_BOOT_ID).expect("Failed to write last boot id file"); + + let source = RebootReasonSource { + name: "test", + func: read_reboot_reason_and_clear_file_internal, + }; + + let mut service_manager = MockMemfaultdServiceManager::new(); + service_manager + .expect_service_manager_status() + .once() + .returning(|| Ok(ServiceManagerStatus::Stopping)); + + let mar_staging_path = config.mar_staging_path(); + + // Create mar staging dir + std::fs::create_dir_all(&mar_staging_path).expect("Failed to create mar staging dir"); + + let tracker = + RebootReasonTracker::new_with_sources(&config, vec![source], &service_manager); + tracker + .track_reboot() + .expect("Failed to init reboot tracker"); + + verify_mar_reboot_reason(reboot_reason, &mar_staging_path); + } + + fn verify_mar_reboot_reason(reboot_reason: RebootReason, mar_staging_path: &Path) { + let mar_dir = std::fs::read_dir(mar_staging_path) + .expect("Failed to read temp dir") + .filter_map(|entry| entry.ok()) + .collect::>(); + + // There should only be an entry for the reboot reason + assert_eq!(mar_dir.len(), 1); + + let mar_manifest = mar_dir[0].path().join("manifest.json"); + let manifest_string = std::fs::read_to_string(mar_manifest).unwrap(); + let manifest: Manifest = serde_json::from_str(&manifest_string).unwrap(); + + if let Metadata::LinuxReboot { reason, .. } = manifest.metadata { + assert_eq!(reboot_reason, reason); + } else { + panic!("Unexpected metadata type"); + } + } +} diff --git a/memfaultd/src/reboot/reason.rs b/memfaultd/src/reboot/reason.rs new file mode 100644 index 0000000..3e904c4 --- /dev/null +++ b/memfaultd/src/reboot/reason.rs @@ -0,0 +1,110 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::fmt::{Debug, Display, Formatter, Result as FmtResult}; +use std::str::FromStr; + +use eyre::{eyre, ErrReport, Result}; +use serde::{Deserialize, Serialize}; + +use crate::{ + reboot::reason_codes::RebootReasonCode, util::patterns::alphanum_slug_dots_colon_is_valid, +}; + +#[derive(Clone, PartialEq, Eq, Deserialize, Serialize)] +#[serde(untagged)] +pub enum RebootReason { + Code(RebootReasonCode), + Custom(RebootReasonString), +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +pub struct RebootReasonString { + unexpected: bool, + name: String, +} + +impl From for RebootReason { + fn from(code: RebootReasonCode) -> RebootReason { + RebootReason::Code(code) + } +} + +impl FromStr for RebootReasonString { + type Err = ErrReport; + fn from_str(s: &str) -> Result { + // Leading '!' indicates an unexpected reboot reason + if let Some(stripped) = s.strip_prefix('!') { + if stripped.is_empty() { + Err(eyre!("\"!\" on its own is not a valid reboot reaosn!")) + } else { + match alphanum_slug_dots_colon_is_valid(stripped, 64) { + Ok(()) => Ok(RebootReasonString { + unexpected: true, + name: stripped.to_string(), + }), + Err(e) => Err(e), + } + } + } else { + match alphanum_slug_dots_colon_is_valid(s, 64) { + Ok(()) => Ok(RebootReasonString { + unexpected: false, + name: s.to_string(), + }), + Err(e) => Err(e), + } + } + } +} + +impl FromStr for RebootReason { + type Err = ErrReport; + + fn from_str(s: &str) -> Result { + match u32::from_str(s) { + Ok(code) => match RebootReasonCode::from_repr(code) { + Some(reset_code) => Ok(RebootReason::Code(reset_code)), + None => Ok(RebootReason::Code(RebootReasonCode::Unknown)), + }, + // If the reboot reason isn't parse-able to a u32, it's custom + Err(_) => match RebootReasonString::from_str(s) { + Ok(reason) => Ok(RebootReason::Custom(reason)), + Err(e) => Err(eyre!("Failed to parse custom reboot reason: {}", e)), + }, + } + } +} + +impl Display for RebootReason { + fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { + match self { + RebootReason::Code(c) => write!(f, "{}", (*c as u32)), + RebootReason::Custom(RebootReasonString { unexpected, name }) => { + if *unexpected { + write!(f, "!{}", name) + } else { + write!(f, "{}", name) + } + } + } + } +} + +impl Debug for RebootReason { + fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { + match self { + RebootReason::Code(c) => write!(f, "{} ({})", (*c as u32), c), + RebootReason::Custom(RebootReasonString { unexpected, name }) => write!( + f, + "{} ({})", + name, + if *unexpected { + "unexpected reboot" + } else { + "expected reboot" + } + ), + } + } +} diff --git a/memfaultd/src/reboot/reason_codes.rs b/memfaultd/src/reboot/reason_codes.rs new file mode 100644 index 0000000..c5c14f6 --- /dev/null +++ b/memfaultd/src/reboot/reason_codes.rs @@ -0,0 +1,85 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use serde_repr::{Deserialize_repr, Serialize_repr}; +use strum_macros::{Display, EnumString, FromRepr}; + +/// Definitions for reboot reasons +/// +/// See the [Memfault docs](https://docs.memfault.com/docs/platform/reference-reboot-reason-ids/) +/// for more details. +#[derive( + Debug, + Clone, + Copy, + EnumString, + Display, + Serialize_repr, + Deserialize_repr, + PartialEq, + Eq, + FromRepr, +)] +#[repr(u32)] +pub enum RebootReasonCode { + Unknown = 0x0000, + + // + // Normal Resets + // + UserShutdown = 0x0001, + UserReset = 0x0002, + FirmwareUpdate = 0x0003, + LowPower = 0x0004, + DebuggerHalted = 0x0005, + ButtonReset = 0x0006, + PowerOnReset = 0x0007, + SoftwareReset = 0x0008, + + /// MCU went through a full reboot due to exit from lowest power state + DeepSleep = 0x0009, + /// MCU reset pin was toggled + PinReset = 0x000A, + + // + // Error Resets + // + /// Can be used to flag an unexpected reset path. i.e NVIC_SystemReset() being called without any + /// reboot logic getting invoked. + UnknownError = 0x8000, + Assert = 0x8001, + + /// Deprecated in favor of HardwareWatchdog & SoftwareWatchdog. + /// + /// This way, the amount of watchdogs not caught by software can be easily tracked. + WatchdogDeprecated = 0x8002, + + BrownOutReset = 0x8003, + Nmi = 0x8004, // Non-Maskable Interrupt + + // More details about nomenclature in https://mflt.io/root-cause-watchdogs + HardwareWatchdog = 0x8005, + SoftwareWatchdog = 0x8006, + + /// A reset triggered due to the MCU losing a stable clock. + /// + /// This can happen, for example, if power to the clock is cut or the lock for the PLL is lost. + ClockFailure = 0x8007, + + /// A software reset triggered when the OS or RTOS end-user code is running on top of identifies + /// a fatal error condition. + KernelPanic = 0x8008, + + /// A reset triggered when an attempt to upgrade to a new OTA image has failed and a rollback + /// to a previous version was initiated + FirmwareUpdateError = 0x8009, + + // Resets from Arm Faults + BusFault = 0x9100, + MemFault = 0x9200, + UsageFault = 0x9300, + HardFault = 0x9400, + /// A reset which is triggered when the processor faults while already + /// executing from a fault handler. + Lockup = 0x9401, +} diff --git a/memfaultd/src/retriable_error.rs b/memfaultd/src/retriable_error.rs new file mode 100644 index 0000000..547698f --- /dev/null +++ b/memfaultd/src/retriable_error.rs @@ -0,0 +1,65 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::{ErrReport, Report, Result}; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum RetriableError { + #[error("Temporary server error: {status_code}")] + ServerError { status_code: u16 }, + #[error("Network error ({source})")] + NetworkError { source: reqwest::Error }, +} + +pub trait IgnoreNonRetriableError { + /// Ignore non-retriable errors, turning them into `Ok(None)`. + /// If the Err holds a RetriableError, it will be returned as-is. + fn ignore_non_retriable_errors_with(self, x: R) -> Result<(), ErrReport>; +} + +impl IgnoreNonRetriableError for Result { + fn ignore_non_retriable_errors_with( + self, + mut on_error: R, + ) -> Result<(), ErrReport> { + match self { + Ok(_) => Ok(()), + Err(e) => { + if e.downcast_ref::().is_some() { + Err(e) + } else { + on_error(&e); + Ok(()) + } + } + } + } +} + +#[cfg(test)] +mod tests { + use eyre::eyre; + use rstest::*; + + use super::*; + + #[rstest] + #[case(Ok(()), false, true)] + #[case(Err(eyre!("Some error")), true, true)] + #[case(Err(eyre!(RetriableError::ServerError { status_code: 503 })), false, false)] + fn test_ignore_non_retriable_errors_with( + #[case] result: Result<(), Report>, + #[case] expected_called: bool, + #[case] expected_ok: bool, + ) { + let mut called = false; + assert_eq!( + result + .ignore_non_retriable_errors_with(|_| called = true) + .is_ok(), + expected_ok + ); + assert_eq!(called, expected_called); + } +} diff --git a/memfaultd/src/service_manager/default.rs b/memfaultd/src/service_manager/default.rs new file mode 100644 index 0000000..3dec942 --- /dev/null +++ b/memfaultd/src/service_manager/default.rs @@ -0,0 +1,20 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use nix::sys::signal::Signal::SIGHUP; + +use crate::util::pid_file::send_signal_to_pid; + +use super::MemfaultdServiceManager; + +pub struct DefaultServiceManager; + +impl MemfaultdServiceManager for DefaultServiceManager { + fn restart_memfaultd_if_running(&self) -> eyre::Result<()> { + send_signal_to_pid(SIGHUP) + } + + fn service_manager_status(&self) -> eyre::Result { + Ok(super::ServiceManagerStatus::Unknown) + } +} diff --git a/memfaultd/src/service_manager/mod.rs b/memfaultd/src/service_manager/mod.rs new file mode 100644 index 0000000..216cdbe --- /dev/null +++ b/memfaultd/src/service_manager/mod.rs @@ -0,0 +1,65 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Memfaultd service management +//! +//! This module contains the trait for managing memfaultd services, as well as +//! the implementation for systemd. +//! + +mod default; +#[cfg(feature = "systemd")] +mod systemd; + +/// Return the system manager that was configured at build time. +pub fn get_service_manager() -> impl MemfaultdServiceManager { + #[cfg(feature = "systemd")] + { + use systemd::SystemdServiceManager; + SystemdServiceManager + } + #[cfg(not(feature = "systemd"))] + { + use default::DefaultServiceManager; + DefaultServiceManager + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ServiceManagerStatus { + Starting, + Running, + Stopping, + Stopped, + Unknown, +} + +/// Trait for managing memfaultd services +/// +/// This trait is implemented for different service managers, such as systemd. +#[cfg_attr(test, mockall::automock)] +pub trait MemfaultdServiceManager { + fn restart_memfaultd_if_running(&self) -> eyre::Result<()>; + fn service_manager_status(&self) -> eyre::Result; +} + +impl TryFrom<&str> for ServiceManagerStatus { + type Error = eyre::Error; + + fn try_from(status: &str) -> Result { + let status = match status { + "starting" => ServiceManagerStatus::Starting, + "running" => ServiceManagerStatus::Running, + "stopping" => ServiceManagerStatus::Stopping, + "stopped" => ServiceManagerStatus::Stopped, + _ => { + return Err(eyre::eyre!( + "Unknown systemd service manager status: {}", + status + )) + } + }; + + Ok(status) + } +} diff --git a/memfaultd/src/service_manager/systemd.rs b/memfaultd/src/service_manager/systemd.rs new file mode 100644 index 0000000..9f640e7 --- /dev/null +++ b/memfaultd/src/service_manager/systemd.rs @@ -0,0 +1,42 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::ffi::{CStr, CString}; + +use crate::service_manager::{MemfaultdServiceManager, ServiceManagerStatus}; +use memfaultc_sys::systemd::{ + memfaultd_get_systemd_bus_state, memfaultd_restart_systemd_service_if_running, +}; + +/// Systemd service manager +/// +/// This service manager uses the systemd D-Bus API to manage services. +pub struct SystemdServiceManager; + +impl MemfaultdServiceManager for SystemdServiceManager { + fn restart_memfaultd_if_running(&self) -> eyre::Result<()> { + let service_cstring = CString::new("memfaultd.service")?; + let restart_result = + unsafe { memfaultd_restart_systemd_service_if_running(service_cstring.as_ptr()) }; + + if restart_result { + Ok(()) + } else { + Err(eyre::eyre!("Failed to restart memfaultd service")) + } + } + + fn service_manager_status(&self) -> eyre::Result { + let status_ptr = unsafe { memfaultd_get_systemd_bus_state() }; + if status_ptr.is_null() { + return Err(eyre::eyre!("Failed to get systemd service bus state")); + } + + let status_str = unsafe { CStr::from_ptr(status_ptr).to_str()? }; + let status = ServiceManagerStatus::try_from(status_str)?; + + unsafe { libc::free(status_ptr as *mut libc::c_void) }; + + Ok(status) + } +} diff --git a/memfaultd/src/swupdate/config.rs b/memfaultd/src/swupdate/config.rs new file mode 100644 index 0000000..36b95cb --- /dev/null +++ b/memfaultd/src/swupdate/config.rs @@ -0,0 +1,51 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::{eyre, Result}; +use std::{ffi::CString, os::unix::prelude::OsStrExt}; + +use memfaultc_sys::swupdate::{memfault_swupdate_generate_config, MemfaultSwupdateCtx}; + +use crate::config::Config; + +pub fn generate_swupdate_config(config: &Config) -> Result<()> { + let base_url_cstring = CString::new(config.config_file.base_url.clone())?; + let software_version_cstring = CString::new(config.software_version())?; + let software_type_cstring = CString::new(config.software_type())?; + let hardware_version_cstring = CString::new(config.device_info.hardware_version.clone())?; + let device_id_cstring = CString::new(config.device_info.device_id.clone())?; + let project_key_cstring = CString::new(config.config_file.project_key.clone())?; + + let input_file_cstring = CString::new( + config + .config_file + .swupdate + .input_file + .as_os_str() + .as_bytes(), + )?; + let output_file_cstring = CString::new( + config + .config_file + .swupdate + .output_file + .as_os_str() + .as_bytes(), + )?; + + let ctx = MemfaultSwupdateCtx { + base_url: base_url_cstring.as_ptr(), + software_version: software_version_cstring.as_ptr(), + software_type: software_type_cstring.as_ptr(), + hardware_version: hardware_version_cstring.as_ptr(), + device_id: device_id_cstring.as_ptr(), + project_key: project_key_cstring.as_ptr(), + + input_file: input_file_cstring.as_ptr(), + output_file: output_file_cstring.as_ptr(), + }; + match unsafe { memfault_swupdate_generate_config(&ctx) } { + true => Ok(()), + false => Err(eyre!("Unable to prepare swupdate config.")), + } +} diff --git a/memfaultd/src/swupdate/mod.rs b/memfaultd/src/swupdate/mod.rs new file mode 100644 index 0000000..7c26353 --- /dev/null +++ b/memfaultd/src/swupdate/mod.rs @@ -0,0 +1,6 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +mod config; + +pub use config::generate_swupdate_config; diff --git a/memfaultd/src/test_utils.rs b/memfaultd/src/test_utils.rs new file mode 100644 index 0000000..13fbcf0 --- /dev/null +++ b/memfaultd/src/test_utils.rs @@ -0,0 +1,131 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Memfault Test Utils +//! +//! A collection of useful structs and functions for unit and integration testing. +//! + +use std::cmp::min; +use std::path::Path; +use std::str::FromStr; +use std::{ + fs::File, + io::{ErrorKind, Seek, Write}, +}; + +use rstest::fixture; + +mod test_instant; +pub use test_instant::*; + +mod test_connection_checker; +pub use test_connection_checker::*; + +use crate::metrics::{KeyedMetricReading, MetricReading, MetricStringKey, MetricTimestamp}; + +/// A file that will trigger write errors when it reaches a certain size. +/// Note that we currently enforce the limit on the total number of bytes +/// written, regardless of where they were written. We do implement Seek but do +/// not try to keep track of the file size. +pub struct SizeLimitedFile { + file: File, + limit: usize, + written: usize, +} + +impl SizeLimitedFile { + /// Create a new SizeLimitedFile which will write to file until limit is + /// reached. + #[allow(dead_code)] + pub fn new(file: File, limit: usize) -> Self { + Self { + file, + limit, + written: 0, + } + } +} + +impl Write for SizeLimitedFile { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let bytes_to_write = buf.len().min(self.limit - self.written); + + if bytes_to_write == 0 { + Err(std::io::Error::new( + ErrorKind::WriteZero, + "File size limit reached", + )) + } else { + self.written += bytes_to_write; + self.file.write(&buf[..bytes_to_write]) + } + } + fn flush(&mut self) -> std::io::Result<()> { + self.file.flush() + } +} + +impl Seek for SizeLimitedFile { + fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { + self.file.seek(pos) + } +} + +pub fn create_file_with_size(path: &Path, size: u64) -> std::io::Result<()> { + let mut file = File::create(path)?; + let buffer = vec![0; min(4096, size as usize)]; + let mut remaining = size; + while remaining > 0 { + let bytes_to_write = min(remaining, buffer.len() as u64); + file.write_all(&buffer[..bytes_to_write as usize])?; + remaining -= bytes_to_write; + } + Ok(()) +} + +pub fn create_file_with_contents(path: &Path, contents: &[u8]) -> std::io::Result<()> { + let mut file = File::create(path)?; + file.write_all(contents)?; + Ok(()) +} + +#[fixture] +/// Simple fixture to add to a test when you want the logger to work. +pub fn setup_logger() { + let _ = stderrlog::new().module("memfaultd").verbosity(10).init(); +} + +/// Macro to set the snapshot suffix for a test. +/// +/// This is a workaround suggested by the insta docs to scope the snapshot suffix +/// to a test when using rstest cases. See docs +/// [here](https://insta.rs/docs/patterns/). +macro_rules! set_snapshot_suffix { + ($($expr:expr),*) => { + let mut settings = insta::Settings::clone_current(); + settings.set_snapshot_suffix(format!($($expr,)*)); + let _guard = settings.bind_to_scope(); + } +} + +#[cfg(test)] +/// Constructs an iterator of Gauge metric readings for tests +/// to easily generate mock data +pub fn in_histograms( + metrics: Vec<(&'static str, f64)>, +) -> impl Iterator { + metrics + .into_iter() + .enumerate() + .map(|(i, (name, value))| KeyedMetricReading { + name: MetricStringKey::from_str(name).unwrap(), + value: MetricReading::Histogram { + value, + timestamp: MetricTimestamp::from_str("2021-01-01T00:00:00Z").unwrap() + + chrono::Duration::seconds(i as i64), + }, + }) +} + +pub(crate) use set_snapshot_suffix; diff --git a/memfaultd/src/test_utils/test_connection_checker.rs b/memfaultd/src/test_utils/test_connection_checker.rs new file mode 100644 index 0000000..06ccdbe --- /dev/null +++ b/memfaultd/src/test_utils/test_connection_checker.rs @@ -0,0 +1,41 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{cell::RefCell, net::IpAddr, time::Duration}; + +use eyre::{eyre, Result}; + +use crate::util::can_connect::CanConnect; + +#[derive(Debug, Clone, Copy)] +pub struct TestConnectionChecker {} + +thread_local! { + static CONNECTED: RefCell = RefCell::new(true); +} + +impl TestConnectionChecker { + pub fn connect() { + CONNECTED.with(|c| *c.borrow_mut() = true); + } + + pub fn disconnect() { + CONNECTED.with(|c| *c.borrow_mut() = false); + } +} + +impl CanConnect for TestConnectionChecker { + fn new(_timeout: Duration) -> Self { + Self {} + } + + fn can_connect(&self, _ip: &IpAddr, _port: u16) -> Result<()> { + let mut connected_status = true; + CONNECTED.with(|c| connected_status = *c.borrow()); + if connected_status { + Ok(()) + } else { + Err(eyre!("MockPinger is set to disconnected")) + } + } +} diff --git a/memfaultd/src/test_utils/test_instant.rs b/memfaultd/src/test_utils/test_instant.rs new file mode 100644 index 0000000..a5ac154 --- /dev/null +++ b/memfaultd/src/test_utils/test_instant.rs @@ -0,0 +1,82 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{cell::RefCell, ops::Add, ops::Sub, time::Duration}; + +use crate::util::time_measure::TimeMeasure; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct TestInstant { + t0: Duration, +} + +thread_local! { + static TIME: RefCell = RefCell::new(Duration::from_secs(0)); +} + +impl TestInstant { + pub fn now() -> Self { + let mut now = Duration::from_secs(0); + TIME.with(|t| now = *t.borrow()); + TestInstant { t0: now } + } + + pub fn from(d: Duration) -> Self { + TestInstant { t0: d } + } + + pub fn sleep(d: Duration) { + TIME.with(|t| { + let new_time = t.borrow().saturating_add(d); + *t.borrow_mut() = new_time; + }) + } +} + +impl TimeMeasure for TestInstant { + fn now() -> Self { + Self::now() + } + fn elapsed(&self) -> Duration { + Self::now().t0 - self.t0 + } + + fn since(&self, other: &Self) -> Duration { + self.t0.sub(other.t0) + } +} + +impl Add for TestInstant { + type Output = TestInstant; + + fn add(self, rhs: Duration) -> Self::Output { + TestInstant { + t0: self.t0.add(rhs), + } + } +} +impl Sub for TestInstant { + type Output = TestInstant; + + fn sub(self, rhs: Duration) -> Self::Output { + TestInstant { + t0: self.t0.sub(rhs), + } + } +} + +impl Add for TestInstant { + type Output = Duration; + + fn add(self, rhs: TestInstant) -> Duration { + self.t0.add(rhs.t0) + } +} + +impl Sub for TestInstant { + type Output = Duration; + + fn sub(self, rhs: TestInstant) -> Duration { + self.t0.sub(rhs.t0) + } +} diff --git a/memfaultd/src/util/can_connect.rs b/memfaultd/src/util/can_connect.rs new file mode 100644 index 0000000..321a5c1 --- /dev/null +++ b/memfaultd/src/util/can_connect.rs @@ -0,0 +1,78 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + net::{IpAddr, Shutdown, SocketAddr, TcpStream}, + time::Duration, +}; + +use eyre::Result; + +pub trait CanConnect { + fn new(timeout: Duration) -> Self; + fn can_connect(&self, ip_addr: &IpAddr, port: u16) -> Result<()>; +} + +pub struct TcpConnectionChecker { + timeout: Duration, +} + +impl CanConnect for TcpConnectionChecker { + fn new(timeout: Duration) -> Self { + Self { timeout } + } + + fn can_connect(&self, ip_addr: &IpAddr, port: u16) -> Result<()> { + let socket = SocketAddr::new(*ip_addr, port); + let stream = TcpStream::connect_timeout(&socket, self.timeout)?; + stream.shutdown(Shutdown::Both)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::{ + net::{IpAddr, TcpListener}, + str::FromStr, + time::Duration, + }; + + use rstest::rstest; + + use super::{CanConnect, TcpConnectionChecker}; + + #[rstest] + fn test_localhost_reachable() { + let ip_str = "127.0.0.1"; + let port = 19443; + + // Need a TCP listener to be running on the port + // TcpConnectionChecker will try to create a connection with + let listener = TcpListener::bind(format!("{}:{}", ip_str, port)) + .expect("Could not create TcpListener for testing!"); + + let connection_checker = TcpConnectionChecker::new(Duration::from_secs(10)); + assert!(connection_checker + .can_connect( + &IpAddr::from_str(ip_str) + .unwrap_or_else(|_| panic!("{} should be parseable to an IP Address", ip_str)), + port + ) + .is_ok()); + drop(listener); + } + + #[rstest] + fn test_unreachable_ip_errors() { + let ip_str = "127.0.0.1"; + let connection_checker = TcpConnectionChecker::new(Duration::from_secs(10)); + assert!(connection_checker + .can_connect( + &IpAddr::from_str(ip_str) + .unwrap_or_else(|_| panic!("{} should be parseable to an IP Address", ip_str)), + 443 + ) + .is_err()); + } +} diff --git a/memfaultd/src/util/circular_queue.rs b/memfaultd/src/util/circular_queue.rs new file mode 100644 index 0000000..c351aab --- /dev/null +++ b/memfaultd/src/util/circular_queue.rs @@ -0,0 +1,121 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! A simple circular queue implementation +//! +//! This module provides a simple circular queue implementation. The buffer has a +//! fixed capacity and will overwrite the oldest item when it is full. + +use std::collections::VecDeque; + +#[derive(Debug, Default, Clone)] +pub struct CircularQueue { + buffer: VecDeque, + capacity: usize, +} + +impl CircularQueue { + /// Create a new circular queue with the given capacity + pub fn new(capacity: usize) -> Self { + CircularQueue { + buffer: VecDeque::with_capacity(capacity + 1), + capacity, + } + } + + /// Push an item onto the buffer. + /// + /// If the buffer is full, the oldest item will be removed. + pub fn push(&mut self, item: T) { + self.buffer.push_back(item); + + // If the buffer is full, remove the oldest item + if self.buffer.len() > self.capacity { + self.buffer.pop_front(); + } + } + + /// Pop an item from the buffer. + pub fn pop(&mut self) -> Option { + self.buffer.pop_front() + } + + /// Return true if the buffer is empty + pub fn is_empty(&self) -> bool { + self.buffer.is_empty() + } + + /// Return true if the buffer is full + pub fn is_full(&self) -> bool { + self.buffer.len() == self.capacity + } + + /// Return the number of items in the buffer + pub fn len(&self) -> usize { + self.buffer.len() + } + + /// Return the buffer capacity + pub fn capacity(&self) -> usize { + self.capacity + } + + /// Return a reference to the back item in the buffer + pub fn back(&self) -> Option<&T> { + self.buffer.back() + } + + pub fn iter(&self) -> impl Iterator { + self.buffer.iter() + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_ring_buffer() { + let mut rb = CircularQueue::new(3); + assert_eq!(rb.len(), 0); + assert_eq!(rb.capacity(), 3); + assert!(rb.is_empty()); + + rb.push(1); + assert_eq!(rb.len(), 1); + assert_eq!(rb.pop(), Some(1)); + assert!(rb.is_empty()); + + rb.push(2); + rb.push(3); + rb.push(4); + assert_eq!(rb.len(), 3); + assert_eq!(rb.pop(), Some(2)); + assert_eq!(rb.pop(), Some(3)); + assert_eq!(rb.pop(), Some(4)); + assert!(rb.is_empty()); + } + + #[test] + fn test_wrap_around() { + let mut rb = CircularQueue::new(3); + rb.push(1); + rb.push(2); + rb.push(3); + assert_eq!(rb.len(), 3); + + rb.push(4); + rb.push(5); + assert_eq!(rb.len(), 3); + assert_eq!(rb.pop(), Some(3)); + assert_eq!(rb.pop(), Some(4)); + assert_eq!(rb.pop(), Some(5)); + assert!(rb.is_empty()); + } + + #[test] + fn empty_pop() { + let mut rb: CircularQueue = CircularQueue::new(3); + assert_eq!(rb.pop(), None); + } +} diff --git a/memfaultd/src/util/die.rs b/memfaultd/src/util/die.rs new file mode 100644 index 0000000..bbb3941 --- /dev/null +++ b/memfaultd/src/util/die.rs @@ -0,0 +1,26 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! In loving memory of the practical extraction and report language. +use std::error::Error; + +use log::error; + +/// Prints the error message to the error log, and then panics. +pub fn die(e: E) -> ! { + error!("Irrecoverable error: {:#}", e); + panic!("Irrecoverable error: {:#}", e) +} + +pub trait UnwrapOrDie { + fn unwrap_or_die(self) -> T; +} + +impl UnwrapOrDie for Result { + fn unwrap_or_die(self) -> T { + match self { + Ok(v) => v, + Err(e) => die(e), + } + } +} diff --git a/memfaultd/src/util/disk_backed.rs b/memfaultd/src/util/disk_backed.rs new file mode 100644 index 0000000..fd39eb1 --- /dev/null +++ b/memfaultd/src/util/disk_backed.rs @@ -0,0 +1,221 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::Result; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use std::{ + fs::File, + io::BufReader, + path::{Path, PathBuf}, +}; + +/// A serializable struct that is backed by a JSON file on disk. +/// +/// The disk version will be loaded on object creation. If it does not exist, +/// or is invalid, the default value will be returned. +pub struct DiskBacked> { + path: PathBuf, + cache: Option, + default: T, +} + +#[derive(Debug, Eq, PartialEq)] +pub enum UpdateStatus { + Unchanged, + Updated, +} + +impl DiskBacked { + /// New instance from a path on disk + pub fn from_path(path: &Path) -> Self { + Self::from_path_with_default(path, T::default()) + } +} + +impl DiskBacked { + pub fn from_path_with_default(path: &Path, default: T) -> Self { + let cache = match File::open(path) { + Ok(file) => { + let reader = BufReader::new(file); + serde_json::from_reader(reader).ok() + } + Err(_) => None, + }; + Self { + path: path.to_owned(), + cache, + default, + } + } + + /// Return current value (disk value or default). + pub fn get(&self) -> &T { + match self.cache.as_ref() { + Some(v) => v, + None => &self.default, + } + } + + /// Updates self and writes the provided value to disk. + pub fn set(&mut self, new_value: T) -> Result { + let has_changed_from_previous_effective_value = match self.cache.as_ref() { + Some(v) => new_value != *v, + None => new_value != self.default, + }; + let needs_writing = match &self.cache { + Some(v) => v != &new_value, + None => true, + }; + + if needs_writing { + let file = File::create(&self.path)?; + serde_json::to_writer_pretty(file, &new_value)?; + self.cache = Some(new_value); + } + + Ok(match has_changed_from_previous_effective_value { + true => UpdateStatus::Updated, + false => UpdateStatus::Unchanged, + }) + } +} + +#[cfg(test)] +mod tests { + use std::{borrow::Cow, io::BufReader, path::PathBuf}; + + use rstest::{fixture, rstest}; + + use crate::test_utils::create_file_with_contents; + + use super::*; + + #[rstest] + fn test_defaults_to_default(fixture: Fixture) { + assert_eq!( + *DiskBacked::::from_path(&fixture.path).get(), + TestJson::default() + ); + } + + #[rstest] + fn test_load_from_disk(#[with(Some(TEST1))] fixture: Fixture) { + let config = DiskBacked::::from_path(&fixture.path); + assert_eq!(*config.get(), TEST1); + } + + #[rstest] + fn test_write_with_no_existing_file_and_new_equals_default(#[with(None)] fixture: Fixture) { + let mut config = DiskBacked::::from_path(&fixture.path); + + let result = config.set(TestJson::default()); + + assert!(matches!(result, Ok(UpdateStatus::Unchanged))); + + // We should have created a file on disk and stored the config + assert_eq!(fixture.read_config(), Some(TestJson::default())); + } + + #[rstest] + fn test_write_with_no_existing_file_and_new_is_not_default(#[with(None)] fixture: Fixture) { + let mut config = DiskBacked::::from_path(&fixture.path); + + let result = config.set(TEST1); + + assert!(matches!(result, Ok(UpdateStatus::Updated))); + + // We should have created a file on disk and stored the config + assert_eq!(fixture.read_config(), Some(TEST1)); + } + + #[rstest] + fn test_write_with_corrupted_local_file(#[with(None)] fixture: Fixture) { + create_file_with_contents(&fixture.path, "DIS*IS*NOT*JSON".as_bytes()).unwrap(); + let mut config = DiskBacked::::from_path(&fixture.path); + + let result = config.set(TestJson::default()); + + // Unchanged because we use default when file is corrupted + assert!(matches!(result, Ok(UpdateStatus::Unchanged))); + + // We should have created a file on disk and stored the config + assert_eq!(fixture.read_config(), Some(TestJson::default())); + } + + #[rstest] + fn test_write_without_change(#[with(Some(TEST1))] fixture: Fixture) { + let mut config = DiskBacked::::from_path(&fixture.path); + + // Delete the config file so we can see if it has been recreated + std::fs::remove_file(&fixture.path).unwrap(); + + let result = config.set(TEST1); + + assert!(matches!(result, Ok(UpdateStatus::Unchanged))); + + // We should NOT have re-created a file on disk and stored the config + assert_eq!(fixture.read_config(), None); + } + + #[rstest] + fn test_write_with_change(#[with(Some(TEST1))] fixture: Fixture) { + let mut config = DiskBacked::::from_path(&fixture.path); + + // Delete the config file so we can see if it has been recreated + std::fs::remove_file(&fixture.path).unwrap(); + + let result = config.set(TestJson::default()); + + assert!(matches!(result, Ok(UpdateStatus::Updated))); + + assert_eq!(fixture.read_config(), Some(TestJson::default())); + } + + #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Default, Debug)] + struct TestJson<'a> { + pub message: Cow<'a, str>, + } + + const TEST1: TestJson = TestJson { + message: Cow::Borrowed("test1"), + }; + + #[fixture] + fn fixture(#[default(None)] config: Option) -> Fixture { + Fixture::new(config) + } + + struct Fixture { + _temp_dir: tempfile::TempDir, + path: PathBuf, + } + + impl Fixture { + fn new(preexisting: Option) -> Self { + let temp_dir = tempfile::tempdir().unwrap(); + let path = temp_dir.path().join("data.json"); + + if let Some(value) = preexisting { + let file = File::create(&path).unwrap(); + serde_json::to_writer_pretty(file, &value).unwrap(); + } + + Self { + _temp_dir: temp_dir, + path, + } + } + + fn read_config(&self) -> Option { + let file = match File::open(&self.path) { + Ok(file) => file, + Err(_) => return None, + }; + let reader = BufReader::new(file); + match serde_json::from_reader(reader) { + Ok(config) => Some(config), + Err(_) => None, + } + } + } +} diff --git a/memfaultd/src/util/disk_size.rs b/memfaultd/src/util/disk_size.rs new file mode 100644 index 0000000..c36a27a --- /dev/null +++ b/memfaultd/src/util/disk_size.rs @@ -0,0 +1,176 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + ffi::CString, + fs::{read_dir, Metadata}, + mem, + ops::{Add, AddAssign}, + os::unix::prelude::OsStrExt, + path::Path, +}; + +use eyre::{eyre, Result}; + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// Disk space information in bytes and inodes. +pub struct DiskSize { + /// Bytes on disk + pub bytes: u64, + /// Number of inodes + pub inodes: u64, +} + +impl DiskSize { + pub fn new_capacity(bytes: u64) -> Self { + Self { + bytes, + inodes: u64::MAX, + } + } + + pub const ZERO: Self = Self { + bytes: 0, + inodes: 0, + }; + + pub fn min(a: Self, b: Self) -> Self { + Self { + bytes: a.bytes.min(b.bytes), + inodes: a.inodes.min(b.inodes), + } + } + + pub fn max(a: Self, b: Self) -> Self { + Self { + bytes: a.bytes.max(b.bytes), + inodes: a.inodes.max(b.inodes), + } + } + + pub fn exceeds(&self, other: &Self) -> bool { + (self.bytes != other.bytes || self.inodes != other.inodes) + && self.bytes >= other.bytes + && self.inodes >= other.inodes + } + + pub fn saturating_sub(self, other: Self) -> Self { + Self { + bytes: self.bytes.saturating_sub(other.bytes), + inodes: self.inodes.saturating_sub(other.inodes), + } + } +} + +impl Add for DiskSize { + type Output = Self; + + fn add(self, other: Self) -> Self { + Self { + bytes: self.bytes + other.bytes, + inodes: self.inodes + other.inodes, + } + } +} + +impl AddAssign for DiskSize { + fn add_assign(&mut self, rhs: Self) { + self.bytes += rhs.bytes; + self.inodes += rhs.inodes; + } +} + +impl From for DiskSize { + fn from(metadata: Metadata) -> Self { + Self { + bytes: metadata.len(), + inodes: 1, + } + } +} + +// We need to cast to u64 here on some platforms. +#[allow(clippy::unnecessary_cast)] +pub fn get_disk_space(path: &Path) -> Result { + let mut stat: libc::statvfs = unsafe { mem::zeroed() }; + let cpath = CString::new(path.as_os_str().as_bytes()).map_err(|_| eyre!("Invalid path"))?; + // danburkert/fs2-rs#1: cast is necessary for platforms where c_char != u8. + if unsafe { libc::statvfs(cpath.as_ptr() as *const _, &mut stat) } != 0 { + Err(eyre!("Unable to call statvfs")) + } else { + Ok(DiskSize { + // Note that we use f_bavail/f_favail instead of f_bfree/f_bavail. + // f_bfree is the number of free blocks available to the + // superuser, but we want to stop before getting to that + // point. [bf]avail is what is available to normal users. + bytes: stat.f_frsize as u64 * stat.f_bavail as u64, + inodes: stat.f_favail as u64, + }) + } +} + +/// fs_extra::get_size but also returning the number of inodes +pub fn get_size

(path: P) -> Result +where + P: AsRef, +{ + // Using `fs::symlink_metadata` since we don't want to follow symlinks, + // as we're calculating the exact size of the requested path itself. + let path_metadata = path.as_ref().symlink_metadata()?; + + let mut size = DiskSize::ZERO; + + if path_metadata.is_dir() { + for entry in read_dir(&path)? { + let entry = entry?; + // `DirEntry::metadata` does not follow symlinks (unlike `fs::metadata`), so in the + // case of symlinks, this is the size of the symlink itself, not its target. + let entry_metadata = entry.metadata()?; + + if entry_metadata.is_dir() { + // The size of the directory entry itself will be counted inside the `get_size()` call, + // so we intentionally don't also add `entry_metadata.len()` to the total here. + size += get_size(entry.path())?; + } else { + size += entry_metadata.into(); + } + } + } else { + size = path_metadata.into(); + } + + Ok(size) +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + #[rstest] + #[case(0, 0, 0, 0, false)] + // When bytes and inodes are greater + #[case(1024, 1, 0, 0, true)] + // When bytes and inodes are lesser + #[case(1024, 10, 2048, 20, false)] + // When bytes or inodes are greater + #[case(1024, 100, 2048, 20, false)] + #[case(4096, 10, 2048, 20, false)] + // When bytes are equal and inodes are greater + #[case(1024, 100, 1024, 10, true)] + fn test_size_cmp( + #[case] bytes: u64, + #[case] inodes: u64, + #[case] free_bytes: u64, + #[case] free_inodes: u64, + + #[case] exceeds: bool, + ) { + let size1 = DiskSize { bytes, inodes }; + let size2 = DiskSize { + bytes: free_bytes, + inodes: free_inodes, + }; + assert_eq!(size1.exceeds(&size2), exceeds); + } +} diff --git a/memfaultd/src/util/etc_os_release.rs b/memfaultd/src/util/etc_os_release.rs new file mode 100644 index 0000000..bd4e706 --- /dev/null +++ b/memfaultd/src/util/etc_os_release.rs @@ -0,0 +1,107 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Parser for /etc/os-release. +//! +//! This file is used by various Linux distributions to store information about the operating system. +//! See documentation for more details: https://www.freedesktop.org/software/systemd/man/latest/os-release.html +//! Currently only the `ID` and `VERSION_ID` fields are parsed. Below is an example of the file: +//! +//! ```text +//! PRETTY_NAME="Ubuntu 22.04.3 LTS" +//! NAME="Ubuntu" +//! VERSION_ID="22.04" +//! VERSION="22.04.3 LTS (Jammy Jellyfish)" +//! VERSION_CODENAME=jammy +//! ID=ubuntu +//! ID_LIKE=debian +//! HOME_URL="https://www.ubuntu.com/" +//! SUPPORT_URL="https://help.ubuntu.com/" +//! BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" +//! PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" +//! UBUNTU_CODENAME=jammy +//! ``` + +use eyre::Result; +use nom::{ + bytes::complete::{tag, take_until}, + character::complete::not_line_ending, + sequence::separated_pair, + IResult, +}; + +use std::collections::HashMap; + +const OS_RELEASE_PATH: &str = "/etc/os-release"; +const OS_RELEASE_ID: &str = "ID"; +const OS_RELEASE_VERSION_ID: &str = "VERSION_ID"; + +pub struct EtcOsRelease { + id: Option, + version_id: Option, +} + +impl EtcOsRelease { + pub fn load() -> Result { + let etc_os_release_str = std::fs::read_to_string(OS_RELEASE_PATH)?; + Self::parse(&etc_os_release_str) + } + + pub fn id(&self) -> Option { + self.id.clone() + } + + pub fn version_id(&self) -> Option { + self.version_id.clone() + } + + fn parse(etc_os_release_str: &str) -> Result { + let parsed_map = etc_os_release_str + .trim() + .lines() + .filter_map(|line| { + let parse_result: IResult<&str, (&str, &str)> = + separated_pair(take_until("="), tag("="), not_line_ending)(line); + + // Silently ignore lines that don't parse correctly. + parse_result.map(|(_, (key, value))| (key, value)).ok() + }) + .collect::>(); + + let id = parsed_map + .get(OS_RELEASE_ID) + .map(|s| s.trim_matches('\"').to_string()); + let version_id = parsed_map + .get(OS_RELEASE_VERSION_ID) + .map(|s| s.trim_matches('\"').to_string()); + + Ok(Self { id, version_id }) + } +} + +#[cfg(test)] +mod test { + use super::*; + + use rstest::rstest; + + #[rstest] + #[case::version_with_quotes("VERSION_ID=\"12.1\"\nNAME=\"Ubuntu\"", Some("12.1".to_string()))] + #[case::version_without_quotes("VERSION_ID=12.1\nNAME=\"Ubuntu\"", Some("12.1".to_string()))] + #[case::no_version("BAD_ID=\"12.1\"\nNAME=\"Ubuntu\"", None)] + fn test_version_id_parse(#[case] etc_os_release: &str, #[case] expected: Option) { + let os_release = EtcOsRelease::parse(etc_os_release).unwrap(); + + assert_eq!(os_release.version_id(), expected); + } + + #[rstest] + #[case::id_with_quotes("ID=\"ubuntu\"\nVERSION_ID=\"12.1\"", Some("ubuntu".to_string()))] + #[case::id_without_quotes("ID=ubuntu\nVERSION_ID=\"12.1\"", Some("ubuntu".to_string()))] + #[case::no_id("BAD_ID=ubuntu\nVERSION_ID=\"12.1\"", None)] + fn test_id_parse(#[case] etc_os_release: &str, #[case] expected: Option) { + let os_release = EtcOsRelease::parse(etc_os_release).unwrap(); + + assert_eq!(os_release.id(), expected); + } +} diff --git a/memfaultd/src/util/fs.rs b/memfaultd/src/util/fs.rs new file mode 100644 index 0000000..ff24a77 --- /dev/null +++ b/memfaultd/src/util/fs.rs @@ -0,0 +1,52 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + fs::{self}, + path::{Path, PathBuf}, +}; + +use eyre::{Context, Result}; +use log::warn; + +/// Takes a directory and returns a vector of all files in that directory, sorted +/// by creation date: +#[allow(dead_code)] // Required to build without warnings and --no-default-features +pub fn get_files_sorted_by_mtime(dir: &Path) -> Result> { + let read_dir = std::fs::read_dir(dir)?; + let mut entries = read_dir + .filter_map(|e| match e { + Ok(e) => Some(e), + Err(e) => { + warn!("Error reading directory entry: {:#}", e); + None + } + }) + .filter(|entry| entry.path().is_file()) + .collect::>(); + // Order by oldest first: + entries.sort_by_key(|entry| { + entry + .metadata() + .map(|m| m.modified().unwrap_or(std::time::SystemTime::UNIX_EPOCH)) + .unwrap_or(std::time::SystemTime::UNIX_EPOCH) + }); + Ok(entries.into_iter().map(|m| m.path()).collect()) +} + +/// Move a file. Try fs::rename first which is most efficient but only works if source +/// and destination are on the same filesystem. +/// Use Copy/Delete strategy if rename failed. +pub fn move_file(source: &PathBuf, target: &PathBuf) -> Result<()> { + if fs::rename(source, target).is_err() { + fs::copy(source, target).wrap_err_with(|| { + format!( + "Error moving file {} to {}", + source.display(), + target.display() + ) + })?; + fs::remove_file(source)?; + } + Ok(()) +} diff --git a/memfaultd/src/util/io.rs b/memfaultd/src/util/io.rs new file mode 100644 index 0000000..bec7c51 --- /dev/null +++ b/memfaultd/src/util/io.rs @@ -0,0 +1,194 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::io::{ + self, copy, sink, BufReader, Chain, Cursor, ErrorKind, IoSlice, Read, Seek, SeekFrom, Write, +}; + +/// A trait for getting the length of a stream. +/// Note std::io::Seek also has a stream_len() method, but that method is fallible. +pub trait StreamLen { + /// Gets the length of the stream in bytes. + fn stream_len(&self) -> u64; +} + +impl StreamLen for BufReader { + fn stream_len(&self) -> u64 { + self.get_ref().stream_len() + } +} + +impl, B: StreamLen> StreamLen for Chain, B> { + fn stream_len(&self) -> u64 { + let (a, b) = self.get_ref(); + + a.get_ref().as_ref().len() as u64 + b.stream_len() + } +} + +pub trait StreamPosition { + /// Gets the current position in the stream. + fn stream_position(&self) -> usize; +} + +/// A passthrough `Write` implementation that keeps track of position. +/// +/// This is used to keep track of the current position in the output stream, since we can't use +/// `Seek` on all output streams. Additionally this allows us to keep track of the position +/// when using functions like `copy` that may call write several times and potentially fail. +pub struct StreamPositionTracker { + writer: T, + pos: usize, +} + +impl StreamPositionTracker { + pub fn new(writer: T) -> Self { + Self { writer, pos: 0 } + } +} + +impl Write for StreamPositionTracker { + /// Passthrough to the underlying writer, but also updates the position. + fn write(&mut self, buf: &[u8]) -> io::Result { + let written = self.writer.write(buf)?; + self.pos += written; + Ok(written) + } + + // Passthrough to the underlying writer. + fn flush(&mut self) -> io::Result<()> { + self.writer.flush() + } + + fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> std::io::Result { + let written = self.writer.write_vectored(bufs)?; + self.pos += written; + Ok(written) + } +} + +impl StreamPosition for StreamPositionTracker { + fn stream_position(&self) -> usize { + self.pos + } +} + +/// Stream wrapper that implements a subset of `Seek`. +/// +/// This is needed because we read from streams that do not implement `Seek`, +/// such as `Stdin`. Instead of implementing ad-hoc skipping we'll implement +/// `Seek` such that it only allows seeking forward. Any seek from the end of +/// the stream, or that would go backwards, will result in an error. +pub struct ForwardOnlySeeker { + reader: T, + pos: u64, +} + +impl ForwardOnlySeeker { + pub fn new(reader: T) -> Self { + Self { reader, pos: 0 } + } +} + +impl Read for ForwardOnlySeeker { + /// Passthrough to the underlying reader, but also updates the position. + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let bytes_read = self.reader.read(buf)?; + self.pos += bytes_read as u64; + Ok(bytes_read) + } +} + +impl Seek for ForwardOnlySeeker { + /// Seeks forward in the stream, returns an error if seeking backwards. + fn seek(&mut self, pos: SeekFrom) -> io::Result { + let seek_size = match pos { + SeekFrom::Current(offset) => u64::try_from(offset).ok(), + SeekFrom::Start(offset) => offset.checked_sub(self.pos), + SeekFrom::End(_) => { + return Err(io::Error::new( + ErrorKind::InvalidInput, + "Cannot seek from end", + )) + } + }; + + match seek_size { + Some(seek_size) => { + copy(&mut self.by_ref().take(seek_size), &mut sink())?; + Ok(self.pos) + } + None => Err(io::Error::new( + ErrorKind::InvalidInput, + "Only seeking forward allowed", + )), + } + } + + fn stream_position(&mut self) -> io::Result { + Ok(self.pos) + } +} + +#[cfg(test)] +mod test { + use rstest::rstest; + + use super::*; + + #[test] + fn test_write_cursor() { + let mut buf = Vec::new(); + let mut writer = StreamPositionTracker::new(&mut buf); + + writer.write_all(b"hello").unwrap(); + assert_eq!(writer.stream_position(), 5); + writer.write_all(b" world").unwrap(); + assert_eq!(writer.stream_position(), 11); + writer.write_all(b"!").unwrap(); + assert_eq!(writer.stream_position(), 12); + + assert_eq!(buf, b"hello world!"); + } + + #[test] + fn test_write_vectored_cursor() { + let mut buf = Vec::new(); + let mut writer = StreamPositionTracker::new(&mut buf); + + let write_vector = [ + IoSlice::new(b"hello"), + IoSlice::new(b" world"), + IoSlice::new(b"!"), + ]; + let bytes_written = writer.write_vectored(&write_vector).unwrap(); + + assert_eq!(bytes_written, 12); + assert_eq!(writer.stream_position(), 12); + assert_eq!(buf, b"hello world!"); + } + + #[test] + fn test_forward_seeker_stream() { + let mut input_stream = b"Hello world".as_ref(); + + let mut reader = ForwardOnlySeeker::new(&mut input_stream); + let mut out_buf = [0u8; 5]; + reader.read_exact(&mut out_buf).unwrap(); + assert_eq!(&out_buf, b"Hello"); + + reader.seek(SeekFrom::Current(1)).unwrap(); + reader.read_exact(&mut out_buf).unwrap(); + assert_eq!(&out_buf, b"world"); + } + + #[rstest] + #[case(SeekFrom::End(0))] + #[case(SeekFrom::Current(-1))] + #[case(SeekFrom::Start(0))] + fn test_forward_seeker_seek_fail(#[case] seek: SeekFrom) { + let mut reader = ForwardOnlySeeker::new(b"Hello world".as_ref()); + reader.seek(SeekFrom::Start(1)).unwrap(); + assert!(reader.seek(seek).is_err()); + } +} diff --git a/memfaultd/src/util/ipc.rs b/memfaultd/src/util/ipc.rs new file mode 100644 index 0000000..5ac6e2f --- /dev/null +++ b/memfaultd/src/util/ipc.rs @@ -0,0 +1,11 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::Result; +use nix::sys::signal::Signal::SIGUSR1; + +use super::pid_file::send_signal_to_pid; + +pub fn send_flush_signal() -> Result<()> { + send_signal_to_pid(SIGUSR1) +} diff --git a/memfaultd/src/util/math.rs b/memfaultd/src/util/math.rs new file mode 100644 index 0000000..58fd58c --- /dev/null +++ b/memfaultd/src/util/math.rs @@ -0,0 +1,12 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +/// Rounds the given value up to the nearest multiple of the given alignment. +/// +/// For values <= 1, the value is returned unchanged. +pub fn align_up(value: usize, alignment: usize) -> usize { + if alignment <= 1 { + return value; + } + ((value) + (alignment - 1)) & !(alignment - 1) +} diff --git a/memfaultd/src/util/mem.rs b/memfaultd/src/util/mem.rs new file mode 100644 index 0000000..4767097 --- /dev/null +++ b/memfaultd/src/util/mem.rs @@ -0,0 +1,28 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::mem::size_of; +use std::slice::{from_raw_parts, from_raw_parts_mut}; + +pub trait AsBytes { + /// Returns a slice of bytes representing the raw memory of the object. + /// # Safety + /// It is on the caller to ensure the interpretation of the bytes is correct. + unsafe fn as_bytes(&self) -> &[u8]; + + /// Returns a mutable slice of bytes representing the raw memory of the object. + /// # Safety + /// The type must not contain any references, pointers or types that require + /// validating invariants. + unsafe fn as_mut_bytes(&mut self) -> &mut [u8]; +} + +impl AsBytes for T { + unsafe fn as_bytes(&self) -> &[u8] { + from_raw_parts((self as *const T) as *const u8, size_of::()) + } + + unsafe fn as_mut_bytes(&mut self) -> &mut [u8] { + from_raw_parts_mut((self as *mut T) as *mut u8, size_of::()) + } +} diff --git a/memfaultd/src/util/mod.rs b/memfaultd/src/util/mod.rs new file mode 100644 index 0000000..906428b --- /dev/null +++ b/memfaultd/src/util/mod.rs @@ -0,0 +1,31 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +mod disk_backed; +pub use disk_backed::*; +mod die; +pub mod disk_size; +pub mod patterns; +pub use die::*; +pub mod can_connect; +pub mod circular_queue; +pub mod etc_os_release; +pub mod fs; +pub mod io; +pub mod ipc; +pub mod math; +pub mod mem; +pub mod output_arg; +pub mod path; +pub mod persistent_rate_limiter; +pub mod pid_file; +#[cfg(feature = "logging")] +pub mod rate_limiter; +pub mod serialization; +pub mod string; +pub mod system; +pub mod task; +#[cfg(feature = "logging")] +pub mod tcp_server; +pub mod time_measure; +pub mod zip; diff --git a/memfaultd/src/util/output_arg.rs b/memfaultd/src/util/output_arg.rs new file mode 100644 index 0000000..6221fe2 --- /dev/null +++ b/memfaultd/src/util/output_arg.rs @@ -0,0 +1,42 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::{ + fs::File, + io::{stdout, Write}, + path::PathBuf, +}; + +use argh::FromArgValue; +use eyre::{Context, Result}; + +/// An Argh argument that can be either a `PathBuf` or a reference to `stdout` (`-`). +pub enum OutputArg { + Stdout, + File(PathBuf), +} + +impl FromArgValue for OutputArg { + fn from_arg_value(value: &str) -> Result { + if value == "-" { + Ok(OutputArg::Stdout) + } else { + Ok(OutputArg::File(value.into())) + } + } +} + +impl OutputArg { + /// Open the output stream designated by the user. + pub fn get_output_stream(&self) -> Result> { + let stream: Box = + match self { + OutputArg::Stdout => Box::new(stdout()), + OutputArg::File(path) => Box::new(File::create(path).wrap_err_with(|| { + format!("Error opening destination file {}", path.display()) + })?), + }; + + Ok(stream) + } +} diff --git a/memfaultd/src/util/path.rs b/memfaultd/src/util/path.rs new file mode 100644 index 0000000..5444429 --- /dev/null +++ b/memfaultd/src/util/path.rs @@ -0,0 +1,66 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::{eyre, Error, Result}; +use std::path::Path; +use std::{ffi::OsStr, path::PathBuf}; + +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(try_from = "PathBuf")] +/// A path that must be absolute. Use `AbsolutePath::try_from` to construct. +pub struct AbsolutePath(PathBuf); + +impl TryFrom for AbsolutePath { + type Error = Error; + + fn try_from(path: PathBuf) -> Result { + if path.is_absolute() { + Ok(Self(path)) + } else { + Err(eyre!("Path must be absolute: {:?}", path)) + } + } +} +impl From for PathBuf { + fn from(p: AbsolutePath) -> PathBuf { + p.0 + } +} +impl PartialEq for PathBuf { + fn eq(&self, other: &AbsolutePath) -> bool { + *self == *other.0 + } +} + +impl std::ops::Deref for AbsolutePath { + type Target = Path; + #[inline] + fn deref(&self) -> &Path { + self.0.as_path() + } +} + +/// Splits the filename at the first dot. +/// This is similar to the nighly-only std::path::Path::file_prefix. +#[allow(dead_code)] +pub fn file_prefix(path: &Path) -> Option<&OsStr> { + let file_name = path.file_name()?; + file_name.to_str()?.split('.').next().map(OsStr::new) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case("/test", "test")] + #[case("/test.log", "test")] + #[case("/test.log.zlib", "test")] + fn test_file_prefix(#[case] path: &str, #[case] expected: &str) { + assert_eq!(file_prefix(Path::new(path)), Some(OsStr::new(expected))); + } +} diff --git a/memfaultd/src/util/patterns.rs b/memfaultd/src/util/patterns.rs new file mode 100644 index 0000000..82c6c63 --- /dev/null +++ b/memfaultd/src/util/patterns.rs @@ -0,0 +1,84 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +pub fn alphanum_slug_is_valid(s: &str, max_len: usize) -> eyre::Result<()> { + match ( + (1..max_len).contains(&s.len()), + s.chars() + .all(|c| matches!(c, 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' )), + ) { + (true, true) => Ok(()), + (false, _) => Err(eyre::eyre!("Must be with 1 and 128 characters long")), + (_, false) => Err(eyre::eyre!( + "Must only contain alphanumeric characters and - or _" + )), + } +} + +pub fn alphanum_slug_is_valid_and_starts_alpha(s: &str, max_len: usize) -> eyre::Result<()> { + alphanum_slug_is_valid(s, max_len)?; + if s.starts_with(char::is_alphabetic) { + Ok(()) + } else { + Err(eyre::eyre!("Must start with an alphabetic character")) + } +} + +pub fn alphanum_slug_dots_colon_is_valid(s: &str, max_len: usize) -> eyre::Result<()> { + match ( + (1..max_len).contains(&s.len()), + s.chars() + .all(|c| matches!(c, 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' | '+' | '.' | ':')), + ) { + (true, true) => Ok(()), + (false, _) => Err(eyre::eyre!("Must be with 1 and 128 characters long")), + (_, false) => Err(eyre::eyre!( + "Must only contain alphanumeric characters, -,_,+,., and :" + )), + } +} + +pub fn alphanum_slug_dots_colon_spaces_parens_slash_is_valid( + s: &str, + max_len: usize, +) -> eyre::Result<()> { + match ((1..max_len).contains(&s.len()), s.chars().all(|c| matches!(c, 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' | '+' | '.' | ':' | ' ' | '[' | ']' | '(' | ')' | '\\'))) { + (true, true) => Ok(()), + (false, _) => Err(eyre::eyre!("Must be with 1 and 128 characters long")), + (_, false) => Err(eyre::eyre!("Must only contain alphanumeric characters, spaces, -,_,+,.,:,[,],(,), and \\")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + #[rstest] + #[case("1.0.0-rc2", true)] + #[case("qemuarm64", true)] + #[case("1.0.0-$", false)] + #[case("2.30^.1-rc4", false)] + #[case("2.30\\.1-rc4", false)] + #[case("spaces are invalid", false)] + fn test_alphanum_slug_dots_colon_is_valid(#[case] input: &str, #[case] result: bool) { + assert_eq!(alphanum_slug_dots_colon_is_valid(input, 64).is_ok(), result); + } + + #[rstest] + #[case("1.0.0-rc2", true)] + #[case("qemuarm64", true)] + #[case("1.0.0-$", false)] + #[case("2.30^.1-rc4", false)] + #[case("2.30\\.1-rc4", true)] + #[case("spaces are valid", true)] + fn test_alphanum_slug_dots_colon_spaces_parens_slash_is_valid( + #[case] input: &str, + #[case] result: bool, + ) { + assert_eq!( + alphanum_slug_dots_colon_spaces_parens_slash_is_valid(input, 64).is_ok(), + result + ); + } +} diff --git a/memfaultd/src/util/persistent_rate_limiter.rs b/memfaultd/src/util/persistent_rate_limiter.rs new file mode 100644 index 0000000..e007ae0 --- /dev/null +++ b/memfaultd/src/util/persistent_rate_limiter.rs @@ -0,0 +1,146 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use chrono::{DateTime, Duration, TimeZone, Utc}; +use eyre::Result; +use itertools::Itertools; +use std::{ + collections::VecDeque, + fs::File, + io::{BufRead, BufReader, Write}, + path::{Path, PathBuf}, +}; + +/// A `PersistentRateLimiter` that enforces action limits based on time +/// duration. +/// +/// - It allows actions up to the specified `count` within the `duration`. +/// - It stores a history of attempts in a file `path`. +pub struct PersistentRateLimiter { + path: PathBuf, + count: u32, + duration: Duration, + /// A list of all the times the rate limiter has been hit. We keep them + /// sorted from most recent to oldest and we cap this list to count. + history: VecDeque>, +} + +impl PersistentRateLimiter { + /// Load the rate limiter state from disk. + /// + /// Non-existent file is not considered an error. Garbage in the file will be skipped over. + pub fn load>(path: P, count: u32, duration: Duration) -> Result { + if count == 0 { + return Err(eyre::eyre!("count must be greater than 0")); + } + if duration.num_milliseconds() == 0 { + return Err(eyre::eyre!("duration must be greater than 0")); + } + + // Load previous invocations of the rate limiter, discarding anything that is not parseable. + let history = match File::open(&path) { + Ok(file) => BufReader::new(file) + .split(b' ') + .filter_map::, _>(|t| { + let ts: i64 = std::str::from_utf8(&t.ok()?).ok()?.parse().ok()?; + Utc.timestamp_opt(ts, 0).single() + }) + .sorted_by(|a, b| b.cmp(a)) + .collect::>(), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => VecDeque::new(), + Err(e) => return Err(e.into()), + }; + Ok(Self { + path: path.as_ref().to_owned(), + count, + duration, + history, + }) + } + + fn check_with_time(&mut self, now: DateTime) -> bool { + if self.history.len() >= self.count as usize { + if let Some(oldest) = self.history.back() { + if now.signed_duration_since(*oldest) < self.duration { + return false; + } + } + } + + self.history.push_front(now); + self.history.truncate(self.count as usize); + + true + } + + /// Check if the rate limiter will allow one call now. The state is updated + /// but not written to disk. Call `save()` to persist the rate limiter. + pub fn check(&mut self) -> bool { + self.check_with_time(Utc::now()) + } + + /// Writes the rate limiter state to disk. + pub fn save(self) -> Result<()> { + let mut file = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(self.path)?; + for time in self.history.iter() { + write!(file, "{} ", time.timestamp())?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use tempfile::tempdir; + + use super::*; + + #[rstest] + #[case::invalid_count(0, Duration::seconds(1))] + #[case::invalid_duration(1, Duration::seconds(0))] + fn invalid_init(#[case] count: u32, #[case] duration: Duration) { + let tmpdir = tempdir().unwrap(); + let path = tmpdir.path().join("test"); + + assert!(PersistentRateLimiter::load(path, count, duration).is_err()); + } + + #[rstest] + #[case(1, Duration::seconds(10), vec![0, 10, 20, 30, 35, 40], vec![true, true, true, true, false, true])] + #[case(3, Duration::seconds(10), vec![0, 0, 9, 10, 11, 12], vec![true, true, true, true, true, false ])] + #[case(3, Duration::seconds(10), vec![0, 0, 9, 9, 9, 9, 18, 19], vec![true, true, true, false, false, false, true, true ])] + #[case(3, Duration::seconds(10), vec![0, 100, 200, 300, 400, 500, 600], vec![true, true, true, true, true, true, true])] + fn test_rate_limiter( + #[case] count: u32, + #[case] duration: Duration, + #[case] timestamps: Vec, + #[case] expected: Vec, + ) { + assert_eq!( + timestamps.len(), + expected.len(), + "timestamps and expected results should be the same length" + ); + + let tmpdir = tempdir().unwrap(); + let path = tmpdir.path().join("test"); + + for (time, expected) in timestamps.into_iter().zip(expected.into_iter()) { + let mut limiter = + PersistentRateLimiter::load(&path, count, duration).expect("load error"); + assert_eq!( + limiter.check_with_time(Utc.timestamp_opt(time, 0).single().unwrap()), + expected, + "time: {} - history: {:?}", + time, + limiter.history + ); + limiter.save().expect("save error"); + } + } +} diff --git a/memfaultd/src/util/pid_file.rs b/memfaultd/src/util/pid_file.rs new file mode 100644 index 0000000..bb47a14 --- /dev/null +++ b/memfaultd/src/util/pid_file.rs @@ -0,0 +1,59 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::fs::{remove_file, OpenOptions}; +use std::io::{ErrorKind, Write}; +use std::path::Path; + +use eyre::{eyre, Report, Result, WrapErr}; +use nix::unistd::Pid; + +const PID_FILE: &str = "/var/run/memfaultd.pid"; + +pub fn write_pid_file() -> Result<()> { + let file = OpenOptions::new() + .write(true) + .create_new(true) + .open(Path::new(PID_FILE)); + + match file { + Ok(mut file) => writeln!(file, "{}", Pid::this()).wrap_err("Failed to write PID file"), + Err(e) => { + let msg = match e.kind() { + ErrorKind::AlreadyExists => "Daemon already running, aborting.", + _ => "Failed to open PID file, aborting.", + }; + Err(Report::new(e).wrap_err(msg)) + } + } +} + +/// Returns true if (and only if) our PID file exists, is readable and contains our current PID. +pub fn is_pid_file_about_me() -> bool { + match get_pid_from_file() { + Ok(pid) => pid == Pid::this(), + _ => false, + } +} + +pub fn get_pid_from_file() -> Result { + match std::fs::read_to_string(PID_FILE) { + Ok(pid_string) => { + let pid = pid_string + .trim() + .parse() + .wrap_err("Failed to parse PID file contents")?; + Ok(Pid::from_raw(pid)) + } + Err(_) => Err(eyre!("Couldn't read memfaultd PID file at {}.", PID_FILE)), + } +} + +pub fn remove_pid_file() -> Result<()> { + remove_file(PID_FILE).wrap_err("Failed to remove PID file") +} + +pub fn send_signal_to_pid(signal: nix::sys::signal::Signal) -> Result<()> { + let pid = get_pid_from_file()?; + nix::sys::signal::kill(pid, signal).wrap_err("Failed to send signal to memfaultd") +} diff --git a/memfaultd/src/util/rate_limiter.rs b/memfaultd/src/util/rate_limiter.rs new file mode 100644 index 0000000..16502a2 --- /dev/null +++ b/memfaultd/src/util/rate_limiter.rs @@ -0,0 +1,154 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +//! Support for rate limiting some execution paths. +use std::num::NonZeroU32; + +use eyre::Result; +use governor::{ + clock, + middleware::NoOpMiddleware, + state::{InMemoryState, NotKeyed}, + Quota, RateLimiter as GovRateLimiter, +}; + +/// A rate limiter which keeps track of how many calls were rate limited. +/// You can also provide a `info: I` parameter with each call. The latest one +/// will be passed to the runner when not rate limited anymore. +pub struct RateLimiter { + rate_limiter: GovRateLimiter>, + limited_calls: Option>, +} + +pub struct RateLimitedCalls { + pub count: usize, + pub latest_call: I, +} + +impl RateLimiter { + /// Create a new rate limiter with given capacity, quantum and rate (see ratelimit::Ratelimiter). + pub fn new(capacity_per_minute: NonZeroU32) -> Self { + Self { + rate_limiter: GovRateLimiter::direct(Quota::per_minute(capacity_per_minute)), + limited_calls: None, + } + } +} + +impl RateLimiter { + /// Run the provided work function if the rate limitings limits have not been reached. + pub fn run_within_limits(&mut self, info: I, work: W) -> Result<()> + where + W: FnOnce(Option>) -> Result<()>, + { + if self.rate_limiter.check().is_ok() { + work(self.limited_calls.take()) + } else { + self.limited_calls = Some(match self.limited_calls.take() { + None => RateLimitedCalls { + count: 1, + latest_call: info, + }, + Some(l) => RateLimitedCalls { + count: l.count + 1, + latest_call: info, + }, + }); + Ok(()) + } + } +} + +#[cfg(test)] +mod tests { + use std::{num::NonZeroU32, time::Duration}; + + use governor::clock; + use governor::clock::FakeRelativeClock; + use rstest::fixture; + use rstest::rstest; + + use super::RateLimiter; + + #[rstest] + fn test_sustained_100_per_minute(mut rl: RLFixture) { + for _ in 0..20 { + rl.assert_wait_and_grab_tokens(15_000, 25); + } + } + + #[rstest] + fn test_bursty_start(mut rl: RLFixture) { + rl.assert_wait_and_grab_tokens(0, 100); + rl.assert_empty(); + rl.assert_wait_and_grab_tokens(15_000, 25); + rl.assert_empty(); + } + + #[rstest] + fn test_reject_burst(mut rl: RLFixture) { + rl.assert_wait_and_grab_tokens(200_000, 100); + rl.assert_empty(); + rl.assert_wait_and_grab_tokens(1000, 1); + } + + #[fixture(limit = 100)] + fn rl(limit: u32) -> RLFixture { + let clock = FakeRelativeClock::default(); + RLFixture { + rl: RateLimiter::new_with_clock(NonZeroU32::new(limit).unwrap(), &clock), + clock, + } + } + + struct RLFixture { + rl: RateLimiter<(), FakeRelativeClock>, + clock: FakeRelativeClock, + } + + impl RLFixture { + pub fn assert_wait_and_grab_tokens(&mut self, sleep_ms: u64, count_tokens: u64) { + let grabbed = self.wait_and_grab_tokens(sleep_ms, count_tokens); + assert!( + count_tokens == grabbed, + "Expected to grab {count_tokens} but only {grabbed} available." + ); + } + + pub fn assert_empty(&mut self) { + assert!(self.grab_tokens(1) == 0, "Rate limiter is not empty"); + } + + pub fn wait_and_grab_tokens(&mut self, sleep_ms: u64, count_tokens: u64) -> u64 { + self.clock.advance(Duration::from_millis(sleep_ms)); + self.grab_tokens(count_tokens) + } + + pub fn grab_tokens(&mut self, c: u64) -> u64 { + for i in 0..c { + let work_done = &mut false; + let _result = self.rl.run_within_limits((), |_info| { + *work_done = true; + Ok(()) + }); + if !*work_done { + return i; + } + } + c + } + } + + impl RateLimiter { + #[cfg(test)] + pub fn new_with_clock(capacity: NonZeroU32, clock: &C) -> Self { + use governor::Quota; + use governor::RateLimiter as GovRateLimiter; + + Self { + rate_limiter: GovRateLimiter::direct_with_clock(Quota::per_minute(capacity), clock), + limited_calls: None, + } + } + } +} diff --git a/memfaultd/src/util/serialization/datetime_to_rfc3339.rs b/memfaultd/src/util/serialization/datetime_to_rfc3339.rs new file mode 100644 index 0000000..1502e1c --- /dev/null +++ b/memfaultd/src/util/serialization/datetime_to_rfc3339.rs @@ -0,0 +1,24 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Deserializer, Serializer}; + +pub fn serialize(time: &DateTime, serializer: S) -> Result +where + S: Serializer, +{ + let datetime_str = time.to_rfc3339(); + serializer.serialize_str(&datetime_str) +} + +pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let datetime_str = String::deserialize(deserializer)?; + let datetime = DateTime::parse_from_rfc3339(&datetime_str) + .map_err(|e| serde::de::Error::custom(format!("invalid timestamp: {}", e)))?; + + Ok(datetime.with_timezone(&Utc)) +} diff --git a/memfaultd/src/util/serialization/float_to_datetime.rs b/memfaultd/src/util/serialization/float_to_datetime.rs new file mode 100644 index 0000000..b70e4ac --- /dev/null +++ b/memfaultd/src/util/serialization/float_to_datetime.rs @@ -0,0 +1,30 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use chrono::{DateTime, NaiveDateTime, Utc}; +use serde::{Deserialize, Deserializer, Serializer}; + +pub fn serialize(time: &DateTime, serializer: S) -> Result +where + S: Serializer, +{ + serializer.serialize_f64( + time.timestamp() as f64 + (time.timestamp_subsec_micros() as f64 / 1_000_000.0), + ) +} + +pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let secs = f64::deserialize(deserializer)?; + + // Collectd only sends milli-seconds. We round the float to the nearest ms + // to avoid precision error. + let ms = ((secs.rem_euclid(1.0)) * 1e3).round() as u32; + + match NaiveDateTime::from_timestamp_opt(secs.floor() as i64, ms * 1_000_000) { + Some(naive) => Ok(DateTime::::from_utc(naive, Utc)), + None => Err(serde::de::Error::custom("invalid timestamp")), + } +} diff --git a/memfaultd/src/util/serialization/float_to_duration.rs b/memfaultd/src/util/serialization/float_to_duration.rs new file mode 100644 index 0000000..53f3f22 --- /dev/null +++ b/memfaultd/src/util/serialization/float_to_duration.rs @@ -0,0 +1,23 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use chrono::Duration; +use serde::{Deserialize, Deserializer, Serializer}; + +pub fn serialize(duration: &Duration, serializer: S) -> Result +where + S: Serializer, +{ + serializer + .serialize_f64(duration.num_seconds() as f64 + duration.num_milliseconds() as f64 / 1000.0) +} + +pub fn deserialize<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let d = f64::deserialize(deserializer)?; + let seconds = d.trunc() as i64; + let ms = (d.rem_euclid(1.0) * 1000.0) as i64; + Ok(Duration::seconds(seconds) + Duration::milliseconds(ms)) +} diff --git a/memfaultd/src/util/serialization/kib_to_usize.rs b/memfaultd/src/util/serialization/kib_to_usize.rs new file mode 100644 index 0000000..438e2b3 --- /dev/null +++ b/memfaultd/src/util/serialization/kib_to_usize.rs @@ -0,0 +1,45 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +pub fn serialize(size: &usize, serializer: S) -> Result +where + S: Serializer, +{ + if size % 1024 != 0 { + return Err(serde::ser::Error::custom( + "Cannot serialize non-multiple of 1024 to kib.", + )); + } + (size / 1024).serialize(serializer) +} + +pub fn deserialize<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let size = usize::deserialize(deserializer)?; + Ok(size * 1024) +} + +#[cfg(test)] +mod tests { + + #[test] + fn serialize_error() { + let mut serializer = serde_json::Serializer::new(std::io::stdout()); + let r = super::serialize(&1025, &mut serializer); + assert!(r.is_err()); + } + + #[test] + fn serialize_multiple_of_1024() { + let mut buf = Vec::new(); + let mut serializer = serde_json::Serializer::new(&mut buf); + let r = super::serialize(&43008, &mut serializer); + assert!(r.is_ok()); + + assert_eq!(&buf, b"42"); + } +} diff --git a/memfaultd/src/util/serialization/milliseconds_to_duration.rs b/memfaultd/src/util/serialization/milliseconds_to_duration.rs new file mode 100644 index 0000000..210ce52 --- /dev/null +++ b/memfaultd/src/util/serialization/milliseconds_to_duration.rs @@ -0,0 +1,21 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use serde::{Deserialize, Deserializer, Serializer}; + +use std::time::Duration; + +pub fn serialize(duration: &Duration, serializer: S) -> Result +where + S: Serializer, +{ + serializer.serialize_u128(duration.as_millis()) +} + +pub fn deserialize<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let secs = u64::deserialize(deserializer)?; + Ok(Duration::from_millis(secs)) +} diff --git a/memfaultd/src/util/serialization/mod.rs b/memfaultd/src/util/serialization/mod.rs new file mode 100644 index 0000000..568902f --- /dev/null +++ b/memfaultd/src/util/serialization/mod.rs @@ -0,0 +1,12 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +pub mod datetime_to_rfc3339; +pub mod float_to_datetime; +pub mod float_to_duration; +pub mod kib_to_usize; +pub mod milliseconds_to_duration; +pub mod number_to_compression; +pub mod optional_milliseconds_to_duration; +pub mod seconds_to_duration; +pub mod sorted_map; diff --git a/memfaultd/src/util/serialization/number_to_compression.rs b/memfaultd/src/util/serialization/number_to_compression.rs new file mode 100644 index 0000000..f11bfe1 --- /dev/null +++ b/memfaultd/src/util/serialization/number_to_compression.rs @@ -0,0 +1,25 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use flate2::Compression; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +pub fn serialize(compression: &Compression, serializer: S) -> Result +where + S: Serializer, +{ + (compression.level()).serialize(serializer) +} + +pub fn deserialize<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let level = u32::deserialize(deserializer)?; + match level { + 0..=9 => Ok(Compression::new(level)), + _ => Err(serde::de::Error::custom( + "Compression level must be between 0 and 9.", + )), + } +} diff --git a/memfaultd/src/util/serialization/optional_milliseconds_to_duration.rs b/memfaultd/src/util/serialization/optional_milliseconds_to_duration.rs new file mode 100644 index 0000000..cb79173 --- /dev/null +++ b/memfaultd/src/util/serialization/optional_milliseconds_to_duration.rs @@ -0,0 +1,28 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use serde::{Deserialize, Deserializer, Serializer}; + +use std::time::Duration; + +pub fn serialize(duration: &Option, serializer: S) -> Result +where + S: Serializer, +{ + if let Some(duration) = duration { + serializer.serialize_u128(duration.as_millis()) + } else { + serializer.serialize_none() + } +} + +pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + if let Ok(secs) = u64::deserialize(deserializer) { + Ok(Some(Duration::from_millis(secs))) + } else { + Ok(None) + } +} diff --git a/memfaultd/src/util/serialization/seconds_to_duration.rs b/memfaultd/src/util/serialization/seconds_to_duration.rs new file mode 100644 index 0000000..d62a5db --- /dev/null +++ b/memfaultd/src/util/serialization/seconds_to_duration.rs @@ -0,0 +1,21 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use serde::{Deserialize, Deserializer, Serializer}; + +use std::time::Duration; + +pub fn serialize(duration: &Duration, serializer: S) -> Result +where + S: Serializer, +{ + serializer.serialize_u64(duration.as_secs()) +} + +pub fn deserialize<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let secs = u64::deserialize(deserializer)?; + Ok(Duration::from_secs(secs)) +} diff --git a/memfaultd/src/util/serialization/sorted_map.rs b/memfaultd/src/util/serialization/sorted_map.rs new file mode 100644 index 0000000..0d30c4d --- /dev/null +++ b/memfaultd/src/util/serialization/sorted_map.rs @@ -0,0 +1,15 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::collections::{BTreeMap, HashMap}; + +use serde::{Serialize, Serializer}; + +pub fn sorted_map( + value: &HashMap, + serializer: S, +) -> Result { + let mut items: Vec<(_, _)> = value.iter().collect(); + items.sort_by(|a, b| a.0.cmp(b.0)); + BTreeMap::from_iter(items).serialize(serializer) +} diff --git a/memfaultd/src/util/string.rs b/memfaultd/src/util/string.rs new file mode 100644 index 0000000..e4ef612 --- /dev/null +++ b/memfaultd/src/util/string.rs @@ -0,0 +1,94 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +/// Simple implementation of remove-C-style-comments +/// Saves code space by not relying on an external library. +pub fn remove_comments(config_string: &str) -> String { + let mut data = String::from(config_string); + while let Some(index) = data.find("/*") { + if let Some(index_end) = data.find("*/") { + data = String::from(&data[..index]) + &data[index_end + 2..]; + } else { + // No matching close. Keep everything + break; + } + } + data +} + +pub trait Ellipsis { + fn truncate_with_ellipsis(&mut self, len_bytes: usize); +} + +impl Ellipsis for String { + fn truncate_with_ellipsis(&mut self, len_bytes: usize) { + const ELLIPSIS: &str = "…"; // Note: 3 bytes in UTF-8 + let max_len_bytes = len_bytes - ELLIPSIS.len(); + if self.len() <= max_len_bytes { + return; + } + let idx = (0..=max_len_bytes) + .rev() + .find(|idx| self.is_char_boundary(*idx)) + .unwrap_or(0); + self.truncate(idx); + self.push_str(ELLIPSIS); + } +} + +/// Capitalize the first letter of a string +pub fn capitalize(s: &str) -> String { + let mut c = s.chars(); + match c.next() { + None => String::new(), + Some(f) => f.to_uppercase().collect::() + c.as_str(), + } +} + +#[cfg(test)] +mod test { + use super::*; + use rstest::rstest; + + #[test] + fn test_remove_comments() { + assert_eq!(remove_comments(""), ""); + assert_eq!(remove_comments("hello world"), "hello world"); + assert_eq!(remove_comments("hello /* comment */ world"), "hello world"); + assert_eq!( + remove_comments("hello /* comment world"), + "hello /* comment world" + ); + assert_eq!( + remove_comments("hello /* comment */world/* comment */"), + "hello world" + ); + } + + #[rstest] + // No truncation: + #[case("foobar", 10, "foobar")] + // Truncation basic: + #[case("foobar", 6, "foo…")] + // Truncation inside a multi-byte character (smiling pile of poo is 4 bytes): + #[case("f💩bar", 6, "f…")] + // Panic: len_bytes too short to fit ellipsis: + #[should_panic] + #[case("foobar", 0, "…")] + fn truncate_with_ellipsis( + #[case] input: &str, + #[case] len_bytes: usize, + #[case] expected: &str, + ) { + let mut s = String::from(input); + s.truncate_with_ellipsis(len_bytes); + assert_eq!(&s, expected); + } + + #[rstest] + #[case("capital")] + #[case("Capital")] + fn test_capitalize(#[case] input: &str) { + assert_eq!(capitalize(input), "Capital"); + } +} diff --git a/memfaultd/src/util/system.rs b/memfaultd/src/util/system.rs new file mode 100644 index 0000000..b6b0553 --- /dev/null +++ b/memfaultd/src/util/system.rs @@ -0,0 +1,94 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::{eyre, Result}; +use libc::{clockid_t, timespec, CLOCK_MONOTONIC}; + +#[cfg(target_os = "linux")] +use libc::{sysconf, _SC_CLK_TCK, _SC_PAGE_SIZE}; + +use uuid::Uuid; + +#[cfg(target_os = "linux")] +pub fn read_system_boot_id() -> Result { + use eyre::Context; + use std::{fs::read_to_string, str::FromStr}; + + const BOOT_ID_PATH: &str = "/proc/sys/kernel/random/boot_id"; + let boot_id = read_to_string(BOOT_ID_PATH); + + match boot_id { + Ok(boot_id_str) => Uuid::from_str(boot_id_str.trim()).wrap_err("Invalid boot id"), + Err(_) => Err(eyre!("Unable to read boot id from system.")), + } +} + +#[cfg(target_os = "linux")] +pub fn clock_ticks_per_second() -> u64 { + unsafe { sysconf(_SC_CLK_TCK) as u64 } +} + +#[cfg(target_os = "linux")] +pub fn bytes_per_page() -> u64 { + unsafe { sysconf(_SC_PAGE_SIZE) as u64 } +} + +/// Calls clock_gettime +/// Most interesting to us are: +/// CLOCK_MONOTONIC: "clock that increments monotonically, tracking the time +/// since an arbitrary point, and will continue to increment while the system is +/// asleep." +/// CLOCK_BOOTTIME A nonsettable system-wide clock that is identical to +/// CLOCK_MONOTONIC, except that it also includes any time that the system is +/// suspended. This allows applications to get a suspend-aware monotonic clock +/// without having to deal with the complications of CLOCK_REALTIME, which may +/// have discontinuities if the time is changed using settimeofday(2) or +/// similar. +pub enum Clock { + Monotonic, + Boottime, +} +pub fn get_system_clock(clock: Clock) -> Result { + // Linux only so we define it here. + const CLOCK_BOOTTIME: clockid_t = 7; + + let mut t = timespec { + tv_sec: 0, + tv_nsec: 0, + }; + if unsafe { + libc::clock_gettime( + match clock { + Clock::Monotonic => CLOCK_MONOTONIC, + Clock::Boottime if cfg!(target_os = "linux") => CLOCK_BOOTTIME, + // Falls back to monotonic if not linux + Clock::Boottime => CLOCK_MONOTONIC, + }, + &mut t, + ) + } != 0 + { + Err(eyre!("Error getting system clock.")) + } else { + Ok(std::time::Duration::new(t.tv_sec as u64, t.tv_nsec as u32)) + } +} + +/// Provide some mock implementations for non-Linux systems. Designed for development. Not actual use. + +#[cfg(not(target_os = "linux"))] +pub fn read_system_boot_id() -> Result { + use once_cell::sync::Lazy; + static MOCK_BOOT_ID: Lazy = Lazy::new(Uuid::new_v4); + Ok(*MOCK_BOOT_ID) +} + +#[cfg(not(target_os = "linux"))] +pub fn clock_ticks_per_second() -> u64 { + 10_000 +} + +#[cfg(not(target_os = "linux"))] +pub fn bytes_per_page() -> u64 { + 4096 +} diff --git a/memfaultd/src/util/task.rs b/memfaultd/src/util/task.rs new file mode 100644 index 0000000..b3ce5af --- /dev/null +++ b/memfaultd/src/util/task.rs @@ -0,0 +1,263 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::Result; +use std::time::{Duration, Instant}; + +use log::{trace, warn}; + +use super::time_measure::TimeMeasure; + +/// Run `work` every `repeat_interval` while `condition` returns true. +/// +/// On error, wait `error_retry` and multiply `error_retry` by 2 every +/// time the error is repeated but never exceeding repeat_interval. +/// +/// This is useful if you need (for example) to make a network request at a +/// fixed interval (1 hour) and want to retry sooner (1 minute) if the connection fails. +/// If the connection keeps on failing, the retry time will be increased (2min, 4min, etc). +/// +/// When the process receives a signal we will immediately check the condition +/// and run the work if the condition is still true. +/// (You have to catch the signal somewhere - otherwise the process will be terminated.) +pub fn loop_with_exponential_error_backoff< + W: FnMut() -> Result<()>, + T: FnMut() -> LoopContinuation, +>( + work: W, + condition: T, + period: Duration, + error_retry: Duration, +) { + loop_with_exponential_error_backoff_internal::<_, _, Instant>( + work, + condition, + period, + error_retry, + interruptiple_sleep, + ) +} + +// std::thread::sleep automatically continues sleeping on SIGINT but we want to be interrupted so we use shuteye::sleep. +fn interruptiple_sleep(d: Duration) { + shuteye::sleep(d); +} + +/// Specify how to continue execution +#[derive(PartialEq, Eq)] +pub enum LoopContinuation { + /// Continue running the loop normally + KeepRunning, + /// Immediately re-process the loop + RerunImmediately, + /// Stop running the loop + Stop, +} + +fn loop_with_exponential_error_backoff_internal< + W: FnMut() -> Result<()>, + T: FnMut() -> LoopContinuation, + Time: TimeMeasure, +>( + mut work: W, + mut condition: T, + period: Duration, + error_retry: Duration, + sleep: fn(Duration), +) { + const BACKOFF_MULTIPLIER: u32 = 2; + let mut count_errors_since_success = 0; + while condition() != LoopContinuation::Stop { + let start_work = Time::now(); + let next_run_in = match work() { + Ok(_) => { + count_errors_since_success = 0; + period + } + Err(e) => { + let next_run = Duration::min( + error_retry.saturating_mul( + BACKOFF_MULTIPLIER.saturating_pow(count_errors_since_success), + ), + period, + ); + + count_errors_since_success += 1; + warn!("Error in Memfaultd main loop: {}", e); + next_run + } + }; + + if condition() == LoopContinuation::KeepRunning { + let sleep_maybe = next_run_in.checked_sub(start_work.elapsed()); + if let Some(howlong) = sleep_maybe { + trace!("Sleep for {:?}", howlong); + sleep(howlong); + } + } + } +} + +#[cfg(test)] +mod tests { + use eyre::eyre; + use std::cell::{Cell, RefCell}; + + use crate::test_utils::TestInstant; + + use super::*; + use rstest::rstest; + + #[rstest] + #[case::everything_ok(vec![ + TestInvocation { + ..Default::default() + }, + TestInvocation { + expect_called_at: TestInstant::from(TEST_PERIOD), + run_time: Duration::from_millis(150), + ..Default::default() + }, + TestInvocation { + expect_called_at: TestInstant::from(TEST_PERIOD * 2), + ..Default::default() + } + ])] + #[case::errors_are_retried_sooner(vec![ + TestInvocation { + run_time: Duration::from_millis(10), + is_error: true, + ..Default::default() + }, + TestInvocation { + expect_called_at: TestInstant::from(TEST_ERROR_RETRY), + ..Default::default() + }, + TestInvocation { + expect_called_at: TestInstant::from(TEST_ERROR_RETRY + TEST_PERIOD), + ..Default::default() + } + ])] + #[case::long_runs_will_rerun_immediately(vec![ + TestInvocation { + expect_called_at: TestInstant::from(Duration::from_secs(0)), + run_time: TEST_PERIOD * 10, + ..Default::default() + }, + TestInvocation { + expect_called_at: TestInstant::from( TEST_PERIOD * 10), + ..Default::default() + } + ])] + #[case::errors_retry_backoff(vec![ + TestInvocation { + run_time: Duration::from_millis(10), + is_error: true, + ..Default::default() + }, + TestInvocation { + expect_called_at: TestInstant::from(TEST_ERROR_RETRY), + is_error: true, + ..Default::default() + }, + TestInvocation { + expect_called_at: TestInstant::from(TEST_ERROR_RETRY + TEST_ERROR_RETRY * 2), + is_error: true, + ..Default::default() + }, + TestInvocation { + expect_called_at: TestInstant::from(TEST_ERROR_RETRY + TEST_ERROR_RETRY * 2 + TEST_ERROR_RETRY * 4), + ..Default::default() + }, + TestInvocation { + expect_called_at: TestInstant::from(TEST_ERROR_RETRY + TEST_ERROR_RETRY * 2 + TEST_ERROR_RETRY * 4 + TEST_PERIOD), + is_error: true, + ..Default::default() + }, + // This one should have reset to normal error retry + TestInvocation { + expect_called_at: TestInstant::from(TEST_ERROR_RETRY + TEST_ERROR_RETRY * 2 + TEST_ERROR_RETRY * 4 + TEST_PERIOD + TEST_ERROR_RETRY), + is_error: true, + ..Default::default() + }, + ])] + #[case::can_rerun_immediately(vec![ + TestInvocation { + run_time: Duration::from_millis(10), + is_error: false, + ..Default::default() + }, + TestInvocation { + expect_called_at: TestInstant::from(Duration::from_millis(10)), + run_immediately: true, + ..Default::default() + }, + ])] + fn test_loop_with_exponential_backoff(#[case] calls: Vec) { + let step = Cell::new(0); + let call_times = RefCell::new(vec![]); + + let work = || { + let invocation = &calls[step.get()]; + + call_times.borrow_mut().push(TestInstant::now()); + step.set(step.get() + 1); + + TestInstant::sleep(invocation.run_time); + + match invocation.is_error { + true => Err(eyre!("invocation failed")), + false => Ok(()), + } + }; + // Run until we have executed all the provided steps. + let condition = || { + if step.get() < calls.len() { + // We do not need a +1 here because the step has already been incremented + // when condition is called after doing the work. + if step.get() < calls.len() && calls[step.get()].run_immediately { + LoopContinuation::RerunImmediately + } else { + LoopContinuation::KeepRunning + } + } else { + LoopContinuation::Stop + } + }; + + loop_with_exponential_error_backoff_internal::<_, _, TestInstant>( + work, + condition, + TEST_PERIOD, + TEST_ERROR_RETRY, + TestInstant::sleep, + ); + + let expected_call_times = calls + .into_iter() + .map(|c| c.expect_called_at) + .collect::>(); + assert_eq!(expected_call_times, *call_times.borrow()); + } + + #[derive(Clone)] + struct TestInvocation { + run_time: Duration, + is_error: bool, + run_immediately: bool, + expect_called_at: TestInstant, + } + impl Default for TestInvocation { + fn default() -> Self { + Self { + run_time: Duration::from_millis(30), + is_error: false, + run_immediately: false, + expect_called_at: TestInstant::from(Duration::from_millis(0)), + } + } + } + + const TEST_PERIOD: Duration = Duration::from_secs(3600); + const TEST_ERROR_RETRY: Duration = Duration::from_secs(60); +} diff --git a/memfaultd/src/util/tcp_server.rs b/memfaultd/src/util/tcp_server.rs new file mode 100644 index 0000000..2edfe41 --- /dev/null +++ b/memfaultd/src/util/tcp_server.rs @@ -0,0 +1,86 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use eyre::Result; +use log::{trace, warn}; +use std::io::Read; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::thread; +use threadpool::ThreadPool; + +/// A TCP server that spawns threads to handle incoming connections. +/// Incoming connections will be delegated to a `TcpConnectionHandler`. +pub struct ThreadedTcpServer {} + +pub trait TcpConnectionHandler: Send + Sync + Clone + 'static { + fn handle_connection(&self, stream: TcpStream) -> Result<()>; +} + +impl ThreadedTcpServer { + pub fn start( + bind_address: SocketAddr, + max_connections: usize, + handler: impl TcpConnectionHandler, + ) -> Result { + let listener = TcpListener::bind(bind_address)?; + thread::spawn(move || Self::run(listener, max_connections, handler)); + Ok(ThreadedTcpServer {}) + } + + fn run( + listener: TcpListener, + max_connections: usize, + handler: impl TcpConnectionHandler, + ) -> Result<()> { + let pool = ThreadPool::new(max_connections); + + for stream in listener.incoming() { + match stream { + Ok(stream) => { + trace!( + "Connection from {:?} - Threads {}/{}", + stream.peer_addr(), + pool.active_count(), + pool.max_count() + ); + let handler = handler.clone(); + pool.execute(move || { + if let Err(e) = handler.handle_connection(stream) { + warn!("Error while handling connection: {}", e) + } + }) + } + Err(e) => { + warn!("TCP server listener error {}", e); + break; + } + } + } + trace!("Done listening - waiting for pool to terminate"); + pool.join(); + trace!("Pool joined."); + + Ok(()) + } +} + +/// Handler that reads and drops all data. +#[derive(Clone)] +pub struct TcpNullConnectionHandler {} + +impl TcpConnectionHandler for TcpNullConnectionHandler { + fn handle_connection(&self, mut stream: TcpStream) -> Result<()> { + loop { + let mut buf = [0; 8 * 1024]; + match stream.read(&mut buf) { + Ok(0) => break, // EOF + Ok(_) => {} // drop the data + Err(e) => { + warn!("TCP read error: {:?}", e); + break; + } + } + } + Ok(()) + } +} diff --git a/memfaultd/src/util/time_measure.rs b/memfaultd/src/util/time_measure.rs new file mode 100644 index 0000000..8b15ff7 --- /dev/null +++ b/memfaultd/src/util/time_measure.rs @@ -0,0 +1,27 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::time::{Duration, Instant}; + +/// A trait for measuring time. +/// +/// This is mostly a way to mock std::time::Instant for testing. +pub trait TimeMeasure { + fn now() -> Self; + fn elapsed(&self) -> Duration; + fn since(&self, other: &Self) -> Duration; +} + +impl TimeMeasure for Instant { + fn now() -> Self { + Instant::now() + } + + fn elapsed(&self) -> Duration { + Self::now().since(self) + } + + fn since(&self, other: &Self) -> Duration { + Instant::duration_since(self, *other) + } +} diff --git a/memfaultd/src/util/zip.rs b/memfaultd/src/util/zip.rs new file mode 100644 index 0000000..96de947 --- /dev/null +++ b/memfaultd/src/util/zip.rs @@ -0,0 +1,487 @@ +// +// Copyright (c) Memfault, Inc. +// See License.txt for details +use std::fs::File; +use std::io::Cursor; +use std::io::Read; +use std::os::unix::ffi::OsStrExt; +use std::path::{Path, PathBuf}; +#[cfg(test)] +use std::str::from_utf8; + +use crate::util::io::StreamLen; +use eyre::Result; +use flate2::CrcReader; +use take_mut::take; + +/// Minimalistic zip encoder which generates a compression-less zip stream on the fly from a list of +/// files. The benefit of this is that it requires no temporary file storage. It implements the +/// std::io::Read trait, so it can be used with any std::io::Read consumer. It can also tell the +/// length of the stream beforehand, only looking at the list of files and their sizes on disk. +/// This is useful for example when needing to specify a Content-Length header for a HTTP request. +/// Note it is very minimalistic in its implementation: it only supports "store" (no compression). +/// It only supports the 32-bit zip format, so it is limited to max. 4GB file sizes and does not +/// allow for more than 65,535 entries. File timestamps are not implemented and neither are UTF-8 +/// filenames. +/// Note that read() calls can copy less than the size of the caller's buffer, due to an +/// implementation detail. Therefore it's recommended to use std::io::BufReader to wrap this stream. + +// Some implementation notes: +// - The zip format is described here: https://en.wikipedia.org/wiki/ZIP_(file_format) +// - For each file in the zip file, the contents of the file is prepended with a "local file header" +// and suffixed with a "data descriptor". The local file header contains some metadata of the file +// that follows and the data descriptor contains additional metadata that is conveniently gathered +// while reading/writing the contents, like the CRC32 checksum of the file's contents. +// - After all files have been written, a "central directory" is written, which contains all the +// metadata of the files again, but in a slightly different, more elaborate format. This is used +// by the decoder/unarchiver to quickly access the list of files in the zip file. + +pub struct ZipEncoder { + files: Vec, + state: ZipEncoderState, + bytes_written: usize, +} + +enum ZipEncoderState { + Init, + LocalFiles { + index: usize, + reader: LocalFileReader, + }, + CentralDirectory { + index: usize, + start_offset: usize, + reader: Cursor>, + }, + EndOfCentralDirectory { + reader: Cursor>, + }, + Done, +} + +impl ZipEncoder { + /// Creates a new ZipEncoder from a list of source files that should be included in the zip. + pub fn new(files: Vec) -> Self { + Self { + files, + state: ZipEncoderState::Init, + bytes_written: 0, + } + } + + fn new_local_files_state(&self, index: usize) -> std::io::Result { + Ok(ZipEncoderState::LocalFiles { + index, + reader: LocalFileReader::new(&self.files[index])?, + }) + } + + fn new_central_directory_state(&self, index: usize, start_offset: usize) -> ZipEncoderState { + ZipEncoderState::CentralDirectory { + index, + start_offset, + reader: Cursor::new(make_file_header( + &self.files[index], + FileHeaderKind::CentralDirectory, + )), + } + } + + fn new_end_of_central_directory_state(&self, start_offset: usize) -> ZipEncoderState { + let num_files = self.files.len(); + ZipEncoderState::EndOfCentralDirectory { + reader: Cursor::new(make_end_of_central_directory( + num_files as u16, + (self.bytes_written - start_offset) as u32, + start_offset as u32, + )), + } + } + + #[cfg(test)] + pub fn file_names(&self) -> Vec<&str> { + self.files + .iter() + .map(|f| from_utf8(f.name.as_slice()).unwrap()) + .collect() + } +} + +impl StreamLen for ZipEncoder { + /// Length of the zip stream in bytes. + fn stream_len(&self) -> u64 { + zip_stream_len(&self.files) as u64 + } +} + +impl Read for ZipEncoder { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + loop { + let reader: &mut dyn Read = match &mut self.state { + ZipEncoderState::Init => { + if self.files.is_empty() { + self.state = self.new_end_of_central_directory_state(0); + } else { + self.state = self.new_local_files_state(0)?; + } + continue; + } + ZipEncoderState::LocalFiles { reader, .. } => reader, + ZipEncoderState::CentralDirectory { reader, .. } => reader, + ZipEncoderState::EndOfCentralDirectory { reader } => reader, + ZipEncoderState::Done => { + return Ok(0); + } + }; + let n = reader.read(buf)?; + self.bytes_written += n; + if n > 0 || buf.is_empty() { + return Ok(n); + } + self.state = match &self.state { + ZipEncoderState::Init => unreachable!(), + ZipEncoderState::LocalFiles { index, reader } => { + if let Some(crc) = reader.crc() { + self.files[*index].crc = crc; + let next_index = *index + 1; + if next_index < self.files.len() { + self.files[next_index].offset = self.bytes_written as u32; + self.new_local_files_state(next_index)? + } else { + self.new_central_directory_state(0, self.bytes_written) + } + } else { + // Will never panic because the above read() call returned Ok(0), which + // means the LocalFileReader is in the LocalFileReaderState::Done state. + unreachable!() + } + } + ZipEncoderState::CentralDirectory { + index, + start_offset, + .. + } => { + let next_index = *index + 1; + if next_index < self.files.len() { + self.new_central_directory_state(next_index, *start_offset) + } else { + self.new_end_of_central_directory_state(*start_offset) + } + } + ZipEncoderState::EndOfCentralDirectory { .. } => ZipEncoderState::Done, + ZipEncoderState::Done => unreachable!(), + } + } + } +} + +#[derive(Clone)] +pub struct ZipEntryInfo { + path: PathBuf, + name: Vec, + size: u64, + /// Offset from the start of the file to the local file header. + /// Gets filled in by ZipEncoder before it reads the file. + offset: u32, + // Checksum of the (uncompressed) file data. + // Gets filled in by ZipEncoder after the file is read. + crc: u32, +} + +impl ZipEntryInfo { + /// Creates a new ZipEntryInfo from a path to a file. + /// The path must be relative to the base path. The base path is used to determine the name of + /// the file in the zip, by stripping the base path from the file path. + pub fn new(path: PathBuf, base: &Path) -> Result { + let name = path.strip_prefix(base)?.as_os_str().as_bytes().to_owned(); + let metadata = path.metadata()?; + Ok(Self { + path, + name, + size: metadata.len(), + crc: 0, + offset: 0, + }) + } +} + +#[derive(Clone, Copy)] +enum FileHeaderKind { + Local, + CentralDirectory, +} + +pub fn zip_stream_len_empty() -> usize { + END_OF_CENTRAL_DIRECTORY_SIZE +} + +/// Returns the size of the entire zip stream in bytes, given a slice of ZipEntryInfo. +pub fn zip_stream_len(files: &[ZipEntryInfo]) -> usize { + files.iter().map(zip_stream_len_for_file).sum::() + zip_stream_len_empty() +} + +/// Returns the size that a single ZipEntryInfo contributes to the size of the zip stream. +pub fn zip_stream_len_for_file(file_info: &ZipEntryInfo) -> usize { + header_size(file_info, FileHeaderKind::Local) + + file_info.size as usize + + DATA_DESCRIPTOR_SIZE + + header_size(file_info, FileHeaderKind::CentralDirectory) +} + +fn header_size(info: &ZipEntryInfo, kind: FileHeaderKind) -> usize { + const LOCAL_FILE_HEADER_SIZE: usize = 30; + const DIRECTORY_HEADER_SIZE: usize = 46; + let name_len = info.name.len(); + match kind { + FileHeaderKind::Local => LOCAL_FILE_HEADER_SIZE + name_len, + FileHeaderKind::CentralDirectory => DIRECTORY_HEADER_SIZE + name_len, + } +} + +fn make_file_header(info: &ZipEntryInfo, kind: FileHeaderKind) -> Vec { + let mut header = Vec::with_capacity(header_size(info, kind)); + header.extend_from_slice(match &kind { + FileHeaderKind::Local => { + // Signature + // Version needed to extract + // General purpose bit flag (data descriptor enabled) + // Compression mode (store / no compression) + // File last modified time (all zeroes) + // File last modified date (all zeroes) + b"PK\x03\x04\ + \x0A\x00\ + \x08\x00\ + \x00\x00\ + \x00\x00\ + \x00\x00\ + " + } + FileHeaderKind::CentralDirectory => { + // Signature + // Version made by + // Version needed to extract + // General purpose bit flag (data descriptor enabled) + // Compression mode (store / no compression) + // File last modified time (all zeroes) + // File last modified date (all zeroes) + b"PK\x01\x02\ + \x0A\x00\ + \x0A\x00\ + \x08\x00\ + \x00\x00\ + \x00\x00\ + \x00\x00\ + " + } + }); + // CRC-32 of uncompressed data: + header.extend_from_slice(&info.crc.to_le_bytes()); + let size_slice = &(info.size as u32).to_le_bytes(); + // Compressed size: + header.extend_from_slice(size_slice); + // Uncompressed size: + header.extend_from_slice(size_slice); + // File name length: + header.extend_from_slice(&(info.name.len() as u16).to_le_bytes()); + // Extra field length (0 bytes): + header.extend_from_slice(b"\x00\x00"); + + if let FileHeaderKind::CentralDirectory = &kind { + // File comment length (0 bytes) + // Disk number where file starts (0) + // Internal file attributes (0) + // External file attributes (0) + header.extend_from_slice( + b"\x00\x00\ + \x00\x00\ + \x00\x00\ + \x00\x00\x00\x00\ + ", + ); + + // Relative offset of local file header: + header.extend_from_slice(&info.offset.to_le_bytes()); + }; + + // File name: + header.extend_from_slice(&info.name); + header +} + +const DATA_DESCRIPTOR_SIZE: usize = 16; + +fn make_data_descriptor(crc: u32, size: u32) -> Vec { + let mut desc = Vec::with_capacity(DATA_DESCRIPTOR_SIZE); + // Signature: + desc.extend_from_slice(b"PK\x07\x08"); + // CRC-32 of uncompressed data: + desc.extend_from_slice(&crc.to_le_bytes()); + let size_slice = &size.to_le_bytes(); + // Compressed size: + desc.extend_from_slice(size_slice); + // Uncompressed size: + desc.extend_from_slice(size_slice); + desc +} + +const END_OF_CENTRAL_DIRECTORY_SIZE: usize = 22; + +fn make_end_of_central_directory(num_files: u16, size: u32, offset: u32) -> Vec { + let mut desc = Vec::with_capacity(END_OF_CENTRAL_DIRECTORY_SIZE); + desc.extend_from_slice( + // Signature + // Number of this disk + // Disk where central directory starts + b"PK\x05\x06\ + \x00\x00\ + \x00\x00\ + ", + ); + let num_files_slice = &num_files.to_le_bytes(); + // Number of central directory records on this disk: + desc.extend_from_slice(num_files_slice); + // Total number of central directory records + desc.extend_from_slice(num_files_slice); + // Size of central directory + desc.extend_from_slice(&size.to_le_bytes()); + // Offset of start of central directory + desc.extend_from_slice(&offset.to_le_bytes()); + // Comment length + desc.extend_from_slice(b"\x00\x00"); + desc +} + +struct LocalFileReader { + state: LocalFileReaderState, +} + +enum LocalFileReaderState { + Header { reader: Cursor>, file: File }, + Data { reader: CrcReader }, + DataDescriptor { reader: Cursor>, crc: u32 }, + Done { crc: u32 }, +} + +impl LocalFileReader { + pub fn new(info: &ZipEntryInfo) -> std::io::Result { + Ok(Self { + state: LocalFileReaderState::Header { + reader: Cursor::new(make_file_header(info, FileHeaderKind::Local)), + file: File::open(&info.path)?, + }, + }) + } + + pub fn crc(&self) -> Option { + match self.state { + LocalFileReaderState::Done { crc } => Some(crc), + _ => None, + } + } +} + +impl Read for LocalFileReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + loop { + let reader: &mut dyn Read = match &mut self.state { + LocalFileReaderState::Header { reader, .. } => reader, + LocalFileReaderState::Data { reader } => reader, + LocalFileReaderState::DataDescriptor { reader, .. } => reader, + LocalFileReaderState::Done { .. } => { + return Ok(0); + } + }; + let n = reader.read(buf)?; + if n > 0 || buf.is_empty() { + return Ok(n); + } + take(&mut self.state, |state| match state { + LocalFileReaderState::Header { file, .. } => LocalFileReaderState::Data { + reader: CrcReader::new(file), + }, + LocalFileReaderState::Data { reader: crc_reader } => { + let crc = crc_reader.crc().sum(); + LocalFileReaderState::DataDescriptor { + reader: Cursor::new(make_data_descriptor(crc, crc_reader.crc().amount())), + crc, + } + } + LocalFileReaderState::DataDescriptor { crc, .. } => { + LocalFileReaderState::Done { crc } + } + LocalFileReaderState::Done { .. } => unreachable!(), + }) + } + } +} + +#[cfg(test)] +mod tests { + use std::io::{copy, Cursor}; + use std::io::{Seek, SeekFrom}; + + use crate::test_utils::create_file_with_contents; + use tempfile::tempdir; + use zip::ZipArchive; + + use super::*; + + /// Makes an empty zip file, then reads it with a "known good" zip unarchiver ('zip' crate). + #[test] + fn test_empty() { + let (zip, zip_encoder) = zip_round_trip(vec![]); + assert!(zip.is_empty()); + + let stream_len = zip.into_inner().into_inner().len(); + assert_eq!(stream_len as u64, zip_encoder.stream_len()); + } + + /// Makes a zip with some files, then reads it with a "known good" zip unarchiver ('zip' crate). + #[test] + fn test_basic() { + let tmp = tempdir().unwrap(); + let tempdir_path = tmp.path(); + + let filenames_and_contents = [("hello.txt", "Hello World"), ("bye.txt", "Goodbye")]; + + for (filename, contents) in filenames_and_contents.iter() { + let file_path = tempdir_path.join(filename); + create_file_with_contents(&file_path, contents.as_bytes()).unwrap(); + } + + let file_infos = filenames_and_contents + .iter() + .map(|(filename, _)| { + let file_path = tempdir_path.join(filename); + ZipEntryInfo::new(file_path, tempdir_path).unwrap() + }) + .collect::>(); + + let (mut zip, zip_encoder) = zip_round_trip(file_infos); + assert_eq!(zip.len(), filenames_and_contents.len()); + + for (filename, contents) in filenames_and_contents.iter() { + let vec: Vec = Vec::with_capacity(1024); + let mut cursor = Cursor::new(vec); + copy(&mut zip.by_name(filename).unwrap(), &mut cursor).unwrap(); + assert_eq!(cursor.into_inner(), contents.as_bytes()); + } + + let stream_len = zip.into_inner().into_inner().len(); + assert_eq!(stream_len as u64, zip_encoder.stream_len()); + } + + fn zip_round_trip( + source_files: Vec, + ) -> (ZipArchive>>, ZipEncoder) { + let mut zip_encoder = ZipEncoder::new(source_files); + let vec: Vec = Vec::with_capacity(1024 * 8); + let mut cursor = Cursor::new(vec); + copy(&mut zip_encoder, &mut cursor).unwrap(); + + // Use flate2's ZipArchive to read the zip file we just created: + cursor.seek(SeekFrom::Start(0)).unwrap(); + let zip = ZipArchive::new(cursor).unwrap(); + (zip, zip_encoder) + } +}