From 2c5197965693b8a2ac79f74d4f44f778deb952a7 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Mon, 28 Oct 2024 14:17:26 -0700 Subject: [PATCH] [omdb] Add `db vmm list` and `db vmm info` (#6936) `omdb db` has commands for listing and displaying details regarding customer VM instances. These commands reference the Propolis VMM processes whose states are recorded in the `vmm` table, including their IDs. However, there are no commands for directly querying that table, which is unfortunate. While debugging an issue on the colo rack last week, @augustuswm and I wanted to be able to list all VMMs in the `Failed` state, but the only way to do this was to find a CockroachDB zone and run our own manual SQL queries. Debugging this type of issue would be easier if there were OMDB commands for querying the VMM table. This commit adds two new commands to OMDB, `omdb db vmm list` (aliases `db vmm ls`, `db vmms`), which lists entries from the `vmm` table (potentially filtered by state), and `omdb db vmm info` (alias `db vmm show`), which looks up a VMM record by UUID and displays its contents. In addition, I factored out some code for displaying VMM records that was shared between the `db vmm info` command and the `db instance info` command. Closes #6928 ## Examples
Querying a VMM record with `omdb db vmm info`: ```console root@oxz_switch1:~# /var/tmp/omdb-eliza-vmms-6 db vmm info 040678cf-2e31-4bc6-baa2-79b20f40ca83 2>/dev/null == VMM ========================================================================= ID: 040678cf-2e31-4bc6-baa2-79b20f40ca83 instance ID: 8ee8bf04-9435-49bb-a8e7-b1368315359a created at: 2024-10-23 20:02:34.980041 UTC state: running updated at: 2024-10-23T20:02:50.078970Z (generation 4) propolis address: fd00:1122:3344:104::1:417:12400 sled ID: 0c7011f7-a4bf-4daf-90cc-1c2410103300 sled serial: BRM42220057 == SLED RESOURCE RESERVATIONS ================================================== hardware threads: 2 RSS RAM: 0 B reservoir RAM: 8 GiB root@oxz_switch1:~# ```
Listing VMMs with `omdb db vmm list`: ```console root@oxz_switch1:~# /var/tmp/omdb-eliza-vmms-6 db vmm ls 2>/dev/null INSTANCE_ID ID STATE GEN SLED 8ee8bf04-9435-49bb-a8e7-b1368315359a 040678cf-2e31-4bc6-baa2-79b20f40ca83 running 4 BRM42220057 6dd05e98-7a36-4799-8a0b-97b7ab101a05 6ff33c2d-3291-4d61-a3b5-43e8203c1b41 running 4 BRM42220057 6ead53ad-e997-4b85-86b8-342905a68ada ae6e42a2-37af-4bbe-80b2-e4515482018d running 4 BRM42220057 1c8803ac-d1a1-4e2f-a827-fca538d5cbb3 c24058bd-c747-44dd-906a-d2d1bc46755d running 4 BRM42220057 228b79fd-b4ff-4f50-97a5-286c949e695d c70b5536-8bdc-4bdf-810b-4d3836ea2947 running 4 BRM42220057 e97e9fb9-62c3-4745-9d91-b0b6fa2baeba fa525216-9f81-4a8f-8fca-f7858994dce2 starting 3 BRM42220057 5fad8b64-1631-4c7e-8cb1-ed089830d406 6e637a89-2bb1-4327-89ac-6f55757fd3ec running 4 BRM44220011 16620dff-d4e5-47ba-b11d-34c59fb01206 770f0d5f-5f87-4d8d-a4d7-7b16602f2408 running 4 BRM44220011 33e6d4a4-6cc7-440f-8a9f-152d94136191 31837146-7de7-4317-9072-cd448b95f56c running 4 BRM42220017 ad5a6c89-2845-4c2e-b247-8ca034e10597 92c59a48-f56e-48f0-9792-88a664072a26 running 4 BRM42220017 0ec2a7ac-fcc5-4bf4-9adc-f05c6823ee93 ca5d77ff-8c37-4740-86fd-89c855fb5e5b running 4 BRM42220017 09bbbb51-757c-4cfc-ac25-b619b8523981 4e65bb95-29ca-4063-b4a4-ef73521a8185 running 4 BRM42220051 1a0effb3-1338-409f-a8ad-99583203dd7e 5f521878-98f3-441a-9674-69987e29666a running 4 BRM42220051 9ecfb1d7-0526-47a9-8b58-2f7d639174ef 62366fdc-3ed6-41e4-8049-dd1b5be29fae running 4 BRM42220051 eb3a39f5-6657-487b-bfb6-5e540ff07b35 a8e83499-eebc-4e4f-a05e-4b24049304e7 running 4 BRM42220051 764977e8-5e6d-4e62-ad72-82bd55948289 dfaabeb6-a621-46f3-ad65-740d6732c045 running 4 BRM42220051 fa658925-3e4d-4e76-b839-53a62beb4f5d 446be12a-2cb9-4b16-aaa7-31720c054304 running 4 BRM44220010 ac595ce5-3a9a-4bb0-8daf-f64d09686525 d6f8970e-8b24-4c6b-8b49-6c8bcefe3d83 running 4 BRM44220010 6b252c74-548c-4f1b-a98a-6c0e2da02c09 0672e7ff-918c-4fcb-b8b7-04d66a3e10dd running 4 BRM42220014 cdffcf35-6ae3-488d-a03d-64cf45f88fb2 5c67e0e6-38b6-4fce-ab08-8e0d348b86ee running 4 BRM42220014 db4aca25-6828-4d7e-8744-83336110e82b bf8faa13-5c4a-4e21-86a8-e2fa6d198c3a running 4 BRM42220014 9e2451c0-6eec-4b90-b49a-6a9a2e02f35f 28dce81a-cf61-4001-a5da-78349926521f running 4 BRM42220031 2cae6a66-1e62-4403-98d0-a5384726cc91 80e80ef2-9948-45ce-9852-eda7d3fc7723 running 4 BRM42220031 18cdcbf1-41b3-4f8c-8395-d8191c916fb3 ceed88d0-6a3e-4ff9-abf7-ce1162631c34 running 4 BRM42220031 ea20be02-a257-467b-83e7-f16f4db68f0e 4fe59185-cb8f-498d-b17f-6b6bfdf92591 running 4 BRM42220006 1cd1945f-c86b-47eb-8b0e-5e50f7b1ea24 db150e5d-dd71-4ccc-a79d-d9188e6ab3ff running 4 BRM42220006 923477a5-1183-48aa-80cc-bac548521f31 ec1b6428-6f1d-400f-aba1-1a442ea55295 running 4 BRM42220006 e3795c9a-1009-4b4b-a706-a94987220a7e ee25dd9f-244b-4512-9417-16dc0aa114e6 running 4 BRM42220006 372b637b-a583-499d-8ac0-21dd6cc8e1ba 99393822-556c-4e09-b3a6-4f5c2750fcce running 4 BRM42220016 b30ae7ac-9092-4c01-8586-23ed388d2363 195138f1-6036-4a44-ad83-c2cfae871641 running 4 BRM44220005 18c8ee75-b51e-4d9e-bfc5-f9d31ee27184 2ac06d55-b2f9-4f66-9eb0-a76e5f77812d running 4 BRM44220005 f93288a6-acbb-4396-a4da-d5a33e189577 79aed6d7-4e47-436a-ad61-8caf86a163ba running 4 BRM44220005 1669ecf5-c3ef-430f-aabc-eb010d5fda39 92a82706-8bc9-48da-8af1-23f91104f8be running 4 BRM44220005 d07b0fcc-17d0-479c-9688-1bc5731f0d6a c44e9370-5af6-400d-b05f-eb038892c3dc running 4 BRM44220005 c9e210b9-037e-4433-b904-1a20ac7052a6 d6a159d4-d13c-4115-a340-3bf359548783 running 4 BRM44220005 b0db674c-de4f-49df-bdce-965632f61915 d6e699ad-bb6a-47b0-a772-864e94dd09b8 starting 3 BRM44220005 ece2768b-2820-463e-b92c-268eaa42c27e d8a57699-1be7-41bf-9baf-a590e626ac37 running 4 BRM44220005 ```
Listing VMMs by state:
```console root@oxz_switch1:~# /var/tmp/omdb-eliza-vmms-6 db vmm ls 2>/dev/null --state starting INSTANCE_ID ID STATE GEN SLED e97e9fb9-62c3-4745-9d91-b0b6fa2baeba fa525216-9f81-4a8f-8fca-f7858994dce2 starting 3 BRM42220057 b0db674c-de4f-49df-bdce-965632f61915 d6e699ad-bb6a-47b0-a772-864e94dd09b8 starting 3 BRM44220005 ```
Listing "failed" or "destroyed" VMMs suggests adding `--include-deleted`, because VMMs in those states may be soft-deleted: ```console root@oxz_switch1:~# /var/tmp/omdb-eliza-vmms-6 db vmm ls --state failed note: database URL not specified. Will search DNS. note: (override with --db-url or OMDB_DB_URL) note: using DNS server for subnet fd00:1122:3344::/48 note: (if this is not right, use --dns-server to specify an alternate DNS server) note: using database URL postgresql://root@[fd00:1122:3344:109::3]:32221,[fd00:1122:3344:105::3]:32221,[fd00:1122:3344:10b::3]:32221,[fd00:1122:3344:107::3]:32221,[fd00:1122:3344:108::3]:32221/omicron?sslmode=disable WARN: found schema version 110.0.0, expected 111.0.0 It's possible the database is running a version that's different from what this tool understands. This may result in errors or incorrect output. WARN: VMMs in the `Failed` state may have been deleted, but `--include-deleted` was not specified INSTANCE_ID ID STATE GEN SLED 08T04:41:51.913Z 2024-10-08T04:41:51.913Z root@oxz_switch1:~# /var/tmp/omdb-eliza-vmms-6 db vmm ls --state failed --include-deleted --fetch-limit 10 note: database URL not specified. Will search DNS. note: (override with --db-url or OMDB_DB_URL) note: using DNS server for subnet fd00:1122:3344::/48 note: (if this is not right, use --dns-server to specify an alternate DNS server) note: using database URL postgresql://root@[fd00:1122:3344:109::3]:32221,[fd00:1122:3344:105::3]:32221,[fd00:1122:3344:10b::3]:32221,[fd00:1122:3344:107::3]:32221,[fd00:1122:3344:108::3]:32221/omicron?sslmode=disable WARN: found schema version 110.0.0, expected 111.0.0 It's possible the database is running a version that's different from what this tool understands. This may result in errors or incorrect output. WARN: loading VMMs: found 10 items (the limit). There may be more items that were ignored. Consider overriding with --fetch-limit. INSTANCE_ID ID STATE GEN SLED TIME_DELETED TIME_DELETED 740c1640-f393-4336-8b2f-94f6a4cb7739 009555fe-cbbc-4685-a3a0-a9421d9eca35 failed 5 BRM42220017 2024-10-11T00:15:55.583Z 2024-10-11T00:15:55.583Z eb3a39f5-6657-487b-bfb6-5e540ff07b35 00aa5a57-91e5-4a1a-bd7e-677a2b53d272 failed 5 BRM42220017 2024-10-22T15:57:42.972Z 2024-10-22T15:57:42.972Z 53b5ddc3-31f2-4bd4-8fb3-5f3a6d895ee1 031bdbcb-d8ed-4eef-bbe8-717d97141162 failed 6 BRM42220051 2024-10-11T00:16:03.142Z 2024-10-11T00:16:03.142Z 49520f14-e4f9-49b7-ad41-cea1f3ac0deb 0330bafd-e27c-44ed-855b-155ab8803821 failed 6 BRM42220051 2024-10-02T18:19:07.577Z 2024-10-02T18:19:07.577Z ca808ed8-01d9-49d4-9f5b-578af1720ab2 02720bfe-84a5-4185-a2b6-319fce430b6c failed 6 BRM44220010 2024-10-11T00:16:12.615Z 2024-10-11T00:16:12.615Z 060589b6-f22b-47f4-a27b-666c49990a5b 0342228b-e590-4bb9-a1de-98848494942a failed 5 BRM42220031 2024-10-08T04:41:50.745Z 2024-10-08T04:41:50.745Z 9fa2efa0-a2a5-4ce0-98ce-5de4dd7166ee 036740c5-88a1-4ed7-8340-cc972ce0e96b failed 6 BRM42220031 2024-10-09T04:32:35.365Z 2024-10-09T04:32:35.365Z 9e2451c0-6eec-4b90-b49a-6a9a2e02f35f 00211b27-b9a8-4771-b608-3f8fb117c4f7 failed 5 BRM42220006 2024-09-29T22:14:09.703Z 2024-09-29T22:14:09.703Z a2dbe0a0-c9ea-43df-b08c-7c418da21fcf 045df015-3229-4454-87ea-f88b593c5746 failed 6 BRM42220006 2024-10-02T18:19:44.825Z 2024-10-02T18:19:44.825Z 0b492c84-c1b1-45a0-b8fc-bb5b4f79bac6 01e1d5c9-e564-4390-bc8e-cc45b9e98134 failed 5 BRM42220006 2024-10-07T05:41:46.991Z 2024-10-07T05:41:46.991Z ```
`--verbose` includes more contents in the table (for wide terminal windows/small font sizes!): ```console root@oxz_switch1:~# /var/tmp/omdb-eliza-vmms-6 db vmm ls --verbose --fetch-limit 5 note: database URL not specified. Will search DNS. note: (override with --db-url or OMDB_DB_URL) note: using DNS server for subnet fd00:1122:3344::/48 note: (if this is not right, use --dns-server to specify an alternate DNS server) note: using database URL postgresql://root@[fd00:1122:3344:109::3]:32221,[fd00:1122:3344:105::3]:32221,[fd00:1122:3344:10b::3]:32221,[fd00:1122:3344:107::3]:32221,[fd00:1122:3344:108::3]:32221/omicron?sslmode=disable WARN: found schema version 110.0.0, expected 111.0.0 It's possible the database is running a version that's different from what this tool understands. This may result in errors or incorrect output. WARN: loading VMMs: found 5 items (the limit). There may be more items that were ignored. Consider overriding with --fetch-limit. INSTANCE_ID ID STATE GEN SLED SLED_ID ADDRESS TIME_CREATED TIME_UPDATED 8ee8bf04-9435-49bb-a8e7-b1368315359a 040678cf-2e31-4bc6-baa2-79b20f40ca83 running 4 BRM42220057 0c7011f7-a4bf-4daf-90cc-1c2410103300 [fd00:1122:3344:104::1:417]:12400 2024-10-23T20:02:34.980Z 2024-10-23T20:02:50.078Z 6dd05e98-7a36-4799-8a0b-97b7ab101a05 6ff33c2d-3291-4d61-a3b5-43e8203c1b41 running 4 BRM42220057 0c7011f7-a4bf-4daf-90cc-1c2410103300 [fd00:1122:3344:104::1:41b]:12400 2024-10-23T20:15:51.922Z 2024-10-23T20:16:05.275Z 6ead53ad-e997-4b85-86b8-342905a68ada ae6e42a2-37af-4bbe-80b2-e4515482018d running 4 BRM42220057 0c7011f7-a4bf-4daf-90cc-1c2410103300 [fd00:1122:3344:104::1:418]:12400 2024-10-23T20:02:35.211Z 2024-10-23T20:07:25.195Z 1c8803ac-d1a1-4e2f-a827-fca538d5cbb3 c24058bd-c747-44dd-906a-d2d1bc46755d running 4 BRM42220057 0c7011f7-a4bf-4daf-90cc-1c2410103300 [fd00:1122:3344:104::1:41a]:12400 2024-10-23T20:06:22.391Z 2024-10-23T20:07:24.170Z 228b79fd-b4ff-4f50-97a5-286c949e695d c70b5536-8bdc-4bdf-810b-4d3836ea2947 running 4 BRM42220057 0c7011f7-a4bf-4daf-90cc-1c2410103300 [fd00:1122:3344:104::1:419]:12400 2024-10-23T20:06:06.612Z 2024-10-23T20:07:35.934Z root@oxz_switch1:~# ```
--- dev-tools/omdb/src/bin/omdb/db.rs | 550 +++++++++++++++++++++++--- dev-tools/omdb/tests/usage_errors.out | 6 + nexus/db-model/src/instance_state.rs | 22 +- nexus/db-model/src/vmm_state.rs | 55 ++- 4 files changed, 576 insertions(+), 57 deletions(-) diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs index ac25a96d7e..3a5281a3a8 100644 --- a/dev-tools/omdb/src/bin/omdb/db.rs +++ b/dev-tools/omdb/src/bin/omdb/db.rs @@ -16,6 +16,7 @@ #![allow(clippy::useless_vec)] use crate::check_allow_destructive::DestructiveOperationToken; +use crate::helpers::const_max_len; use crate::helpers::CONNECTION_OPTIONS_HEADING; use crate::helpers::DATABASE_OPTIONS_HEADING; use crate::Omdb; @@ -30,6 +31,7 @@ use chrono::SecondsFormat; use chrono::Utc; use clap::builder::PossibleValue; use clap::builder::PossibleValuesParser; +use clap::builder::TypedValueParser; use clap::ArgAction; use clap::Args; use clap::Subcommand; @@ -138,7 +140,6 @@ use std::fmt::Display; use std::num::NonZeroU32; use std::sync::Arc; use strum::IntoEnumIterator; -use strum::VariantArray; use tabled::Tabled; use uuid::Uuid; @@ -326,6 +327,11 @@ enum DbCommands { Validate(ValidateArgs), /// Print information about volumes Volumes(VolumeArgs), + /// Print information about Propolis virtual machine manager (VMM) + /// processes. + Vmm(VmmArgs), + /// Alias to `omdb db vmm list`. + Vmms(VmmListArgs), } #[derive(Debug, Args)] @@ -438,10 +444,11 @@ struct InstanceListArgs { long = "state", conflicts_with = "running", value_parser = PossibleValuesParser::new( - db::model::InstanceState::VARIANTS + db::model::InstanceState::ALL_STATES .iter() .map(|v| PossibleValue::new(v.label())) - ), + ).try_map(|s| s.parse::()), + action = ArgAction::Append, )] states: Vec, } @@ -798,6 +805,53 @@ struct VolumeInfoArgs { uuid: Uuid, } +#[derive(Debug, Args)] +struct VmmArgs { + #[command(subcommand)] + command: VmmCommands, +} + +#[derive(Debug, Subcommand)] +enum VmmCommands { + /// Get info for a specific VMM process + #[clap(alias = "show")] + Info(VmmInfoArgs), + /// List VMM processes + #[clap(alias = "ls")] + List(VmmListArgs), +} + +#[derive(Debug, Args)] +struct VmmInfoArgs { + /// The UUID of the VMM process. + uuid: Uuid, +} + +#[derive(Debug, Args)] +struct VmmListArgs { + /// Enable verbose output. + /// + /// You may need a really wide monitor for this! + #[arg(long, short)] + verbose: bool, + + /// Only show VMMs in the provided state(s). + /// + /// By default, all VMM states are selected. + #[arg( + short, + long = "state", + value_parser = PossibleValuesParser::new( + db::model::VmmState::ALL_STATES + .iter() + .map(|v| PossibleValue::new(v.label())) + ) + .try_map(|s| s.parse::()), + action = ArgAction::Append, + )] + states: Vec, +} + impl DbArgs { /// Run a `omdb db` subcommand. pub(crate) async fn run_cmd( @@ -1030,6 +1084,15 @@ impl DbArgs { DbCommands::Volumes(VolumeArgs { command: VolumeCommands::List, }) => cmd_db_volume_list(&datastore, &self.fetch_opts).await, + + DbCommands::Vmm(VmmArgs { command: VmmCommands::Info(args) }) => { + cmd_db_vmm_info(&opctx, &datastore, &self.fetch_opts, &args) + .await + } + DbCommands::Vmm(VmmArgs { command: VmmCommands::List(args) }) + | DbCommands::Vmms(args) => { + cmd_db_vmm_list(&datastore, &self.fetch_opts, args).await + } }; datastore.terminate().await; res @@ -2925,7 +2988,7 @@ async fn cmd_db_instance_info( }; use nexus_db_model::{ Instance, InstanceKarmicStatus, InstanceRuntimeState, Migration, - Reincarnatability, Vmm, + Reincarnatability, }; let &InstanceInfoArgs { ref id, history } = args; @@ -3102,6 +3165,7 @@ async fn cmd_db_instance_info( println!(" {ACTIVE_VMM:>WIDTH$}: {propolis_id:?}"); println!(" {TARGET_VMM:>WIDTH$}: {dst_propolis_id:?}"); + println!( "{}{MIGRATION_ID:>WIDTH$}: {migration_id:?}", if migration_id.is_some() { "(i) " } else { " " }, @@ -3113,16 +3177,14 @@ async fn cmd_db_instance_info( } println!(" at generation: {}", instance.updater_gen.0); - fn print_vmm(slug: &str, kind: &str, id: Uuid, vmm: Option<&Vmm>) { + fn print_vmm(kind: &str, id: Uuid, vmm: Option<&Vmm>) { match vmm { Some(vmm) => { println!( - "\n {slug:>WIDTH$}:\n{}", - textwrap::indent( - &format!("{vmm:#?}"), - &" ".repeat(WIDTH - slug.len() + 8) - ) + "\n{:=<80}", + format!("== {} VMM ", kind.to_ascii_uppercase()) ); + prettyprint_vmm(" ", vmm, Some(WIDTH), None, true); if vmm.time_deleted.is_some() { eprintln!( "\n/!\\ BAD: dangling foreign key to deleted {kind} \ @@ -3140,7 +3202,7 @@ async fn cmd_db_instance_info( } if let Some(id) = propolis_id { - print_vmm(ACTIVE_VMM_RECORD, "active", id, active_vmm.as_ref()); + print_vmm("active", id, active_vmm.as_ref()); } if let Some(id) = dst_propolis_id { @@ -3153,7 +3215,7 @@ async fn cmd_db_instance_info( match fetch_result { Ok(rs) => { let vmm = rs.into_iter().next(); - print_vmm(TARGET_VMM_RECORD, "target", id, vmm.as_ref()); + print_vmm("target", id, vmm.as_ref()); } Err(e) => { eprintln!("error looking up target VMM record {id}: {e}"); @@ -3216,7 +3278,7 @@ async fn cmd_db_instance_info( #[derive(Tabled)] #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] struct DiskRow { - #[tabled(display_with = "display_option_blank")] + #[tabled(rename = "#", display_with = "display_option_blank")] slot: Option, #[tabled(inline)] identity: DiskIdentity, @@ -3244,7 +3306,7 @@ async fn cmd_db_instance_info( } if !disks.is_empty() { - println!("\n{:=<80}\n", "== ATTACHED DISKS"); + println!("\n{:=<80}\n", "== ATTACHED DISKS "); check_limit(&disks, fetch_opts.fetch_limit, ctx); let table = if fetch_opts.include_deleted { @@ -3298,6 +3360,17 @@ async fn cmd_db_instance_info( } let ctx = || "listing past VMMs"; + #[derive(Tabled)] + #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] + struct VmmRow { + #[tabled(inline)] + state: VmmStateRow, + sled_id: Uuid, + #[tabled(display_with = "datetime_rfc3339_concise")] + time_created: chrono::DateTime, + #[tabled(display_with = "datetime_opt_rfc3339_concise")] + time_deleted: Option>, + } let vmms = vmm_dsl::vmm .filter(vmm_dsl::instance_id.eq(id.into_untyped_uuid())) .limit(i64::from(u32::from(fetch_opts.fetch_limit))) @@ -3312,10 +3385,36 @@ async fn cmd_db_instance_info( check_limit(&vmms, fetch_opts.fetch_limit, ctx); - let table = tabled::Table::new(vmms.iter().map(VmmStateRow::from)) - .with(tabled::settings::Style::empty()) - .with(tabled::settings::Padding::new(0, 1, 0, 0)) - .to_string(); + let table = tabled::Table::new(vmms.iter().map(|vmm| { + let &Vmm { + id, + sled_id, + propolis_ip: _, + propolis_port: _, + instance_id: _, + time_created, + time_deleted, + runtime: + db::model::VmmRuntimeState { + time_state_updated: _, + r#gen, + state, + }, + } = vmm; + VmmRow { + state: VmmStateRow { + id, + state, + generation: r#gen.0.into(), + }, + sled_id, + time_created, + time_deleted, + } + })) + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)) + .to_string(); println!("{table}"); } } @@ -3330,36 +3429,8 @@ struct VmmStateRow { state: db::model::VmmState, #[tabled(rename = "GEN")] generation: u64, - sled_id: Uuid, - #[tabled(display_with = "datetime_rfc3339_concise")] - time_created: chrono::DateTime, - #[tabled(display_with = "datetime_opt_rfc3339_concise")] - time_deleted: Option>, } -impl From<&'_ Vmm> for VmmStateRow { - fn from(vmm: &Vmm) -> Self { - let &Vmm { - id, - time_created, - time_deleted, - sled_id, - propolis_ip: _, - propolis_port: _, - instance_id: _, - runtime: - db::model::VmmRuntimeState { time_state_updated: _, r#gen, state }, - } = vmm; - Self { - id, - state, - time_created, - time_deleted, - generation: r#gen.0.into(), - sled_id, - } - } -} #[derive(Tabled)] #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] struct CustomerInstanceRow { @@ -5614,6 +5685,393 @@ impl From<&'_ Migration> for MigrationVmms { } } +impl From<&'_ Migration> for SingleInstanceMigrationRow { + fn from(migration: &Migration) -> Self { + Self { + created: migration.time_created, + vmms: MigrationVmms::from(migration), + } + } +} + +// VMMs + +async fn cmd_db_vmm_info( + opctx: &OpContext, + datastore: &DataStore, + fetch_opts: &DbFetchOptions, + &VmmInfoArgs { uuid }: &VmmInfoArgs, +) -> Result<(), anyhow::Error> { + use db::schema::migration::dsl as migration_dsl; + use db::schema::sled_resource::dsl as resource_dsl; + use db::schema::vmm::dsl as vmm_dsl; + + let vmm = vmm_dsl::vmm + .filter(vmm_dsl::id.eq(uuid)) + .select(Vmm::as_select()) + .limit(1) + .load_async(&*datastore.pool_connection_for_tests().await?) + .await + .with_context(|| format!("failed to fetch VMM record for {uuid}"))? + .into_iter() + .next() + .ok_or_else(|| anyhow::anyhow!("no VMM found with ID {uuid}"))?; + let sled_result = + LookupPath::new(opctx, datastore).sled_id(vmm.sled_id).fetch().await; + let sled = match sled_result { + Ok((_, sled)) => Some(sled), + Err(err) => { + eprintln!( + "WARN: failed to fetch sled with ID {}: {err}", + vmm.sled_id + ); + None + } + }; + + println!("\n{:=<80}", "== VMM "); + prettyprint_vmm( + " ", + &vmm, + None, + sled.as_ref().map(|sled| sled.serial_number()), + true, + ); + + fn prettyprint_reservation( + resource: db::model::SledResource, + include_sled_id: bool, + ) { + use db::model::ByteCount; + let db::model::SledResource { + id: _, + sled_id, + kind: _, + resources: + db::model::Resources { + hardware_threads, + rss_ram: ByteCount(rss), + reservoir_ram: ByteCount(reservoir), + }, + } = resource; + const SLED_ID: &'static str = "sled ID"; + const THREADS: &'static str = "hardware threads"; + const RSS: &'static str = "RSS RAM"; + const RESERVOIR: &'static str = "reservoir RAM"; + const WIDTH: usize = const_max_len(&[SLED_ID, THREADS, RSS, RESERVOIR]); + if include_sled_id { + println!(" {SLED_ID:>WIDTH$}: {sled_id}"); + } + println!(" {THREADS:>WIDTH$}: {hardware_threads}"); + println!(" {RSS:>WIDTH$}: {rss}"); + println!(" {RESERVOIR:>WIDTH$}: {reservoir}"); + } + + let reservations = resource_dsl::sled_resource + .filter(resource_dsl::id.eq(uuid)) + .select(db::model::SledResource::as_select()) + .load_async::( + &*datastore.pool_connection_for_tests().await?, + ) + .await + .with_context(|| { + format!("failed to fetch sled resource records for {uuid}") + })?; + + if !reservations.is_empty() { + println!("\n{:=<80}", "== SLED RESOURCE RESERVATIONS "); + + let multiple_reservations = reservations.len() > 1; + if multiple_reservations { + println!( + "/!\\ VMM has multiple sled resource reservation records! \ + This is a bug; please open an issue about it here:\n\ + https://github.com/oxidecomputer/omicron/issues/new?template=Blank+issue", + ); + } + for r in reservations { + prettyprint_reservation(r, multiple_reservations); + println!(); + } + } + + let ctx = || format!("listing migrations involving VMM {uuid}"); + let migrations = migration_dsl::migration + .filter( + migration_dsl::source_propolis_id + .eq(uuid) + .or(migration_dsl::target_propolis_id.eq(uuid)), + ) + // A single VMM will typically only have 0-1 migrations in, but it may + // have any number of migrations out, since attempts to migrate out of + // the VMM may have failed on the migration target's side. + .limit(i64::from(u32::from(fetch_opts.fetch_limit))) + .order_by(migration_dsl::time_created) + // This is just to prove to CRDB that it can use the + // migrations-by-time-created index, it doesn't actually do anything. + .filter(migration_dsl::time_created.gt(chrono::DateTime::UNIX_EPOCH)) + .select(db::model::Migration::as_select()) + .load_async(&*datastore.pool_connection_for_tests().await?) + .await + .with_context(ctx)?; + + check_limit(&migrations, fetch_opts.fetch_limit, ctx); + + if !migrations.is_empty() { + println!("\n{:=<80}", "== MIGRATIONS "); + // TODO: since this command is focused on the individual VMM, we could + // potentially be a bit fancier when displaying migrations, and print + // something like "IN"/"OUT" based on the VMM's role in that migration, + // rather than just sticking its UUID in the source/target column as + // appropriate. + let table = tabled::Table::new( + migrations.iter().map(SingleInstanceMigrationRow::from), + ) + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)) + .to_string(); + println!("{table}"); + } + + Ok(()) +} + +fn prettyprint_vmm( + indent: &str, + vmm: &Vmm, + width: Option, + sled_serial: Option<&str>, + inst_id: bool, +) { + const ID: &'static str = "ID"; + const CREATED: &'static str = "created at"; + const DELETED: &'static str = "deleted at"; + const UPDATED: &'static str = "updated at"; + const INSTANCE_ID: &'static str = "instance ID"; + const SLED_ID: &'static str = "sled ID"; + const SLED_SERIAL: &'static str = "sled serial"; + const ADDRESS: &'static str = "propolis address"; + const STATE: &'static str = "state"; + const WIDTH: usize = const_max_len(&[ + ID, + CREATED, + DELETED, + UPDATED, + INSTANCE_ID, + SLED_ID, + SLED_SERIAL, + STATE, + ADDRESS, + ]); + + let width = std::cmp::max(width, Some(WIDTH)).unwrap_or(WIDTH); + let Vmm { + id, + time_created, + time_deleted, + instance_id, + sled_id, + propolis_ip, + propolis_port, + runtime: db::model::VmmRuntimeState { state, r#gen, time_state_updated }, + } = vmm; + + println!("{indent}{ID:>width$}: {id}"); + if inst_id { + println!("{indent}{INSTANCE_ID:>width$}: {instance_id}"); + } + println!("{indent}{CREATED:>width$}: {time_created}"); + if let Some(deleted) = time_deleted { + println!("{indent}{DELETED:width$}: {deleted}"); + } + println!("{indent}{STATE:>width$}: {state}"); + let g = u64::from(r#gen.0); + println!( + "{indent}{UPDATED:>width$}: {time_state_updated:?} (generation {g})" + ); + + println!( + "{indent}{ADDRESS:>width$}: {}:{}", + propolis_ip.ip(), + propolis_port.0 + ); + println!("{indent}{SLED_ID:>width$}: {sled_id}"); + if let Some(serial) = sled_serial { + println!("{indent}{SLED_SERIAL:>width$}: {serial}"); + } +} + +async fn cmd_db_vmm_list( + datastore: &DataStore, + fetch_opts: &DbFetchOptions, + &VmmListArgs { ref states, verbose }: &VmmListArgs, +) -> Result<(), anyhow::Error> { + use db::model::VmmState; + use db::schema::{sled::dsl as sled_dsl, vmm::dsl}; + + let ctx = || "loading VMMs"; + let mut query = dsl::vmm.into_boxed(); + + if !fetch_opts.include_deleted { + query = query.filter(dsl::time_deleted.is_null()); + + // If the user wanted to see VMMs in states that the control plane may + // have soft-deleted, but didn't ask to include deleted records, let + // them know that some stuff may be missing. + let maybe_deleted_states = + states.iter().filter(|s| VmmState::DESTROYABLE_STATES.contains(s)); + for state in maybe_deleted_states { + eprintln!( + "WARN: VMMs in the `{state:?}` state may have been deleted, \ + but `--include-deleted` was not specified", + ); + } + } + + if !states.is_empty() { + query = query.filter(dsl::state.eq_any(states.clone())); + } + + let vmms = datastore + .pool_connection_for_tests() + .await? + .transaction_async(|conn| async move { + // If we are including deleted VMMs, we can no longer use indices on + // the VMM table, which do not index deleted VMMs. Thus, we must + // allow a full-table scan in that case. + if fetch_opts.include_deleted { + conn.batch_execute_async(ALLOW_FULL_TABLE_SCAN_SQL).await?; + } + + query + .left_join(sled_dsl::sled.on(sled_dsl::id.eq(dsl::sled_id))) + .limit(i64::from(u32::from(fetch_opts.fetch_limit))) + .select((Vmm::as_select(), Option::::as_select())) + .load_async::<(Vmm, Option)>(&conn) + .await + }) + .await + .with_context(ctx)?; + + check_limit(&vmms, fetch_opts.fetch_limit, ctx); + + #[derive(Tabled)] + #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] + struct VmmRow<'a> { + instance_id: Uuid, + #[tabled(inline)] + state: VmmStateRow, + sled: &'a str, + } + + impl<'a> From<&'a (Vmm, Option)> for VmmRow<'a> { + fn from((ref vmm, ref sled): &'a (Vmm, Option)) -> Self { + let &Vmm { + id, + time_created: _, + time_deleted: _, + instance_id, + sled_id, + propolis_ip: _, + propolis_port: _, + runtime: + db::model::VmmRuntimeState { + state, + r#gen, + time_state_updated: _, + }, + } = vmm; + let sled = match sled { + Some(sled) => sled.serial_number(), + None => { + eprintln!("WARN: no sled found with ID {sled_id}"); + "" + } + }; + VmmRow { + instance_id, + state: VmmStateRow { id, state, generation: r#gen.0.into() }, + sled, + } + } + } + + #[derive(Tabled)] + #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] + struct VerboseVmmRow<'a> { + #[tabled(inline)] + inner: VmmRow<'a>, + sled_id: Uuid, + address: std::net::SocketAddr, + #[tabled(display_with = "datetime_rfc3339_concise")] + time_created: DateTime, + #[tabled(display_with = "datetime_rfc3339_concise")] + time_updated: DateTime, + } + + impl<'a> From<&'a (Vmm, Option)> for VerboseVmmRow<'a> { + fn from(it: &'a (Vmm, Option)) -> Self { + let Vmm { + time_created, + time_deleted: _, + sled_id, + propolis_ip, + propolis_port, + ref runtime, + .. + } = it.0; + VerboseVmmRow { + sled_id, + inner: VmmRow::from(it), + address: std::net::SocketAddr::new( + propolis_ip.ip(), + propolis_port.into(), + ), + time_created, + time_updated: runtime.time_state_updated, + } + } + } + + #[derive(Tabled)] + #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] + struct WithDeleted { + #[tabled(inline)] + inner: T, + #[tabled(display_with = "datetime_opt_rfc3339_concise")] + time_deleted: Option>, + } + + impl<'a, T> From<&'a (Vmm, Option)> for WithDeleted + where + T: Tabled + From<&'a (Vmm, Option)>, + { + fn from(it: &'a (Vmm, Option)) -> Self { + Self { inner: T::from(it), time_deleted: it.0.time_deleted } + } + } + + let mut table = match (verbose, fetch_opts.include_deleted) { + (true, true) => tabled::Table::new( + vmms.iter().map(WithDeleted::::from), + ), + (true, false) => { + tabled::Table::new(vmms.iter().map(VerboseVmmRow::from)) + } + (false, true) => { + tabled::Table::new(vmms.iter().map(WithDeleted::::from)) + } + (false, false) => tabled::Table::new(vmms.iter().map(VmmRow::from)), + }; + table + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)); + + println!("{table}"); + + Ok(()) +} + // Display an empty cell for an Option if it's None. fn display_option_blank(opt: &Option) -> String { opt.as_ref().map(|x| x.to_string()).unwrap_or_else(|| "".to_string()) diff --git a/dev-tools/omdb/tests/usage_errors.out b/dev-tools/omdb/tests/usage_errors.out index 568151fa52..61241d9d26 100644 --- a/dev-tools/omdb/tests/usage_errors.out +++ b/dev-tools/omdb/tests/usage_errors.out @@ -130,6 +130,9 @@ Commands: snapshots Print information about snapshots validate Validate the contents of the database volumes Print information about volumes + vmm Print information about Propolis virtual machine manager (VMM) + processes + vmms Alias to `omdb db vmm list` help Print this message or the help of the given subcommand(s) Options: @@ -178,6 +181,9 @@ Commands: snapshots Print information about snapshots validate Validate the contents of the database volumes Print information about volumes + vmm Print information about Propolis virtual machine manager (VMM) + processes + vmms Alias to `omdb db vmm list` help Print this message or the help of the given subcommand(s) Options: diff --git a/nexus/db-model/src/instance_state.rs b/nexus/db-model/src/instance_state.rs index 79bbab24b8..5c8f496832 100644 --- a/nexus/db-model/src/instance_state.rs +++ b/nexus/db-model/src/instance_state.rs @@ -7,7 +7,6 @@ use omicron_common::api::external; use serde::Deserialize; use serde::Serialize; use std::fmt; -use strum::VariantArray; impl_enum_type!( #[derive(SqlType, Debug)] @@ -23,7 +22,7 @@ impl_enum_type!( FromSqlRow, Serialize, Deserialize, - VariantArray, + strum::VariantArray, )] #[diesel(sql_type = InstanceStateEnum)] pub enum InstanceState; @@ -50,6 +49,9 @@ impl InstanceState { InstanceState::Destroyed => "destroyed", } } + + pub const ALL_STATES: &'static [Self] = + ::VARIANTS; } impl fmt::Display for InstanceState { @@ -72,15 +74,15 @@ impl From for omicron_common::api::external::InstanceState { } impl std::str::FromStr for InstanceState { - type Err = FromStrError; + type Err = InstanceStateParseError; fn from_str(s: &str) -> Result { - for &v in Self::VARIANTS { + for &v in Self::ALL_STATES { if s.eq_ignore_ascii_case(v.label()) { return Ok(v); } } - Err(FromStrError(())) + Err(InstanceStateParseError(())) } } @@ -90,12 +92,12 @@ impl diesel::query_builder::QueryId for InstanceStateEnum { } #[derive(Debug, Eq, PartialEq)] -pub struct FromStrError(()); +pub struct InstanceStateParseError(()); -impl fmt::Display for FromStrError { +impl fmt::Display for InstanceStateParseError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "expected one of [")?; - let mut variants = InstanceState::VARIANTS.iter(); + let mut variants = InstanceState::ALL_STATES.iter(); if let Some(v) = variants.next() { write!(f, "{v}")?; for v in variants { @@ -106,7 +108,7 @@ impl fmt::Display for FromStrError { } } -impl std::error::Error for FromStrError {} +impl std::error::Error for InstanceStateParseError {} #[cfg(test)] mod tests { @@ -114,7 +116,7 @@ mod tests { #[test] fn test_from_str_roundtrips() { - for &variant in InstanceState::VARIANTS { + for &variant in InstanceState::ALL_STATES { assert_eq!(Ok(dbg!(variant)), dbg!(variant.to_string().parse())); } } diff --git a/nexus/db-model/src/vmm_state.rs b/nexus/db-model/src/vmm_state.rs index 24724c7988..76cb813d64 100644 --- a/nexus/db-model/src/vmm_state.rs +++ b/nexus/db-model/src/vmm_state.rs @@ -13,7 +13,17 @@ impl_enum_type!( #[diesel(postgres_type(name = "vmm_state", schema = "public"))] pub struct VmmStateEnum; - #[derive(Copy, Clone, Debug, PartialEq, AsExpression, FromSqlRow, Serialize, Deserialize)] + #[derive( + Copy, + Clone, + Debug, + PartialEq, + AsExpression, + FromSqlRow, + Serialize, + Deserialize, + strum::VariantArray, + )] #[diesel(sql_type = VmmStateEnum)] pub enum VmmState; @@ -45,6 +55,10 @@ impl VmmState { } } + /// All VMM states. + pub const ALL_STATES: &'static [Self] = + ::VARIANTS; + /// States in which it is safe to deallocate a VMM's sled resources and mark /// it as deleted. pub const DESTROYABLE_STATES: &'static [Self] = @@ -160,6 +174,38 @@ impl diesel::query_builder::QueryId for VmmStateEnum { const HAS_STATIC_QUERY_ID: bool = false; } +impl std::str::FromStr for VmmState { + type Err = VmmStateParseError; + fn from_str(s: &str) -> Result { + for &state in Self::ALL_STATES { + if s.eq_ignore_ascii_case(state.label()) { + return Ok(state); + } + } + + Err(VmmStateParseError(())) + } +} + +#[derive(Debug, Eq, PartialEq)] +pub struct VmmStateParseError(()); + +impl fmt::Display for VmmStateParseError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "expected one of [")?; + let mut variants = VmmState::ALL_STATES.iter(); + if let Some(v) = variants.next() { + write!(f, "{v}")?; + for v in variants { + write!(f, ", {v}")?; + } + } + f.write_str("]") + } +} + +impl std::error::Error for VmmStateParseError {} + #[cfg(test)] mod tests { use super::*; @@ -175,4 +221,11 @@ mod tests { ); } } + + #[test] + fn test_from_str_roundtrips() { + for &variant in VmmState::ALL_STATES { + assert_eq!(Ok(dbg!(variant)), dbg!(variant.to_string().parse())); + } + } }