From c9fb7a6534d47fa40939260c6e64194f6eb77ad0 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jan 2025 12:08:58 -0800 Subject: [PATCH 01/14] [nexus] Add Affinity/Anti-Affinity Groups to API (unimplemented) --- common/src/api/external/mod.rs | 54 + nexus/external-api/output/nexus_tags.txt | 21 + nexus/external-api/src/lib.rs | 225 +++ nexus/src/external_api/http_entrypoints.rs | 355 ++++ .../output/uncovered-authz-endpoints.txt | 18 + nexus/types/src/external_api/params.rs | 76 +- nexus/types/src/external_api/views.rs | 24 +- openapi/nexus.json | 1439 ++++++++++++++++- 8 files changed, 2134 insertions(+), 78 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index fd5cbe38042..98832487d5a 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -982,6 +982,10 @@ impl JsonSchema for Hostname { pub enum ResourceType { AddressLot, AddressLotBlock, + AffinityGroup, + AffinityGroupMember, + AntiAffinityGroup, + AntiAffinityGroupMember, AllowList, BackgroundTask, BgpConfig, @@ -1312,6 +1316,56 @@ pub enum InstanceAutoRestartPolicy { BestEffort, } +// AFFINITY GROUPS + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum AffinityPolicy { + /// If the affinity request cannot be satisfied, allow it anyway. + /// + /// This enables a "best-effort" attempt to satisfy the affinity policy. + Allow, + + /// If the affinity request cannot be satisfied, fail explicitly. + Fail, +} + +/// Describes the scope of affinity for the purposes of co-location. +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum FailureDomain { + /// Instances are considered co-located if they are on the same sled + Sled, +} + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +#[serde(tag = "type", content = "value", rename_all = "snake_case")] +pub enum AffinityGroupMember { + Instance(Uuid), +} + +impl SimpleIdentity for AffinityGroupMember { + fn id(&self) -> Uuid { + match self { + AffinityGroupMember::Instance(id) => *id, + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +#[serde(tag = "type", content = "value", rename_all = "snake_case")] +pub enum AntiAffinityGroupMember { + Instance(Uuid), +} + +impl SimpleIdentity for AntiAffinityGroupMember { + fn id(&self) -> Uuid { + match self { + AntiAffinityGroupMember::Instance(id) => *id, + } + } +} + // DISKS /// View of a Disk diff --git a/nexus/external-api/output/nexus_tags.txt b/nexus/external-api/output/nexus_tags.txt index 82767e399c0..59ba14e5fb8 100644 --- a/nexus/external-api/output/nexus_tags.txt +++ b/nexus/external-api/output/nexus_tags.txt @@ -1,3 +1,24 @@ +API operations found with tag "affinity" +OPERATION ID METHOD URL PATH +affinity_group_create POST /v1/affinity-groups +affinity_group_delete DELETE /v1/affinity-groups/{affinity_group} +affinity_group_list GET /v1/affinity-groups +affinity_group_member_instance_add POST /v1/affinity-groups/{affinity_group}/members/instance/{instance} +affinity_group_member_instance_delete DELETE /v1/affinity-groups/{affinity_group}/members/instance/{instance} +affinity_group_member_instance_view GET /v1/affinity-groups/{affinity_group}/members/instance/{instance} +affinity_group_member_list GET /v1/affinity-groups/{affinity_group}/members +affinity_group_update PUT /v1/affinity-groups/{affinity_group} +affinity_group_view GET /v1/affinity-groups/{affinity_group} +anti_affinity_group_create POST /v1/anti-affinity-groups +anti_affinity_group_delete DELETE /v1/anti-affinity-groups/{anti_affinity_group} +anti_affinity_group_list GET /v1/anti-affinity-groups +anti_affinity_group_member_instance_add POST /v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance} +anti_affinity_group_member_instance_delete DELETE /v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance} +anti_affinity_group_member_instance_view GET /v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance} +anti_affinity_group_member_list GET /v1/anti-affinity-groups/{anti_affinity_group}/members +anti_affinity_group_update PUT /v1/anti-affinity-groups/{anti_affinity_group} +anti_affinity_group_view GET /v1/anti-affinity-groups/{anti_affinity_group} + API operations found with tag "disks" OPERATION ID METHOD URL PATH disk_bulk_write_import POST /v1/disks/{disk}/bulk-write diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index a832bde3199..7daa5514619 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -74,6 +74,13 @@ const PUT_UPDATE_REPOSITORY_MAX_BYTES: usize = 4 * GIB; allow_other_tags = false, policy = EndpointTagPolicy::ExactlyOne, tags = { + "affinity" = { + description = "Affinity groups give control over instance placement.", + external_docs = { + url = "http://docs.oxide.computer/api/affinity" + } + + }, "disks" = { description = "Virtual disks are used to store instance-local data which includes the operating system.", external_docs = { @@ -1257,6 +1264,224 @@ pub trait NexusExternalApi { disk_to_detach: TypedBody, ) -> Result, HttpError>; + // Affinity Groups + + /// List affinity groups + #[endpoint { + method = GET, + path = "/v1/affinity-groups", + tags = ["affinity"], + }] + async fn affinity_group_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + /// Fetch an affinity group + #[endpoint { + method = GET, + path = "/v1/affinity-groups/{affinity_group}", + tags = ["affinity"], + }] + async fn affinity_group_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// List members of an affinity group + #[endpoint { + method = GET, + path = "/v1/affinity-groups/{affinity_group}/members", + tags = ["affinity"], + }] + async fn affinity_group_member_list( + rqctx: RequestContext, + query_params: Query>, + path_params: Path, + ) -> Result>, HttpError>; + + /// Fetch an affinity group member + #[endpoint { + method = GET, + path = "/v1/affinity-groups/{affinity_group}/members/instance/{instance}", + tags = ["affinity"], + }] + async fn affinity_group_member_instance_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// Add a member to an affinity group + #[endpoint { + method = POST, + path = "/v1/affinity-groups/{affinity_group}/members/instance/{instance}", + tags = ["affinity"], + }] + async fn affinity_group_member_instance_add( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// Remove a member from an affinity group + #[endpoint { + method = DELETE, + path = "/v1/affinity-groups/{affinity_group}/members/instance/{instance}", + tags = ["affinity"], + }] + async fn affinity_group_member_instance_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result; + + /// Create an affinity group + #[endpoint { + method = POST, + path = "/v1/affinity-groups", + tags = ["affinity"], + }] + async fn affinity_group_create( + rqctx: RequestContext, + query_params: Query, + new_affinity_group_params: TypedBody, + ) -> Result, HttpError>; + + /// Update an affinity group + #[endpoint { + method = PUT, + path = "/v1/affinity-groups/{affinity_group}", + tags = ["affinity"], + }] + async fn affinity_group_update( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + updated_group: TypedBody, + ) -> Result, HttpError>; + + /// Delete an affinity group + #[endpoint { + method = DELETE, + path = "/v1/affinity-groups/{affinity_group}", + tags = ["affinity"], + }] + async fn affinity_group_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result; + + /// List anti-affinity groups + #[endpoint { + method = GET, + path = "/v1/anti-affinity-groups", + tags = ["affinity"], + }] + async fn anti_affinity_group_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + /// Fetch an anti-affinity group + #[endpoint { + method = GET, + path = "/v1/anti-affinity-groups/{anti_affinity_group}", + tags = ["affinity"], + }] + async fn anti_affinity_group_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// List members of an anti-affinity group + #[endpoint { + method = GET, + path = "/v1/anti-affinity-groups/{anti_affinity_group}/members", + tags = ["affinity"], + }] + async fn anti_affinity_group_member_list( + rqctx: RequestContext, + query_params: Query>, + path_params: Path, + ) -> Result>, HttpError>; + + /// Fetch an anti-affinity group member + #[endpoint { + method = GET, + path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}", + tags = ["affinity"], + }] + async fn anti_affinity_group_member_instance_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// Add a member to an anti-affinity group + #[endpoint { + method = POST, + path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}", + tags = ["affinity"], + }] + async fn anti_affinity_group_member_instance_add( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// Remove a member from an anti-affinity group + #[endpoint { + method = DELETE, + path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}", + tags = ["affinity"], + }] + async fn anti_affinity_group_member_instance_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result; + + /// Create an anti-affinity group + #[endpoint { + method = POST, + path = "/v1/anti-affinity-groups", + tags = ["affinity"], + }] + async fn anti_affinity_group_create( + rqctx: RequestContext, + query_params: Query, + new_affinity_group_params: TypedBody, + ) -> Result, HttpError>; + + /// Update an anti-affinity group + #[endpoint { + method = PUT, + path = "/v1/anti-affinity-groups/{anti_affinity_group}", + tags = ["affinity"], + }] + async fn anti_affinity_group_update( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + updated_group: TypedBody, + ) -> Result, HttpError>; + + /// Delete an anti-affinity group + #[endpoint { + method = DELETE, + path = "/v1/anti-affinity-groups/{anti_affinity_group}", + tags = ["affinity"], + }] + async fn anti_affinity_group_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result; + // Certificates /// List certificates for external endpoints diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 191c304216d..906684e7825 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -13,6 +13,7 @@ use super::{ Utilization, Vpc, VpcRouter, VpcSubnet, }, }; +use crate::app::Unimpl; use crate::app::external_endpoints::authority_for_request; use crate::app::support_bundles::SupportBundleQueryType; use crate::context::ApiContext; @@ -69,7 +70,9 @@ use omicron_common::api::external::http_pagination::ScanParams; use omicron_common::api::external::AddressLot; use omicron_common::api::external::AddressLotBlock; use omicron_common::api::external::AddressLotCreateResponse; +use omicron_common::api::external::AffinityGroupMember; use omicron_common::api::external::AggregateBgpMessageHistory; +use omicron_common::api::external::AntiAffinityGroupMember; use omicron_common::api::external::BgpAnnounceSet; use omicron_common::api::external::BgpAnnouncement; use omicron_common::api::external::BgpConfig; @@ -2506,6 +2509,358 @@ impl NexusExternalApi for NexusExternalApiImpl { .await } + // Affinity Groups + + async fn affinity_group_list( + rqctx: RequestContext, + _query_params: Query>, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_view( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_member_list( + rqctx: RequestContext, + _query_params: Query>, + _path_params: Path, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_member_instance_view( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_member_instance_add( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_member_instance_delete( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_create( + rqctx: RequestContext, + _query_params: Query, + _new_affinity_group_params: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_update( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + _updated_group: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn affinity_group_delete( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_list( + rqctx: RequestContext, + _query_params: Query>, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_view( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_member_list( + rqctx: RequestContext, + _query_params: Query>, + _path_params: Path, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_member_instance_view( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_member_instance_add( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_member_instance_delete( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_create( + rqctx: RequestContext, + _query_params: Query, + _new_anti_affinity_group_params: TypedBody< + params::AntiAffinityGroupCreate, + >, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_update( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + _updated_group: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_delete( + rqctx: RequestContext, + _query_params: Query, + _path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + Err(nexus.unimplemented_todo(&opctx, Unimpl::Public).await.into()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + // Certificates async fn certificate_list( diff --git a/nexus/tests/output/uncovered-authz-endpoints.txt b/nexus/tests/output/uncovered-authz-endpoints.txt index 8a639f1224c..c943728beb4 100644 --- a/nexus/tests/output/uncovered-authz-endpoints.txt +++ b/nexus/tests/output/uncovered-authz-endpoints.txt @@ -1,10 +1,22 @@ API endpoints with no coverage in authz tests: probe_delete (delete "/experimental/v1/probes/{probe}") +affinity_group_delete (delete "/v1/affinity-groups/{affinity_group}") +affinity_group_member_instance_delete (delete "/v1/affinity-groups/{affinity_group}/members/instance/{instance}") +anti_affinity_group_delete (delete "/v1/anti-affinity-groups/{anti_affinity_group}") +anti_affinity_group_member_instance_delete (delete "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}") probe_list (get "/experimental/v1/probes") probe_view (get "/experimental/v1/probes/{probe}") support_bundle_download (get "/experimental/v1/system/support-bundles/{support_bundle}/download") support_bundle_download_file (get "/experimental/v1/system/support-bundles/{support_bundle}/download/{file}") support_bundle_index (get "/experimental/v1/system/support-bundles/{support_bundle}/index") +affinity_group_list (get "/v1/affinity-groups") +affinity_group_view (get "/v1/affinity-groups/{affinity_group}") +affinity_group_member_list (get "/v1/affinity-groups/{affinity_group}/members") +affinity_group_member_instance_view (get "/v1/affinity-groups/{affinity_group}/members/instance/{instance}") +anti_affinity_group_list (get "/v1/anti-affinity-groups") +anti_affinity_group_view (get "/v1/anti-affinity-groups/{anti_affinity_group}") +anti_affinity_group_member_list (get "/v1/anti-affinity-groups/{anti_affinity_group}/members") +anti_affinity_group_member_instance_view (get "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}") ping (get "/v1/ping") networking_switch_port_lldp_neighbors (get "/v1/system/hardware/rack-switch-port/{rack_id}/{switch_location}/{port}/lldp/neighbors") networking_switch_port_lldp_config_view (get "/v1/system/hardware/switch-port/{port}/lldp/config") @@ -16,6 +28,12 @@ device_auth_confirm (post "/device/confirm") device_access_token (post "/device/token") probe_create (post "/experimental/v1/probes") login_saml (post "/login/{silo_name}/saml/{provider_name}") +affinity_group_create (post "/v1/affinity-groups") +affinity_group_member_instance_add (post "/v1/affinity-groups/{affinity_group}/members/instance/{instance}") +anti_affinity_group_create (post "/v1/anti-affinity-groups") +anti_affinity_group_member_instance_add (post "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}") login_local (post "/v1/login/{silo_name}/local") logout (post "/v1/logout") networking_switch_port_lldp_config_update (post "/v1/system/hardware/switch-port/{port}/lldp/config") +affinity_group_update (put "/v1/affinity-groups/{affinity_group}") +anti_affinity_group_update (put "/v1/anti-affinity-groups/{anti_affinity_group}") diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 8294e23e2f2..e87acd4e590 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -10,11 +10,11 @@ use base64::Engine; use chrono::{DateTime, Utc}; use http::Uri; use omicron_common::api::external::{ - AddressLotKind, AllowedSourceIps, BfdMode, BgpPeer, ByteCount, Hostname, - IdentityMetadataCreateParams, IdentityMetadataUpdateParams, - InstanceAutoRestartPolicy, InstanceCpuCount, LinkFec, LinkSpeed, Name, - NameOrId, PaginationOrder, RouteDestination, RouteTarget, SemverVersion, - TxEqConfig, UserId, + AddressLotKind, AffinityPolicy, AllowedSourceIps, BfdMode, BgpPeer, + ByteCount, FailureDomain, Hostname, IdentityMetadataCreateParams, + IdentityMetadataUpdateParams, InstanceAutoRestartPolicy, InstanceCpuCount, + LinkFec, LinkSpeed, Name, NameOrId, PaginationOrder, RouteDestination, + RouteTarget, SemverVersion, TxEqConfig, UserId, }; use omicron_common::disk::DiskVariant; use oxnet::{IpNet, Ipv4Net, Ipv6Net}; @@ -69,6 +69,8 @@ pub struct UninitializedSledId { pub part: String, } +path_param!(AffinityGroupPath, affinity_group, "affinity group"); +path_param!(AntiAffinityGroupPath, anti_affinity_group, "anti affinity group"); path_param!(ProjectPath, project, "project"); path_param!(InstancePath, instance, "instance"); path_param!(NetworkInterfacePath, interface, "network interface"); @@ -806,6 +808,70 @@ where Ok(v) } +// AFFINITY GROUPS + +/// Create-time parameters for an `AffinityGroup` +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct AffinityGroupCreate { + #[serde(flatten)] + pub identity: IdentityMetadataCreateParams, + + pub policy: AffinityPolicy, + pub failure_domain: FailureDomain, +} + +/// Updateable properties of an `AffinityGroup` +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct AffinityGroupUpdate { + #[serde(flatten)] + pub identity: IdentityMetadataUpdateParams, +} + +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct AffinityInstanceGroupMemberPath { + pub affinity_group: NameOrId, + pub instance: NameOrId, +} + +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct AntiAffinityInstanceGroupMemberPath { + pub anti_affinity_group: NameOrId, + pub instance: NameOrId, +} + +/// Create-time parameters for an `AntiAffinityGroup` +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct AntiAffinityGroupCreate { + #[serde(flatten)] + pub identity: IdentityMetadataCreateParams, + + pub policy: AffinityPolicy, + pub failure_domain: FailureDomain, +} + +/// Updateable properties of an `AntiAffinityGroup` +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct AntiAffinityGroupUpdate { + #[serde(flatten)] + pub identity: IdentityMetadataUpdateParams, +} + +#[derive(Deserialize, JsonSchema, Clone)] +pub struct AffinityGroupSelector { + /// Name or ID of the project, only required if `affinity_group` is provided as a `Name` + pub project: Option, + /// Name or ID of the Affinity Group + pub affinity_group: NameOrId, +} + +#[derive(Deserialize, JsonSchema, Clone)] +pub struct AntiAffinityGroupSelector { + /// Name or ID of the project, only required if `anti_affinity_group` is provided as a `Name` + pub project: Option, + /// Name or ID of the Anti Affinity Group + pub anti_affinity_group: NameOrId, +} + // PROJECTS /// Create-time parameters for a `Project` diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index 3430d06f724..fe93e4cae60 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -13,9 +13,9 @@ use chrono::DateTime; use chrono::Utc; use diffus::Diffus; use omicron_common::api::external::{ - AllowedSourceIps as ExternalAllowedSourceIps, ByteCount, Digest, Error, - IdentityMetadata, InstanceState, Name, ObjectIdentity, RoleName, - SimpleIdentityOrName, + AffinityPolicy, AllowedSourceIps as ExternalAllowedSourceIps, ByteCount, + Digest, Error, FailureDomain, IdentityMetadata, InstanceState, Name, + ObjectIdentity, RoleName, SimpleIdentityOrName, }; use oxnet::{Ipv4Net, Ipv6Net}; use schemars::JsonSchema; @@ -112,6 +112,24 @@ impl SimpleIdentityOrName for SiloUtilization { } } +// AFFINITY GROUPS + +#[derive(ObjectIdentity, Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct AffinityGroup { + #[serde(flatten)] + pub identity: IdentityMetadata, + pub policy: AffinityPolicy, + pub failure_domain: FailureDomain, +} + +#[derive(ObjectIdentity, Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct AntiAffinityGroup { + #[serde(flatten)] + pub identity: IdentityMetadata, + pub policy: AffinityPolicy, + pub failure_domain: FailureDomain, +} + // IDENTITY PROVIDER #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] diff --git a/openapi/nexus.json b/openapi/nexus.json index 45f2d4787cf..2e4ef3ac597 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -683,6 +683,958 @@ } } }, + "/v1/affinity-groups": { + "get": { + "tags": [ + "affinity" + ], + "summary": "List affinity groups", + "operationId": "affinity_group_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroupResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [ + "project" + ] + } + }, + "post": { + "tags": [ + "affinity" + ], + "summary": "Create an affinity group", + "operationId": "affinity_group_create", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroupCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroup" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/affinity-groups/{affinity_group}": { + "get": { + "tags": [ + "affinity" + ], + "summary": "Fetch an affinity group", + "operationId": "affinity_group_view", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "description": "Name or ID of the affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroup" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "tags": [ + "affinity" + ], + "summary": "Update an affinity group", + "operationId": "affinity_group_update", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "description": "Name or ID of the affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroupUpdate" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroup" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "affinity" + ], + "summary": "Delete an affinity group", + "operationId": "affinity_group_delete", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "description": "Name or ID of the affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/affinity-groups/{affinity_group}/members": { + "get": { + "tags": [ + "affinity" + ], + "summary": "List members of an affinity group", + "operationId": "affinity_group_member_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/IdSortMode" + } + }, + { + "in": "path", + "name": "affinity_group", + "description": "Name or ID of the affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroupMemberResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + } + }, + "/v1/affinity-groups/{affinity_group}/members/instance/{instance}": { + "get": { + "tags": [ + "affinity" + ], + "summary": "Fetch an affinity group member", + "operationId": "affinity_group_member_instance_view", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "instance", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroupMember" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "post": { + "tags": [ + "affinity" + ], + "summary": "Add a member to an affinity group", + "operationId": "affinity_group_member_instance_add", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "instance", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AffinityGroupMember" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "affinity" + ], + "summary": "Remove a member from an affinity group", + "operationId": "affinity_group_member_instance_delete", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "instance", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/anti-affinity-groups": { + "get": { + "tags": [ + "affinity" + ], + "summary": "List anti-affinity groups", + "operationId": "anti_affinity_group_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [ + "project" + ] + } + }, + "post": { + "tags": [ + "affinity" + ], + "summary": "Create an anti-affinity group", + "operationId": "anti_affinity_group_create", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroup" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/anti-affinity-groups/{anti_affinity_group}": { + "get": { + "tags": [ + "affinity" + ], + "summary": "Fetch an anti-affinity group", + "operationId": "anti_affinity_group_view", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "description": "Name or ID of the anti affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroup" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "tags": [ + "affinity" + ], + "summary": "Update an anti-affinity group", + "operationId": "anti_affinity_group_update", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "description": "Name or ID of the anti affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupUpdate" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroup" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "affinity" + ], + "summary": "Delete an anti-affinity group", + "operationId": "anti_affinity_group_delete", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "description": "Name or ID of the anti affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/anti-affinity-groups/{anti_affinity_group}/members": { + "get": { + "tags": [ + "affinity" + ], + "summary": "List members of an anti-affinity group", + "operationId": "anti_affinity_group_member_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/IdSortMode" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "description": "Name or ID of the anti affinity group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupMemberResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + } + }, + "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}": { + "get": { + "tags": [ + "affinity" + ], + "summary": "Fetch an anti-affinity group member", + "operationId": "anti_affinity_group_member_instance_view", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "instance", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupMember" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "post": { + "tags": [ + "affinity" + ], + "summary": "Add a member to an anti-affinity group", + "operationId": "anti_affinity_group_member_instance_add", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "instance", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AntiAffinityGroupMember" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "affinity" + ], + "summary": "Remove a member from an anti-affinity group", + "operationId": "anti_affinity_group_member_instance_delete", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "anti_affinity_group", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "instance", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/v1/certificates": { "get": { "tags": [ @@ -11055,13 +12007,227 @@ "type": "string", "format": "ip" } - }, - "required": [ - "first_address", - "last_address" + }, + "required": [ + "first_address", + "last_address" + ] + }, + "AddressLotBlockResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/AddressLotBlock" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "AddressLotCreate": { + "description": "Parameters for creating an address lot.", + "type": "object", + "properties": { + "blocks": { + "description": "The blocks to add along with the new address lot.", + "type": "array", + "items": { + "$ref": "#/components/schemas/AddressLotBlockCreate" + } + }, + "description": { + "type": "string" + }, + "kind": { + "description": "The kind of address lot to create.", + "allOf": [ + { + "$ref": "#/components/schemas/AddressLotKind" + } + ] + }, + "name": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "blocks", + "description", + "kind", + "name" + ] + }, + "AddressLotCreateResponse": { + "description": "An address lot and associated blocks resulting from creating an address lot.", + "type": "object", + "properties": { + "blocks": { + "description": "The address lot blocks that were created.", + "type": "array", + "items": { + "$ref": "#/components/schemas/AddressLotBlock" + } + }, + "lot": { + "description": "The address lot that was created.", + "allOf": [ + { + "$ref": "#/components/schemas/AddressLot" + } + ] + } + }, + "required": [ + "blocks", + "lot" + ] + }, + "AddressLotKind": { + "description": "The kind associated with an address lot.", + "oneOf": [ + { + "description": "Infrastructure address lots are used for network infrastructure like addresses assigned to rack switches.", + "type": "string", + "enum": [ + "infra" + ] + }, + { + "description": "Pool address lots are used by IP pools.", + "type": "string", + "enum": [ + "pool" + ] + } + ] + }, + "AddressLotResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/AddressLot" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "AffinityGroup": { + "description": "Identity-related metadata that's included in nearly all public API objects", + "type": "object", + "properties": { + "description": { + "description": "human-readable free-form text about a resource", + "type": "string" + }, + "failure_domain": { + "$ref": "#/components/schemas/FailureDomain" + }, + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "name": { + "description": "unique, mutable, user-controlled identifier for each resource", + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + }, + "policy": { + "$ref": "#/components/schemas/AffinityPolicy" + }, + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "description", + "failure_domain", + "id", + "name", + "policy", + "time_created", + "time_modified" + ] + }, + "AffinityGroupCreate": { + "description": "Create-time parameters for an `AffinityGroup`", + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "failure_domain": { + "$ref": "#/components/schemas/FailureDomain" + }, + "name": { + "$ref": "#/components/schemas/Name" + }, + "policy": { + "$ref": "#/components/schemas/AffinityPolicy" + } + }, + "required": [ + "description", + "failure_domain", + "name", + "policy" + ] + }, + "AffinityGroupMember": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "instance" + ] + }, + "value": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "type", + "value" + ] + } ] }, - "AddressLotBlockResultsPage": { + "AffinityGroupMemberResultsPage": { "description": "A single page of results", "type": "object", "properties": { @@ -11069,7 +12235,7 @@ "description": "list of items on this page of results", "type": "array", "items": { - "$ref": "#/components/schemas/AddressLotBlock" + "$ref": "#/components/schemas/AffinityGroupMember" } }, "next_page": { @@ -11082,104 +12248,63 @@ "items" ] }, - "AddressLotCreate": { - "description": "Parameters for creating an address lot.", + "AffinityGroupResultsPage": { + "description": "A single page of results", "type": "object", "properties": { - "blocks": { - "description": "The blocks to add along with the new address lot.", + "items": { + "description": "list of items on this page of results", "type": "array", "items": { - "$ref": "#/components/schemas/AddressLotBlockCreate" + "$ref": "#/components/schemas/AffinityGroup" } }, - "description": { + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", "type": "string" - }, - "kind": { - "description": "The kind of address lot to create.", - "allOf": [ - { - "$ref": "#/components/schemas/AddressLotKind" - } - ] - }, - "name": { - "$ref": "#/components/schemas/Name" } }, "required": [ - "blocks", - "description", - "kind", - "name" + "items" ] }, - "AddressLotCreateResponse": { - "description": "An address lot and associated blocks resulting from creating an address lot.", + "AffinityGroupUpdate": { + "description": "Updateable properties of an `AffinityGroup`", "type": "object", "properties": { - "blocks": { - "description": "The address lot blocks that were created.", - "type": "array", - "items": { - "$ref": "#/components/schemas/AddressLotBlock" - } + "description": { + "nullable": true, + "type": "string" }, - "lot": { - "description": "The address lot that was created.", + "name": { + "nullable": true, "allOf": [ { - "$ref": "#/components/schemas/AddressLot" + "$ref": "#/components/schemas/Name" } ] } - }, - "required": [ - "blocks", - "lot" - ] + } }, - "AddressLotKind": { - "description": "The kind associated with an address lot.", + "AffinityPolicy": { "oneOf": [ { - "description": "Infrastructure address lots are used for network infrastructure like addresses assigned to rack switches.", + "description": "If the affinity request cannot be satisfied, allow it anyway.\n\nThis enables a \"best-effort\" attempt to satisfy the affinity policy.", "type": "string", "enum": [ - "infra" + "allow" ] }, { - "description": "Pool address lots are used by IP pools.", + "description": "If the affinity request cannot be satisfied, fail explicitly.", "type": "string", "enum": [ - "pool" + "fail" ] } ] }, - "AddressLotResultsPage": { - "description": "A single page of results", - "type": "object", - "properties": { - "items": { - "description": "list of items on this page of results", - "type": "array", - "items": { - "$ref": "#/components/schemas/AddressLot" - } - }, - "next_page": { - "nullable": true, - "description": "token used to fetch the next page of results (if any)", - "type": "string" - } - }, - "required": [ - "items" - ] - }, "AggregateBgpMessageHistory": { "description": "BGP message history for rack switches.", "type": "object", @@ -11284,6 +12409,161 @@ } ] }, + "AntiAffinityGroup": { + "description": "Identity-related metadata that's included in nearly all public API objects", + "type": "object", + "properties": { + "description": { + "description": "human-readable free-form text about a resource", + "type": "string" + }, + "failure_domain": { + "$ref": "#/components/schemas/FailureDomain" + }, + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "name": { + "description": "unique, mutable, user-controlled identifier for each resource", + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + }, + "policy": { + "$ref": "#/components/schemas/AffinityPolicy" + }, + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "description", + "failure_domain", + "id", + "name", + "policy", + "time_created", + "time_modified" + ] + }, + "AntiAffinityGroupCreate": { + "description": "Create-time parameters for an `AntiAffinityGroup`", + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "failure_domain": { + "$ref": "#/components/schemas/FailureDomain" + }, + "name": { + "$ref": "#/components/schemas/Name" + }, + "policy": { + "$ref": "#/components/schemas/AffinityPolicy" + } + }, + "required": [ + "description", + "failure_domain", + "name", + "policy" + ] + }, + "AntiAffinityGroupMember": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "instance" + ] + }, + "value": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, + "AntiAffinityGroupMemberResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/AntiAffinityGroupMember" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "AntiAffinityGroupResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/AntiAffinityGroup" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "AntiAffinityGroupUpdate": { + "description": "Updateable properties of an `AntiAffinityGroup`", + "type": "object", + "properties": { + "description": { + "nullable": true, + "type": "string" + }, + "name": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + } + } + }, "AuthzScope": { "description": "Authorization scope for a timeseries.\n\nThis describes the level at which a user must be authorized to read data from a timeseries. For example, fleet-scoping means the data is only visible to an operator or fleet reader. Project-scoped, on the other hand, indicates that a user will see data limited to the projects on which they have read permissions.", "oneOf": [ @@ -14635,6 +15915,18 @@ "items" ] }, + "FailureDomain": { + "description": "Describes the scope of affinity for the purposes of co-location.", + "oneOf": [ + { + "description": "Instances are considered co-located if they are on the same sled", + "type": "string", + "enum": [ + "sled" + ] + } + ] + }, "FieldSchema": { "description": "The name and type information for a field of a timeseries schema.", "type": "object", @@ -23264,6 +24556,13 @@ } }, "tags": [ + { + "name": "affinity", + "description": "Affinity groups give control over instance placement.", + "externalDocs": { + "url": "http://docs.oxide.computer/api/affinity" + } + }, { "name": "disks", "description": "Virtual disks are used to store instance-local data which includes the operating system.", From 4020517ef2707fd92978760a990f6fe20b1bdaa2 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jan 2025 12:19:13 -0800 Subject: [PATCH 02/14] [nexus] Add Affinity/Anti-Affinity groups to database --- dev-tools/omdb/src/bin/omdb/db.rs | 1 + nexus/db-model/src/affinity.rs | 260 ++++++++++++++++++ nexus/db-model/src/lib.rs | 2 + nexus/db-model/src/project.rs | 24 +- nexus/db-model/src/schema.rs | 49 +++- nexus/db-model/src/schema_versions.rs | 3 +- nexus/db-model/src/sled_resource.rs | 32 ++- nexus/db-queries/src/db/datastore/sled.rs | 26 +- .../background/tasks/abandoned_vmm_reaper.rs | 6 +- nexus/src/app/sagas/instance_common.rs | 13 +- nexus/src/app/sagas/instance_migrate.rs | 3 +- nexus/src/app/sagas/instance_start.rs | 3 +- nexus/src/app/sled.rs | 10 +- nexus/src/external_api/http_entrypoints.rs | 2 +- nexus/tests/integration_tests/schema.rs | 5 +- schema/crdb/affinity/up01.sql | 8 + schema/crdb/affinity/up02.sql | 5 + schema/crdb/affinity/up03.sql | 13 + schema/crdb/affinity/up04.sql | 7 + schema/crdb/affinity/up05.sql | 7 + schema/crdb/affinity/up06.sql | 4 + schema/crdb/affinity/up07.sql | 13 + schema/crdb/affinity/up08.sql | 6 + schema/crdb/affinity/up09.sql | 7 + schema/crdb/affinity/up10.sql | 4 + schema/crdb/affinity/up11.sql | 1 + schema/crdb/affinity/up12.sql | 4 + schema/crdb/dbinit.sql | 104 ++++++- uuid-kinds/src/lib.rs | 2 + 29 files changed, 579 insertions(+), 45 deletions(-) create mode 100644 nexus/db-model/src/affinity.rs create mode 100644 schema/crdb/affinity/up01.sql create mode 100644 schema/crdb/affinity/up02.sql create mode 100644 schema/crdb/affinity/up03.sql create mode 100644 schema/crdb/affinity/up04.sql create mode 100644 schema/crdb/affinity/up05.sql create mode 100644 schema/crdb/affinity/up06.sql create mode 100644 schema/crdb/affinity/up07.sql create mode 100644 schema/crdb/affinity/up08.sql create mode 100644 schema/crdb/affinity/up09.sql create mode 100644 schema/crdb/affinity/up10.sql create mode 100644 schema/crdb/affinity/up11.sql create mode 100644 schema/crdb/affinity/up12.sql diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs index bfd7c594a3b..c5f8a1f26c5 100644 --- a/dev-tools/omdb/src/bin/omdb/db.rs +++ b/dev-tools/omdb/src/bin/omdb/db.rs @@ -6191,6 +6191,7 @@ async fn cmd_db_vmm_info( rss_ram: ByteCount(rss), reservoir_ram: ByteCount(reservoir), }, + instance_id: _, } = resource; const SLED_ID: &'static str = "sled ID"; const THREADS: &'static str = "hardware threads"; diff --git a/nexus/db-model/src/affinity.rs b/nexus/db-model/src/affinity.rs new file mode 100644 index 00000000000..df97e7d1633 --- /dev/null +++ b/nexus/db-model/src/affinity.rs @@ -0,0 +1,260 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/5.0/. + +// Copyright 2024 Oxide Computer Company + +//! Database representation of affinity and anti-affinity groups + +use super::impl_enum_type; +use super::Name; +use crate::schema::affinity_group; +use crate::schema::affinity_group_instance_membership; +use crate::schema::anti_affinity_group; +use crate::schema::anti_affinity_group_instance_membership; +use crate::typed_uuid::DbTypedUuid; +use chrono::{DateTime, Utc}; +use db_macros::Resource; +use nexus_types::external_api::params; +use nexus_types::external_api::views; +use omicron_common::api::external; +use omicron_common::api::external::IdentityMetadata; +use omicron_uuid_kinds::AffinityGroupKind; +use omicron_uuid_kinds::AffinityGroupUuid; +use omicron_uuid_kinds::AntiAffinityGroupKind; +use omicron_uuid_kinds::AntiAffinityGroupUuid; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceKind; +use omicron_uuid_kinds::InstanceUuid; +use uuid::Uuid; + +impl_enum_type!( + #[derive(SqlType, Debug, QueryId)] + #[diesel(postgres_type(name = "affinity_policy", schema = "public"))] + pub struct AffinityPolicyEnum; + + #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq, Eq, Ord, PartialOrd)] + #[diesel(sql_type = AffinityPolicyEnum)] + pub enum AffinityPolicy; + + // Enum values + Fail => b"fail" + Allow => b"allow" +); + +impl From for external::AffinityPolicy { + fn from(policy: AffinityPolicy) -> Self { + match policy { + AffinityPolicy::Fail => Self::Fail, + AffinityPolicy::Allow => Self::Allow, + } + } +} + +impl From for AffinityPolicy { + fn from(policy: external::AffinityPolicy) -> Self { + match policy { + external::AffinityPolicy::Fail => Self::Fail, + external::AffinityPolicy::Allow => Self::Allow, + } + } +} + +impl_enum_type!( + #[derive(SqlType, Debug)] + #[diesel(postgres_type(name = "failure_domain", schema = "public"))] + pub struct FailureDomainEnum; + + #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq)] + #[diesel(sql_type = FailureDomainEnum)] + pub enum FailureDomain; + + // Enum values + Sled => b"sled" +); + +impl From for external::FailureDomain { + fn from(domain: FailureDomain) -> Self { + match domain { + FailureDomain::Sled => Self::Sled, + } + } +} + +impl From for FailureDomain { + fn from(domain: external::FailureDomain) -> Self { + match domain { + external::FailureDomain::Sled => Self::Sled, + } + } +} + +#[derive( + Queryable, Insertable, Clone, Debug, Resource, Selectable, PartialEq, +)] +#[diesel(table_name = affinity_group)] +pub struct AffinityGroup { + #[diesel(embed)] + pub identity: AffinityGroupIdentity, + pub project_id: Uuid, + pub policy: AffinityPolicy, + pub failure_domain: FailureDomain, +} + +impl AffinityGroup { + pub fn new(project_id: Uuid, params: params::AffinityGroupCreate) -> Self { + Self { + identity: AffinityGroupIdentity::new( + Uuid::new_v4(), + params.identity, + ), + project_id, + policy: params.policy.into(), + failure_domain: params.failure_domain.into(), + } + } +} + +impl From for views::AffinityGroup { + fn from(group: AffinityGroup) -> Self { + let identity = IdentityMetadata { + id: group.identity.id, + name: group.identity.name.into(), + description: group.identity.description, + time_created: group.identity.time_created, + time_modified: group.identity.time_modified, + }; + Self { + identity, + policy: group.policy.into(), + failure_domain: group.failure_domain.into(), + } + } +} + +/// Describes a set of updates for the [`AffinityGroup`] model. +#[derive(AsChangeset)] +#[diesel(table_name = affinity_group)] +pub struct AffinityGroupUpdate { + pub name: Option, + pub description: Option, + pub time_modified: DateTime, +} + +impl From for AffinityGroupUpdate { + fn from(params: params::AffinityGroupUpdate) -> Self { + Self { + name: params.identity.name.map(Name), + description: params.identity.description, + time_modified: Utc::now(), + } + } +} + +#[derive( + Queryable, Insertable, Clone, Debug, Resource, Selectable, PartialEq, +)] +#[diesel(table_name = anti_affinity_group)] +pub struct AntiAffinityGroup { + #[diesel(embed)] + identity: AntiAffinityGroupIdentity, + pub project_id: Uuid, + pub policy: AffinityPolicy, + pub failure_domain: FailureDomain, +} + +impl AntiAffinityGroup { + pub fn new( + project_id: Uuid, + params: params::AntiAffinityGroupCreate, + ) -> Self { + Self { + identity: AntiAffinityGroupIdentity::new( + Uuid::new_v4(), + params.identity, + ), + project_id, + policy: params.policy.into(), + failure_domain: params.failure_domain.into(), + } + } +} + +impl From for views::AntiAffinityGroup { + fn from(group: AntiAffinityGroup) -> Self { + let identity = IdentityMetadata { + id: group.identity.id, + name: group.identity.name.into(), + description: group.identity.description, + time_created: group.identity.time_created, + time_modified: group.identity.time_modified, + }; + Self { + identity, + policy: group.policy.into(), + failure_domain: group.failure_domain.into(), + } + } +} + +/// Describes a set of updates for the [`AntiAffinityGroup`] model. +#[derive(AsChangeset)] +#[diesel(table_name = anti_affinity_group)] +pub struct AntiAffinityGroupUpdate { + pub name: Option, + pub description: Option, + pub time_modified: DateTime, +} + +impl From for AntiAffinityGroupUpdate { + fn from(params: params::AntiAffinityGroupUpdate) -> Self { + Self { + name: params.identity.name.map(Name), + description: params.identity.description, + time_modified: Utc::now(), + } + } +} + +#[derive(Queryable, Insertable, Clone, Debug, Selectable)] +#[diesel(table_name = affinity_group_instance_membership)] +pub struct AffinityGroupInstanceMembership { + pub group_id: DbTypedUuid, + pub instance_id: DbTypedUuid, +} + +impl AffinityGroupInstanceMembership { + pub fn new(group_id: AffinityGroupUuid, instance_id: InstanceUuid) -> Self { + Self { group_id: group_id.into(), instance_id: instance_id.into() } + } +} + +impl From for external::AffinityGroupMember { + fn from(member: AffinityGroupInstanceMembership) -> Self { + Self::Instance(member.instance_id.into_untyped_uuid()) + } +} + +#[derive(Queryable, Insertable, Clone, Debug, Selectable)] +#[diesel(table_name = anti_affinity_group_instance_membership)] +pub struct AntiAffinityGroupInstanceMembership { + pub group_id: DbTypedUuid, + pub instance_id: DbTypedUuid, +} + +impl AntiAffinityGroupInstanceMembership { + pub fn new( + group_id: AntiAffinityGroupUuid, + instance_id: InstanceUuid, + ) -> Self { + Self { group_id: group_id.into(), instance_id: instance_id.into() } + } +} + +impl From + for external::AntiAffinityGroupMember +{ + fn from(member: AntiAffinityGroupInstanceMembership) -> Self { + Self::Instance(member.instance_id.into_untyped_uuid()) + } +} diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index bec35c233c6..939388174e8 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -10,6 +10,7 @@ extern crate diesel; extern crate newtype_derive; mod address_lot; +mod affinity; mod allow_list; mod bfd; mod bgp; @@ -130,6 +131,7 @@ mod db { pub use self::macaddr::*; pub use self::unsigned::*; pub use address_lot::*; +pub use affinity::*; pub use allow_list::*; pub use bfd::*; pub use bgp::*; diff --git a/nexus/db-model/src/project.rs b/nexus/db-model/src/project.rs index 900a13e9c54..2079751a1c4 100644 --- a/nexus/db-model/src/project.rs +++ b/nexus/db-model/src/project.rs @@ -2,9 +2,15 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use super::{Disk, Generation, Instance, Name, Snapshot, Vpc}; +use super::{ + AffinityGroup, AntiAffinityGroup, Disk, Generation, Instance, Name, + Snapshot, Vpc, +}; use crate::collection::DatastoreCollectionConfig; -use crate::schema::{disk, image, instance, project, snapshot, vpc}; +use crate::schema::{ + affinity_group, anti_affinity_group, disk, image, instance, project, + snapshot, vpc, +}; use crate::Image; use chrono::{DateTime, Utc}; use db_macros::Resource; @@ -69,6 +75,20 @@ impl DatastoreCollectionConfig for Project { type CollectionIdColumn = instance::dsl::project_id; } +impl DatastoreCollectionConfig for Project { + type CollectionId = Uuid; + type GenerationNumberColumn = project::dsl::rcgen; + type CollectionTimeDeletedColumn = project::dsl::time_deleted; + type CollectionIdColumn = affinity_group::dsl::project_id; +} + +impl DatastoreCollectionConfig for Project { + type CollectionId = Uuid; + type GenerationNumberColumn = project::dsl::rcgen; + type CollectionTimeDeletedColumn = project::dsl::time_deleted; + type CollectionIdColumn = anti_affinity_group::dsl::project_id; +} + impl DatastoreCollectionConfig for Project { type CollectionId = Uuid; type GenerationNumberColumn = project::dsl::rcgen; diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index f734d1e88f3..8881d7dd0ed 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -468,6 +468,48 @@ table! { } } +table! { + affinity_group (id) { + id -> Uuid, + name -> Text, + description -> Text, + time_created -> Timestamptz, + time_modified -> Timestamptz, + time_deleted -> Nullable, + project_id -> Uuid, + policy -> crate::AffinityPolicyEnum, + failure_domain -> crate::FailureDomainEnum, + } +} + +table! { + anti_affinity_group (id) { + id -> Uuid, + name -> Text, + description -> Text, + time_created -> Timestamptz, + time_modified -> Timestamptz, + time_deleted -> Nullable, + project_id -> Uuid, + policy -> crate::AffinityPolicyEnum, + failure_domain -> crate::FailureDomainEnum, + } +} + +table! { + affinity_group_instance_membership (group_id, instance_id) { + group_id -> Uuid, + instance_id -> Uuid, + } +} + +table! { + anti_affinity_group_instance_membership (group_id, instance_id) { + group_id -> Uuid, + instance_id -> Uuid, + } +} + table! { metric_producer (id) { id -> Uuid, @@ -915,10 +957,11 @@ table! { sled_resource (id) { id -> Uuid, sled_id -> Uuid, - kind -> crate::SledResourceKindEnum, hardware_threads -> Int8, rss_ram -> Int8, reservoir_ram -> Int8, + kind -> crate::SledResourceKindEnum, + instance_id -> Nullable, } } @@ -2025,6 +2068,10 @@ allow_tables_to_appear_in_same_query!( allow_tables_to_appear_in_same_query!(hw_baseboard_id, inv_sled_agent,); allow_tables_to_appear_in_same_query!( + anti_affinity_group, + anti_affinity_group_instance_membership, + affinity_group, + affinity_group_instance_membership, bp_omicron_zone, bp_target, rendezvous_debug_dataset, diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index 150f4d092df..852de12f4c5 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -17,7 +17,7 @@ use std::collections::BTreeMap; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(122, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(123, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -29,6 +29,7 @@ static KNOWN_VERSIONS: Lazy> = Lazy::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(123, "affinity"), KnownVersion::new(122, "tuf-artifact-replication"), KnownVersion::new(121, "dataset-to-crucible-dataset"), KnownVersion::new(120, "rendezvous-debug-dataset"), diff --git a/nexus/db-model/src/sled_resource.rs b/nexus/db-model/src/sled_resource.rs index 27471f4d7a9..de5833aa41f 100644 --- a/nexus/db-model/src/sled_resource.rs +++ b/nexus/db-model/src/sled_resource.rs @@ -3,9 +3,19 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use crate::schema::sled_resource; +use crate::typed_uuid::DbTypedUuid; use crate::{ByteCount, SledResourceKind, SqlU32}; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceKind; +use omicron_uuid_kinds::InstanceUuid; +use omicron_uuid_kinds::PropolisUuid; +use omicron_uuid_kinds::SledKind; +use omicron_uuid_kinds::SledUuid; use uuid::Uuid; +type DbInstanceUuid = DbTypedUuid; +type DbSledUuid = DbTypedUuid; + #[derive(Clone, Selectable, Queryable, Insertable, Debug)] #[diesel(table_name = sled_resource)] pub struct Resources { @@ -29,20 +39,28 @@ impl Resources { #[diesel(table_name = sled_resource)] pub struct SledResource { pub id: Uuid, - pub sled_id: Uuid, - pub kind: SledResourceKind, + pub sled_id: DbSledUuid, #[diesel(embed)] pub resources: Resources, + + pub kind: SledResourceKind, + pub instance_id: Option, } impl SledResource { - pub fn new( - id: Uuid, - sled_id: Uuid, - kind: SledResourceKind, + pub fn new_for_vmm( + id: PropolisUuid, + instance_id: InstanceUuid, + sled_id: SledUuid, resources: Resources, ) -> Self { - Self { id, sled_id, kind, resources } + Self { + id: id.into_untyped_uuid(), + instance_id: Some(instance_id.into()), + sled_id: sled_id.into(), + kind: SledResourceKind::Instance, + resources, + } } } diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index 87d5c324c63..ee4e6c31cf7 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -40,6 +40,8 @@ use omicron_common::api::external::ListResultVec; use omicron_common::api::external::ResourceType; use omicron_common::bail_unless; use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceUuid; +use omicron_uuid_kinds::PropolisUuid; use omicron_uuid_kinds::SledUuid; use std::fmt; use strum::IntoEnumIterator; @@ -185,8 +187,8 @@ impl DataStore { pub async fn sled_reservation_create( &self, opctx: &OpContext, - resource_id: Uuid, - resource_kind: db::model::SledResourceKind, + instance_id: InstanceUuid, + propolis_id: PropolisUuid, resources: db::model::Resources, constraints: db::model::SledReservationConstraints, ) -> CreateResult { @@ -210,7 +212,7 @@ impl DataStore { use db::schema::sled_resource::dsl as resource_dsl; // Check if resource ID already exists - if so, return it. let old_resource = resource_dsl::sled_resource - .filter(resource_dsl::id.eq(resource_id)) + .filter(resource_dsl::id.eq(*propolis_id.as_untyped_uuid())) .select(SledResource::as_select()) .limit(1) .load_async(&conn) @@ -309,10 +311,10 @@ impl DataStore { // Create a SledResource record, associate it with the target // sled. - let resource = SledResource::new( - resource_id, - sled_targets[0], - resource_kind, + let resource = SledResource::new_for_vmm( + propolis_id, + instance_id, + SledUuid::from_untyped_uuid(sled_targets[0]), resources, ); @@ -1113,8 +1115,8 @@ pub(in crate::db::datastore) mod test { let error = datastore .sled_reservation_create( &opctx, - Uuid::new_v4(), - db::model::SledResourceKind::Instance, + InstanceUuid::new_v4(), + PropolisUuid::new_v4(), resources.clone(), constraints, ) @@ -1134,15 +1136,15 @@ pub(in crate::db::datastore) mod test { let resource = datastore .sled_reservation_create( &opctx, - Uuid::new_v4(), - db::model::SledResourceKind::Instance, + InstanceUuid::new_v4(), + PropolisUuid::new_v4(), resources.clone(), constraints, ) .await .unwrap(); assert_eq!( - resource.sled_id, + resource.sled_id.into_untyped_uuid(), provisionable_sled.id(), "resource is always allocated to the provisionable sled" ); diff --git a/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs b/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs index ce232a14113..1853a82fb19 100644 --- a/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs +++ b/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs @@ -201,12 +201,12 @@ mod tests { use nexus_db_model::Generation; use nexus_db_model::Resources; use nexus_db_model::SledResource; - use nexus_db_model::SledResourceKind; use nexus_db_model::Vmm; use nexus_db_model::VmmRuntimeState; use nexus_db_model::VmmState; use nexus_test_utils::resource_helpers; use nexus_test_utils_macros::nexus_test; + use omicron_uuid_kinds::InstanceUuid; use uuid::Uuid; type ControlPlaneTestContext = @@ -267,8 +267,8 @@ mod tests { dbg!(datastore .sled_reservation_create( &opctx, - destroyed_vmm_id.into_untyped_uuid(), - SledResourceKind::Instance, + InstanceUuid::from_untyped_uuid(instance.identity.id), + destroyed_vmm_id, resources.clone(), constraints, ) diff --git a/nexus/src/app/sagas/instance_common.rs b/nexus/src/app/sagas/instance_common.rs index 9f9fefea629..22134e8f6d1 100644 --- a/nexus/src/app/sagas/instance_common.rs +++ b/nexus/src/app/sagas/instance_common.rs @@ -38,6 +38,7 @@ pub(super) struct VmmAndSledIds { /// `propolis_id`. pub async fn reserve_vmm_resources( nexus: &Nexus, + instance_id: InstanceUuid, propolis_id: PropolisUuid, ncpus: u32, guest_memory: ByteCount, @@ -50,8 +51,8 @@ pub async fn reserve_vmm_resources( // See https://rfd.shared.oxide.computer/rfd/0205 for a more complete // discussion. // - // Right now, allocate an instance to any random sled agent. This has a few - // problems: + // Right now, allocate an instance to any random sled agent, as long as + // "constraints" and affinity rules are respected. This has a few problems: // // - There's no consideration for "health of the sled" here, other than // "time_deleted = Null". If the sled is rebooting, in a known unhealthy @@ -61,10 +62,6 @@ pub async fn reserve_vmm_resources( // - This is selecting a random sled from all sleds in the cluster. For // multi-rack, this is going to fling the sled to an arbitrary system. // Maybe that's okay, but worth knowing about explicitly. - // - // - This doesn't take into account anti-affinity - users will want to - // schedule instances that belong to a cluster on different failure - // domains. See https://github.com/oxidecomputer/omicron/issues/1705. let resources = db::model::Resources::new( ncpus, ByteCount::try_from(0i64).unwrap(), @@ -73,8 +70,8 @@ pub async fn reserve_vmm_resources( let resource = nexus .reserve_on_random_sled( - propolis_id.into_untyped_uuid(), - nexus_db_model::SledResourceKind::Instance, + instance_id, + propolis_id, resources, constraints, ) diff --git a/nexus/src/app/sagas/instance_migrate.rs b/nexus/src/app/sagas/instance_migrate.rs index 0789ef7a484..465105a813f 100644 --- a/nexus/src/app/sagas/instance_migrate.rs +++ b/nexus/src/app/sagas/instance_migrate.rs @@ -186,6 +186,7 @@ async fn sim_reserve_sled_resources( let resource = super::instance_common::reserve_vmm_resources( osagactx.nexus(), + InstanceUuid::from_untyped_uuid(params.instance.id()), propolis_id, u32::from(params.instance.ncpus.0 .0), params.instance.memory, @@ -193,7 +194,7 @@ async fn sim_reserve_sled_resources( ) .await?; - Ok(SledUuid::from_untyped_uuid(resource.sled_id)) + Ok(resource.sled_id.into()) } async fn sim_release_sled_resources( diff --git a/nexus/src/app/sagas/instance_start.rs b/nexus/src/app/sagas/instance_start.rs index aa09a165c83..e6af6453291 100644 --- a/nexus/src/app/sagas/instance_start.rs +++ b/nexus/src/app/sagas/instance_start.rs @@ -163,6 +163,7 @@ async fn sis_alloc_server( let resource = super::instance_common::reserve_vmm_resources( osagactx.nexus(), + InstanceUuid::from_untyped_uuid(params.db_instance.id()), propolis_id, u32::from(hardware_threads.0), reservoir_ram, @@ -170,7 +171,7 @@ async fn sis_alloc_server( ) .await?; - Ok(SledUuid::from_untyped_uuid(resource.sled_id)) + Ok(resource.sled_id.into()) } async fn sis_alloc_server_undo( diff --git a/nexus/src/app/sled.rs b/nexus/src/app/sled.rs index d58e111b6e6..ffa507ffb8e 100644 --- a/nexus/src/app/sled.rs +++ b/nexus/src/app/sled.rs @@ -25,7 +25,9 @@ use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceUuid; use omicron_uuid_kinds::PhysicalDiskUuid; +use omicron_uuid_kinds::PropolisUuid; use omicron_uuid_kinds::SledUuid; use sled_agent_client::Client as SledAgentClient; use std::net::SocketAddrV6; @@ -164,16 +166,16 @@ impl super::Nexus { pub(crate) async fn reserve_on_random_sled( &self, - resource_id: Uuid, - resource_kind: db::model::SledResourceKind, + instance_id: InstanceUuid, + propolis_id: PropolisUuid, resources: db::model::Resources, constraints: db::model::SledReservationConstraints, ) -> Result { self.db_datastore .sled_reservation_create( &self.opctx_alloc, - resource_id, - resource_kind, + instance_id, + propolis_id, resources, constraints, ) diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 906684e7825..887a6d35a65 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -13,9 +13,9 @@ use super::{ Utilization, Vpc, VpcRouter, VpcSubnet, }, }; -use crate::app::Unimpl; use crate::app::external_endpoints::authority_for_request; use crate::app::support_bundles::SupportBundleQueryType; +use crate::app::Unimpl; use crate::context::ApiContext; use crate::external_api::shared; use dropshot::Body; diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index fa41d17aa96..cbc041f741a 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -17,6 +17,8 @@ use nexus_test_utils::{load_test_config, ControlPlaneTestContextBuilder}; use omicron_common::api::external::SemverVersion; use omicron_common::api::internal::shared::SwitchLocation; use omicron_test_utils::dev::db::{Client, CockroachInstance}; +use omicron_uuid_kinds::InstanceUuid; +use omicron_uuid_kinds::SledUuid; use pretty_assertions::{assert_eq, assert_ne}; use similar_asserts; use slog::Logger; @@ -987,7 +989,8 @@ async fn dbinit_equals_sum_of_all_up() { diesel::insert_into(dsl::sled_resource) .values(SledResource { id: Uuid::new_v4(), - sled_id: Uuid::new_v4(), + instance_id: Some(InstanceUuid::new_v4().into()), + sled_id: SledUuid::new_v4().into(), kind: SledResourceKind::Instance, resources: Resources { hardware_threads: 8_u32.into(), diff --git a/schema/crdb/affinity/up01.sql b/schema/crdb/affinity/up01.sql new file mode 100644 index 00000000000..06e5fb06f94 --- /dev/null +++ b/schema/crdb/affinity/up01.sql @@ -0,0 +1,8 @@ +CREATE TYPE IF NOT EXISTS omicron.public.affinity_policy AS ENUM ( + -- If the affinity request cannot be satisfied, fail. + 'fail', + + -- If the affinity request cannot be satisfied, allow it anyway. + 'allow' +); + diff --git a/schema/crdb/affinity/up02.sql b/schema/crdb/affinity/up02.sql new file mode 100644 index 00000000000..9165e6c4006 --- /dev/null +++ b/schema/crdb/affinity/up02.sql @@ -0,0 +1,5 @@ +CREATE TYPE IF NOT EXISTS omicron.public.failure_domain AS ENUM ( + -- Instances are co-located if they are on the same sled. + 'sled' +); + diff --git a/schema/crdb/affinity/up03.sql b/schema/crdb/affinity/up03.sql new file mode 100644 index 00000000000..79238402f8d --- /dev/null +++ b/schema/crdb/affinity/up03.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS omicron.public.affinity_group ( + id UUID PRIMARY KEY, + name STRING(63) NOT NULL, + description STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + -- Affinity groups are contained within projects + project_id UUID NOT NULL, + policy omicron.public.affinity_policy NOT NULL, + failure_domain omicron.public.failure_domain NOT NULL +); + diff --git a/schema/crdb/affinity/up04.sql b/schema/crdb/affinity/up04.sql new file mode 100644 index 00000000000..3f455884b98 --- /dev/null +++ b/schema/crdb/affinity/up04.sql @@ -0,0 +1,7 @@ +CREATE UNIQUE INDEX IF NOT EXISTS lookup_affinity_group_by_project ON omicron.public.affinity_group ( + project_id, + name +) WHERE + time_deleted IS NULL; + + diff --git a/schema/crdb/affinity/up05.sql b/schema/crdb/affinity/up05.sql new file mode 100644 index 00000000000..87f5f9e9c11 --- /dev/null +++ b/schema/crdb/affinity/up05.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS omicron.public.affinity_group_instance_membership ( + group_id UUID NOT NULL, + instance_id UUID NOT NULL, + + PRIMARY KEY (group_id, instance_id) +); + diff --git a/schema/crdb/affinity/up06.sql b/schema/crdb/affinity/up06.sql new file mode 100644 index 00000000000..f86037bda3f --- /dev/null +++ b/schema/crdb/affinity/up06.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_affinity_group_instance_membership_by_instance ON omicron.public.affinity_group_instance_membership ( + instance_id +); + diff --git a/schema/crdb/affinity/up07.sql b/schema/crdb/affinity/up07.sql new file mode 100644 index 00000000000..0409c1c4c7b --- /dev/null +++ b/schema/crdb/affinity/up07.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS omicron.public.anti_affinity_group ( + id UUID PRIMARY KEY, + name STRING(63) NOT NULL, + description STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + -- Anti-Affinity groups are contained within projects + project_id UUID NOT NULL, + policy omicron.public.affinity_policy NOT NULL, + failure_domain omicron.public.failure_domain NOT NULL +); + diff --git a/schema/crdb/affinity/up08.sql b/schema/crdb/affinity/up08.sql new file mode 100644 index 00000000000..536fcf35e37 --- /dev/null +++ b/schema/crdb/affinity/up08.sql @@ -0,0 +1,6 @@ +CREATE UNIQUE INDEX IF NOT EXISTS lookup_anti_affinity_group_by_project ON omicron.public.anti_affinity_group ( + project_id, + name +) WHERE + time_deleted IS NULL; + diff --git a/schema/crdb/affinity/up09.sql b/schema/crdb/affinity/up09.sql new file mode 100644 index 00000000000..9b3333c934e --- /dev/null +++ b/schema/crdb/affinity/up09.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS omicron.public.anti_affinity_group_instance_membership ( + group_id UUID NOT NULL, + instance_id UUID NOT NULL, + + PRIMARY KEY (group_id, instance_id) +); + diff --git a/schema/crdb/affinity/up10.sql b/schema/crdb/affinity/up10.sql new file mode 100644 index 00000000000..c57d98310c5 --- /dev/null +++ b/schema/crdb/affinity/up10.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_anti_affinity_group_instance_membership_by_instance ON omicron.public.anti_affinity_group_instance_membership ( + instance_id +); + diff --git a/schema/crdb/affinity/up11.sql b/schema/crdb/affinity/up11.sql new file mode 100644 index 00000000000..e3b328c326d --- /dev/null +++ b/schema/crdb/affinity/up11.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.sled_resource ADD COLUMN IF NOT EXISTS instance_id UUID; diff --git a/schema/crdb/affinity/up12.sql b/schema/crdb/affinity/up12.sql new file mode 100644 index 00000000000..1b76693bf29 --- /dev/null +++ b/schema/crdb/affinity/up12.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_resource_by_instance ON omicron.public.sled_resource ( + instance_id +); + diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 1ee7b985aff..a2fa3d48e23 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -217,7 +217,7 @@ CREATE INDEX IF NOT EXISTS lookup_sled_by_policy_and_state ON omicron.public.sle ); CREATE TYPE IF NOT EXISTS omicron.public.sled_resource_kind AS ENUM ( - -- omicron.public.instance + -- omicron.public.vmm ; this is called "instance" for historical reasons. 'instance' -- We expect to other resource kinds here in the future; e.g., to track -- resources used by control plane services. For now, we only track @@ -242,7 +242,13 @@ CREATE TABLE IF NOT EXISTS omicron.public.sled_resource ( reservoir_ram INT8 NOT NULL, -- Identifies the type of the resource - kind omicron.public.sled_resource_kind NOT NULL + kind omicron.public.sled_resource_kind NOT NULL, + + -- The UUID of an instance, if this resource belongs to an instance. + instance_id UUID + + -- TODO Add constraint that if kind is instance, instance_id is not NULL? + -- Or will that break backwards compatibility? ); -- Allow looking up all resources which reside on a sled @@ -251,6 +257,10 @@ CREATE UNIQUE INDEX IF NOT EXISTS lookup_resource_by_sled ON omicron.public.sled id ); +-- Allow looking up all resources by instance +CREATE INDEX IF NOT EXISTS lookup_resource_by_instance ON omicron.public.sled_resource ( + instance_id +); -- Table of all sled subnets allocated for sleds added to an already initialized -- rack. The sleds in this table and their allocated subnets are created before @@ -4094,6 +4104,94 @@ CREATE INDEX IF NOT EXISTS lookup_usable_rendezvous_debug_dataset COMMIT; BEGIN; +-- Describes what happens when +-- (for affinity groups) instance cannot be co-located, or +-- (for anti-affinity groups) instance must be co-located, or +CREATE TYPE IF NOT EXISTS omicron.public.affinity_policy AS ENUM ( + -- If the affinity request cannot be satisfied, fail. + 'fail', + + -- If the affinity request cannot be satisfied, allow it anyway. + 'allow' +); + +-- Determines what "co-location" means for instances within an affinity +-- or anti-affinity group. +CREATE TYPE IF NOT EXISTS omicron.public.failure_domain AS ENUM ( + -- Instances are co-located if they are on the same sled. + 'sled' +); + +-- Describes a grouping of related instances that should be co-located. +CREATE TABLE IF NOT EXISTS omicron.public.affinity_group ( + id UUID PRIMARY KEY, + name STRING(63) NOT NULL, + description STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + -- Affinity groups are contained within projects + project_id UUID NOT NULL, + policy omicron.public.affinity_policy NOT NULL, + failure_domain omicron.public.failure_domain NOT NULL +); + +-- Names for affinity groups within a project should be unique +CREATE UNIQUE INDEX IF NOT EXISTS lookup_affinity_group_by_project ON omicron.public.affinity_group ( + project_id, + name +) WHERE + time_deleted IS NULL; + +-- Describes an instance's membership within an affinity group +CREATE TABLE IF NOT EXISTS omicron.public.affinity_group_instance_membership ( + group_id UUID NOT NULL, + instance_id UUID NOT NULL, + + PRIMARY KEY (group_id, instance_id) +); + +-- We need to look up all memberships of an instance so we can revoke these +-- memberships efficiently when instances are deleted. +CREATE INDEX IF NOT EXISTS lookup_affinity_group_instance_membership_by_instance ON omicron.public.affinity_group_instance_membership ( + instance_id +); + +-- Describes a collection of instances that should not be co-located. +CREATE TABLE IF NOT EXISTS omicron.public.anti_affinity_group ( + id UUID PRIMARY KEY, + name STRING(63) NOT NULL, + description STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + -- Anti-Affinity groups are contained within projects + project_id UUID NOT NULL, + policy omicron.public.affinity_policy NOT NULL, + failure_domain omicron.public.failure_domain NOT NULL +); + +-- Names for anti-affinity groups within a project should be unique +CREATE UNIQUE INDEX IF NOT EXISTS lookup_anti_affinity_group_by_project ON omicron.public.anti_affinity_group ( + project_id, + name +) WHERE + time_deleted IS NULL; + +-- Describes an instance's membership within an anti-affinity group +CREATE TABLE IF NOT EXISTS omicron.public.anti_affinity_group_instance_membership ( + group_id UUID NOT NULL, + instance_id UUID NOT NULL, + + PRIMARY KEY (group_id, instance_id) +); + +-- We need to look up all memberships of an instance so we can revoke these +-- memberships efficiently when instances are deleted. +CREATE INDEX IF NOT EXISTS lookup_anti_affinity_group_instance_membership_by_instance ON omicron.public.anti_affinity_group_instance_membership ( + instance_id +); + -- Per-VMM state. CREATE TABLE IF NOT EXISTS omicron.public.vmm ( id UUID PRIMARY KEY, @@ -4810,7 +4908,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '122.0.0', NULL) + (TRUE, NOW(), NOW(), '123.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/uuid-kinds/src/lib.rs b/uuid-kinds/src/lib.rs index 1d6dc600226..a27fc43b8e0 100644 --- a/uuid-kinds/src/lib.rs +++ b/uuid-kinds/src/lib.rs @@ -51,6 +51,8 @@ macro_rules! impl_typed_uuid_kind { // Please keep this list in alphabetical order. impl_typed_uuid_kind! { + AffinityGroup => "affinity_group", + AntiAffinityGroup => "anti_affinity_group", Blueprint => "blueprint", Collection => "collection", Dataset => "dataset", From 8f1d37ceae8373cef2e2b815e26c556655f07b32 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jan 2025 12:25:54 -0800 Subject: [PATCH 03/14] [nexus] Add CRUD implementations for Affinity/Anti-Affinity Groups --- nexus/auth/src/authz/api_resources.rs | 16 + nexus/auth/src/authz/oso_generic.rs | 2 + nexus/db-queries/src/db/datastore/affinity.rs | 2701 +++++++++++++++++ nexus/db-queries/src/db/datastore/instance.rs | 7 + nexus/db-queries/src/db/datastore/mod.rs | 1 + nexus/db-queries/src/db/datastore/project.rs | 5 + nexus/db-queries/src/db/lookup.rs | 30 +- 7 files changed, 2761 insertions(+), 1 deletion(-) create mode 100644 nexus/db-queries/src/db/datastore/affinity.rs diff --git a/nexus/auth/src/authz/api_resources.rs b/nexus/auth/src/authz/api_resources.rs index 745a699cf2b..4d588036b7e 100644 --- a/nexus/auth/src/authz/api_resources.rs +++ b/nexus/auth/src/authz/api_resources.rs @@ -713,6 +713,22 @@ authz_resource! { polar_snippet = InProject, } +authz_resource! { + name = "AffinityGroup", + parent = "Project", + primary_key = Uuid, + roles_allowed = false, + polar_snippet = InProject, +} + +authz_resource! { + name = "AntiAffinityGroup", + parent = "Project", + primary_key = Uuid, + roles_allowed = false, + polar_snippet = InProject, +} + authz_resource! { name = "InstanceNetworkInterface", parent = "Instance", diff --git a/nexus/auth/src/authz/oso_generic.rs b/nexus/auth/src/authz/oso_generic.rs index 321bb98b1c6..32b3dbd1f80 100644 --- a/nexus/auth/src/authz/oso_generic.rs +++ b/nexus/auth/src/authz/oso_generic.rs @@ -125,6 +125,8 @@ pub fn make_omicron_oso(log: &slog::Logger) -> Result { Disk::init(), Snapshot::init(), ProjectImage::init(), + AffinityGroup::init(), + AntiAffinityGroup::init(), Instance::init(), IpPool::init(), InstanceNetworkInterface::init(), diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs new file mode 100644 index 00000000000..e09a0e43406 --- /dev/null +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -0,0 +1,2701 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! [`DataStore`] methods on Affinity Groups + +use super::DataStore; +use crate::authz; +use crate::authz::ApiResource; +use crate::db; +use crate::db::collection_insert::AsyncInsertError; +use crate::db::collection_insert::DatastoreCollection; +use crate::db::datastore::OpContext; +use crate::db::error::public_error_from_diesel; +use crate::db::error::ErrorHandler; +use crate::db::identity::Resource; +use crate::db::model::AffinityGroup; +use crate::db::model::AffinityGroupInstanceMembership; +use crate::db::model::AffinityGroupUpdate; +use crate::db::model::AntiAffinityGroup; +use crate::db::model::AntiAffinityGroupInstanceMembership; +use crate::db::model::AntiAffinityGroupUpdate; +use crate::db::model::InstanceState; +use crate::db::model::Name; +use crate::db::model::Project; +use crate::db::pagination::paginated; +use crate::transaction_retry::OptionalError; +use async_bb8_diesel::AsyncRunQueryDsl; +use chrono::Utc; +use diesel::prelude::*; +use omicron_common::api::external; +use omicron_common::api::external::http_pagination::PaginatedBy; +use omicron_common::api::external::CreateResult; +use omicron_common::api::external::DeleteResult; +use omicron_common::api::external::Error; +use omicron_common::api::external::ListResultVec; +use omicron_common::api::external::LookupType; +use omicron_common::api::external::ResourceType; +use omicron_common::api::external::UpdateResult; +use omicron_uuid_kinds::AffinityGroupUuid; +use omicron_uuid_kinds::AntiAffinityGroupUuid; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceUuid; +use ref_cast::RefCast; + +impl DataStore { + pub async fn affinity_group_list( + &self, + opctx: &OpContext, + authz_project: &authz::Project, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + use db::schema::affinity_group::dsl; + + opctx.authorize(authz::Action::ListChildren, authz_project).await?; + + match pagparams { + PaginatedBy::Id(pagparams) => { + paginated(dsl::affinity_group, dsl::id, &pagparams) + } + PaginatedBy::Name(pagparams) => paginated( + dsl::affinity_group, + dsl::name, + &pagparams.map_name(|n| Name::ref_cast(n)), + ), + } + .filter(dsl::project_id.eq(authz_project.id())) + .filter(dsl::time_deleted.is_null()) + .select(AffinityGroup::as_select()) + .get_results_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn anti_affinity_group_list( + &self, + opctx: &OpContext, + authz_project: &authz::Project, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + use db::schema::anti_affinity_group::dsl; + + opctx.authorize(authz::Action::ListChildren, authz_project).await?; + + match pagparams { + PaginatedBy::Id(pagparams) => { + paginated(dsl::anti_affinity_group, dsl::id, &pagparams) + } + PaginatedBy::Name(pagparams) => paginated( + dsl::anti_affinity_group, + dsl::name, + &pagparams.map_name(|n| Name::ref_cast(n)), + ), + } + .filter(dsl::project_id.eq(authz_project.id())) + .filter(dsl::time_deleted.is_null()) + .select(AntiAffinityGroup::as_select()) + .get_results_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn affinity_group_create( + &self, + opctx: &OpContext, + authz_project: &authz::Project, + group: AffinityGroup, + ) -> CreateResult { + use db::schema::affinity_group::dsl; + + opctx.authorize(authz::Action::CreateChild, authz_project).await?; + + let conn = self.pool_connection_authorized(opctx).await?; + let name = group.name().as_str().to_string(); + + let affinity_group: AffinityGroup = Project::insert_resource( + authz_project.id(), + diesel::insert_into(dsl::affinity_group).values(group), + ) + .insert_and_get_result_async(&conn) + .await + .map_err(|e| match e { + AsyncInsertError::CollectionNotFound => authz_project.not_found(), + AsyncInsertError::DatabaseError(diesel_error) => { + public_error_from_diesel( + diesel_error, + ErrorHandler::Conflict(ResourceType::AffinityGroup, &name), + ) + } + })?; + Ok(affinity_group) + } + + pub async fn anti_affinity_group_create( + &self, + opctx: &OpContext, + authz_project: &authz::Project, + group: AntiAffinityGroup, + ) -> CreateResult { + use db::schema::anti_affinity_group::dsl; + + opctx.authorize(authz::Action::CreateChild, authz_project).await?; + + let conn = self.pool_connection_authorized(opctx).await?; + let name = group.name().as_str().to_string(); + + let anti_affinity_group: AntiAffinityGroup = Project::insert_resource( + authz_project.id(), + diesel::insert_into(dsl::anti_affinity_group).values(group), + ) + .insert_and_get_result_async(&conn) + .await + .map_err(|e| match e { + AsyncInsertError::CollectionNotFound => authz_project.not_found(), + AsyncInsertError::DatabaseError(diesel_error) => { + public_error_from_diesel( + diesel_error, + ErrorHandler::Conflict( + ResourceType::AntiAffinityGroup, + &name, + ), + ) + } + })?; + Ok(anti_affinity_group) + } + + pub async fn affinity_group_update( + &self, + opctx: &OpContext, + authz_affinity_group: &authz::AffinityGroup, + updates: AffinityGroupUpdate, + ) -> UpdateResult { + opctx.authorize(authz::Action::Modify, authz_affinity_group).await?; + + use db::schema::affinity_group::dsl; + diesel::update(dsl::affinity_group) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::id.eq(authz_affinity_group.id())) + .set(updates) + .returning(AffinityGroup::as_returning()) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_affinity_group), + ) + }) + } + + pub async fn affinity_group_delete( + &self, + opctx: &OpContext, + authz_affinity_group: &authz::AffinityGroup, + ) -> DeleteResult { + opctx.authorize(authz::Action::Delete, authz_affinity_group).await?; + + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("affinity_group_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + use db::schema::affinity_group::dsl as group_dsl; + let now = Utc::now(); + + // Delete the Affinity Group + diesel::update(group_dsl::affinity_group) + .filter(group_dsl::time_deleted.is_null()) + .filter(group_dsl::id.eq(authz_affinity_group.id())) + .set(group_dsl::time_deleted.eq(now)) + .returning(AffinityGroup::as_returning()) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_affinity_group, + ), + ) + }) + })?; + + // Ensure all memberships in the affinity group are deleted + use db::schema::affinity_group_instance_membership::dsl as member_dsl; + diesel::delete(member_dsl::affinity_group_instance_membership) + .filter(member_dsl::group_id.eq(authz_affinity_group.id())) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + })?; + + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } + + pub async fn anti_affinity_group_update( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + updates: AntiAffinityGroupUpdate, + ) -> UpdateResult { + opctx + .authorize(authz::Action::Modify, authz_anti_affinity_group) + .await?; + + use db::schema::anti_affinity_group::dsl; + diesel::update(dsl::anti_affinity_group) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::id.eq(authz_anti_affinity_group.id())) + .set(updates) + .returning(AntiAffinityGroup::as_returning()) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_anti_affinity_group), + ) + }) + } + + pub async fn anti_affinity_group_delete( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + ) -> DeleteResult { + opctx + .authorize(authz::Action::Delete, authz_anti_affinity_group) + .await?; + + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("anti_affinity_group_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + use db::schema::anti_affinity_group::dsl as group_dsl; + let now = Utc::now(); + + // Delete the Anti Affinity Group + diesel::update(group_dsl::anti_affinity_group) + .filter(group_dsl::time_deleted.is_null()) + .filter(group_dsl::id.eq(authz_anti_affinity_group.id())) + .set(group_dsl::time_deleted.eq(now)) + .returning(AntiAffinityGroup::as_returning()) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_anti_affinity_group, + ), + ) + }) + })?; + + // Ensure all memberships in the anti affinity group are deleted + use db::schema::anti_affinity_group_instance_membership::dsl as member_dsl; + diesel::delete(member_dsl::anti_affinity_group_instance_membership) + .filter(member_dsl::group_id.eq(authz_anti_affinity_group.id())) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + })?; + + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } + + pub async fn affinity_group_member_list( + &self, + opctx: &OpContext, + authz_affinity_group: &authz::AffinityGroup, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + opctx.authorize(authz::Action::Read, authz_affinity_group).await?; + + use db::schema::affinity_group_instance_membership::dsl; + match pagparams { + PaginatedBy::Id(pagparams) => paginated( + dsl::affinity_group_instance_membership, + dsl::instance_id, + &pagparams, + ), + PaginatedBy::Name(_) => { + return Err(Error::invalid_request( + "Cannot paginate group members by name", + )); + } + } + .filter(dsl::group_id.eq(authz_affinity_group.id())) + .select(AffinityGroupInstanceMembership::as_select()) + .load_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn anti_affinity_group_member_list( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + opctx.authorize(authz::Action::Read, authz_anti_affinity_group).await?; + + use db::schema::anti_affinity_group_instance_membership::dsl; + match pagparams { + PaginatedBy::Id(pagparams) => paginated( + dsl::anti_affinity_group_instance_membership, + dsl::instance_id, + &pagparams, + ), + PaginatedBy::Name(_) => { + return Err(Error::invalid_request( + "Cannot paginate group members by name", + )); + } + } + .filter(dsl::group_id.eq(authz_anti_affinity_group.id())) + .select(AntiAffinityGroupInstanceMembership::as_select()) + .load_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn affinity_group_member_view( + &self, + opctx: &OpContext, + authz_affinity_group: &authz::AffinityGroup, + member: external::AffinityGroupMember, + ) -> Result { + opctx.authorize(authz::Action::Read, authz_affinity_group).await?; + let conn = self.pool_connection_authorized(opctx).await?; + + let instance_id = match member { + external::AffinityGroupMember::Instance(id) => id, + }; + + use db::schema::affinity_group_instance_membership::dsl; + dsl::affinity_group_instance_membership + .filter(dsl::group_id.eq(authz_affinity_group.id())) + .filter(dsl::instance_id.eq(instance_id)) + .select(AffinityGroupInstanceMembership::as_select()) + .get_result_async(&*conn) + .await + .map(|m| m.into()) + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::AffinityGroupMember, + LookupType::by_id(instance_id), + ), + ) + }) + } + + pub async fn anti_affinity_group_member_view( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + member: external::AntiAffinityGroupMember, + ) -> Result { + opctx.authorize(authz::Action::Read, authz_anti_affinity_group).await?; + let conn = self.pool_connection_authorized(opctx).await?; + + let instance_id = match member { + external::AntiAffinityGroupMember::Instance(id) => id, + }; + + use db::schema::anti_affinity_group_instance_membership::dsl; + dsl::anti_affinity_group_instance_membership + .filter(dsl::group_id.eq(authz_anti_affinity_group.id())) + .filter(dsl::instance_id.eq(instance_id)) + .select(AntiAffinityGroupInstanceMembership::as_select()) + .get_result_async(&*conn) + .await + .map(|m| m.into()) + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::AntiAffinityGroupMember, + LookupType::by_id(instance_id), + ), + ) + }) + } + + pub async fn affinity_group_member_add( + &self, + opctx: &OpContext, + authz_affinity_group: &authz::AffinityGroup, + member: external::AffinityGroupMember, + ) -> Result<(), Error> { + opctx.authorize(authz::Action::Modify, authz_affinity_group).await?; + + let instance_id = match member { + external::AffinityGroupMember::Instance(id) => id, + }; + + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("affinity_group_member_add") + .transaction(&conn, |conn| { + let err = err.clone(); + use db::schema::affinity_group::dsl as group_dsl; + use db::schema::affinity_group_instance_membership::dsl as membership_dsl; + use db::schema::instance::dsl as instance_dsl; + + async move { + // Check that the group exists + group_dsl::affinity_group + .filter(group_dsl::time_deleted.is_null()) + .filter(group_dsl::id.eq(authz_affinity_group.id())) + .select(group_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_affinity_group, + ), + ) + }) + })?; + + // Check that the instance exists, and has no VMM + // + // NOTE: I'd prefer to use the "LookupPath" infrastructure + // to look up the path, but that API does not give the + // option to use the transaction's database connection. + // + // Looking up the instance on a different database + // connection than the transaction risks several concurrency + // issues, so we do the lookup manually. + let instance_state = instance_dsl::instance + .filter(instance_dsl::time_deleted.is_null()) + .filter(instance_dsl::id.eq(instance_id)) + .select(instance_dsl::state) + .first_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Instance, + LookupType::ById(instance_id) + ), + ) + }) + })?; + + // NOTE: It may be possible to add non-stopped instances to + // affinity groups, depending on where they have already + // been placed. However, only operating on "stopped" + // instances is much easier to work with, as it does not + // require any understanding of the group policy. + match instance_state { + InstanceState::NoVmm => (), + other => { + return Err(err.bail(Error::invalid_request( + format!( + "Instance cannot be added to affinity group in state: {other}" + ) + ))); + }, + } + + diesel::insert_into(membership_dsl::affinity_group_instance_membership) + .values(AffinityGroupInstanceMembership::new( + AffinityGroupUuid::from_untyped_uuid(authz_affinity_group.id()), + InstanceUuid::from_untyped_uuid(instance_id), + )) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::AffinityGroupMember, + &instance_id.to_string(), + ), + ) + }) + })?; + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } + + pub async fn anti_affinity_group_member_add( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + member: external::AntiAffinityGroupMember, + ) -> Result<(), Error> { + opctx + .authorize(authz::Action::Modify, authz_anti_affinity_group) + .await?; + + let instance_id = match member { + external::AntiAffinityGroupMember::Instance(id) => id, + }; + + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("anti_affinity_group_member_add") + .transaction(&conn, |conn| { + let err = err.clone(); + use db::schema::anti_affinity_group::dsl as group_dsl; + use db::schema::anti_affinity_group_instance_membership::dsl as membership_dsl; + use db::schema::instance::dsl as instance_dsl; + + async move { + // Check that the group exists + group_dsl::anti_affinity_group + .filter(group_dsl::time_deleted.is_null()) + .filter(group_dsl::id.eq(authz_anti_affinity_group.id())) + .select(group_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_anti_affinity_group, + ), + ) + }) + })?; + + // Check that the instance exists, and has no VMM + let instance_state = instance_dsl::instance + .filter(instance_dsl::time_deleted.is_null()) + .filter(instance_dsl::id.eq(instance_id)) + .select(instance_dsl::state) + .first_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Instance, + LookupType::ById(instance_id) + ), + ) + }) + })?; + + // NOTE: It may be possible to add non-stopped instances to + // anti affinity groups, depending on where they have already + // been placed. However, only operating on "stopped" + // instances is much easier to work with, as it does not + // require any understanding of the group policy. + match instance_state { + InstanceState::NoVmm => (), + other => { + return Err(err.bail(Error::invalid_request( + format!( + "Instance cannot be added to anti-affinity group in state: {other}" + ) + ))); + }, + } + + diesel::insert_into(membership_dsl::anti_affinity_group_instance_membership) + .values(AntiAffinityGroupInstanceMembership::new( + AntiAffinityGroupUuid::from_untyped_uuid(authz_anti_affinity_group.id()), + InstanceUuid::from_untyped_uuid(instance_id), + )) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::AntiAffinityGroupMember, + &instance_id.to_string(), + ), + ) + }) + })?; + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } + + pub async fn instance_affinity_group_memberships_delete( + &self, + opctx: &OpContext, + instance_id: InstanceUuid, + ) -> Result<(), Error> { + use db::schema::affinity_group_instance_membership::dsl; + + diesel::delete(dsl::affinity_group_instance_membership) + .filter(dsl::instance_id.eq(instance_id.into_untyped_uuid())) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + Ok(()) + } + + pub async fn instance_anti_affinity_group_memberships_delete( + &self, + opctx: &OpContext, + instance_id: InstanceUuid, + ) -> Result<(), Error> { + use db::schema::anti_affinity_group_instance_membership::dsl; + + diesel::delete(dsl::anti_affinity_group_instance_membership) + .filter(dsl::instance_id.eq(instance_id.into_untyped_uuid())) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + Ok(()) + } + + pub async fn affinity_group_member_delete( + &self, + opctx: &OpContext, + authz_affinity_group: &authz::AffinityGroup, + member: external::AffinityGroupMember, + ) -> Result<(), Error> { + opctx.authorize(authz::Action::Modify, authz_affinity_group).await?; + + let instance_id = match member { + external::AffinityGroupMember::Instance(id) => id, + }; + + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("affinity_group_member_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + use db::schema::affinity_group::dsl as group_dsl; + use db::schema::affinity_group_instance_membership::dsl as membership_dsl; + use db::schema::instance::dsl as instance_dsl; + + async move { + // Check that the group exists + group_dsl::affinity_group + .filter(group_dsl::time_deleted.is_null()) + .filter(group_dsl::id.eq(authz_affinity_group.id())) + .select(group_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_affinity_group, + ), + ) + }) + })?; + + // Check that the instance exists + instance_dsl::instance + .filter(instance_dsl::time_deleted.is_null()) + .filter(instance_dsl::id.eq(instance_id)) + .select(instance_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Instance, + LookupType::ById(instance_id) + ), + ) + }) + })?; + + let rows = diesel::delete(membership_dsl::affinity_group_instance_membership) + .filter(membership_dsl::group_id.eq(authz_affinity_group.id())) + .filter(membership_dsl::instance_id.eq(instance_id)) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + })?; + if rows == 0 { + return Err(err.bail(LookupType::ById(instance_id).into_not_found( + ResourceType::AffinityGroupMember, + ))); + } + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } + + pub async fn anti_affinity_group_member_delete( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + member: external::AntiAffinityGroupMember, + ) -> Result<(), Error> { + opctx + .authorize(authz::Action::Modify, authz_anti_affinity_group) + .await?; + + let instance_id = match member { + external::AntiAffinityGroupMember::Instance(id) => id, + }; + + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("anti_affinity_group_member_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + use db::schema::anti_affinity_group::dsl as group_dsl; + use db::schema::anti_affinity_group_instance_membership::dsl as membership_dsl; + use db::schema::instance::dsl as instance_dsl; + + async move { + // Check that the group exists + group_dsl::anti_affinity_group + .filter(group_dsl::time_deleted.is_null()) + .filter(group_dsl::id.eq(authz_anti_affinity_group.id())) + .select(group_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_anti_affinity_group, + ), + ) + }) + })?; + + // Check that the instance exists + instance_dsl::instance + .filter(instance_dsl::time_deleted.is_null()) + .filter(instance_dsl::id.eq(instance_id)) + .select(instance_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Instance, + LookupType::ById(instance_id) + ), + ) + }) + })?; + + let rows = diesel::delete(membership_dsl::anti_affinity_group_instance_membership) + .filter(membership_dsl::group_id.eq(authz_anti_affinity_group.id())) + .filter(membership_dsl::instance_id.eq(instance_id)) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + })?; + if rows == 0 { + return Err(err.bail(LookupType::ById(instance_id).into_not_found( + ResourceType::AntiAffinityGroupMember, + ))); + } + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use crate::db::lookup::LookupPath; + use crate::db::pub_test_utils::TestDatabase; + use nexus_db_model::Instance; + use nexus_types::external_api::params; + use omicron_common::api::external::{ + self, ByteCount, DataPageParams, IdentityMetadataCreateParams, + }; + use omicron_test_utils::dev; + use omicron_uuid_kinds::InstanceUuid; + use std::num::NonZeroU32; + + // Helper function for creating a project + async fn create_project( + opctx: &OpContext, + datastore: &DataStore, + name: &str, + ) -> (authz::Project, Project) { + let authz_silo = opctx.authn.silo_required().unwrap(); + let project = Project::new( + authz_silo.id(), + params::ProjectCreate { + identity: IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "".to_string(), + }, + }, + ); + datastore.project_create(&opctx, project).await.unwrap() + } + + // Helper function for creating an affinity group with + // arbitrary configuration. + async fn create_affinity_group( + opctx: &OpContext, + datastore: &DataStore, + authz_project: &authz::Project, + name: &str, + ) -> CreateResult { + let group = AffinityGroup::new( + authz_project.id(), + params::AffinityGroupCreate { + identity: IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "".to_string(), + }, + policy: external::AffinityPolicy::Fail, + failure_domain: external::FailureDomain::Sled, + }, + ); + datastore.affinity_group_create(&opctx, &authz_project, group).await + } + + // Helper function for creating an anti-affinity group with + // arbitrary configuration. + async fn create_anti_affinity_group( + opctx: &OpContext, + datastore: &DataStore, + authz_project: &authz::Project, + name: &str, + ) -> CreateResult { + let group = AntiAffinityGroup::new( + authz_project.id(), + params::AntiAffinityGroupCreate { + identity: IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "".to_string(), + }, + policy: external::AffinityPolicy::Fail, + failure_domain: external::FailureDomain::Sled, + }, + ); + datastore + .anti_affinity_group_create(&opctx, &authz_project, group) + .await + } + + // Helper function for creating an instance without a VMM. + async fn create_instance_record( + opctx: &OpContext, + datastore: &DataStore, + authz_project: &authz::Project, + name: &str, + ) -> Instance { + let instance = Instance::new( + InstanceUuid::new_v4(), + authz_project.id(), + ¶ms::InstanceCreate { + identity: IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: "".to_string(), + }, + ncpus: 2i64.try_into().unwrap(), + memory: ByteCount::from_gibibytes_u32(16), + hostname: "myhostname".try_into().unwrap(), + user_data: Vec::new(), + network_interfaces: + params::InstanceNetworkInterfaceAttachment::None, + external_ips: Vec::new(), + disks: Vec::new(), + boot_disk: None, + ssh_public_keys: None, + start: false, + auto_restart_policy: Default::default(), + }, + ); + + let instance = datastore + .project_create_instance(&opctx, &authz_project, instance) + .await + .unwrap(); + + set_instance_state_stopped(&datastore, instance.id()).await; + + instance + } + + // Helper for explicitly modifying instance state. + // + // The interaction we typically use to create and modify instance state + // is more complex in production, since it's the result of a back-and-forth + // between Nexus and Sled Agent, using carefully crafted rcgen values. + // + // Here, we just set the value of state explicitly. Be warned, there + // are no guardrails! + async fn set_instance_state_stopped( + datastore: &DataStore, + instance: uuid::Uuid, + ) { + use db::schema::instance::dsl; + diesel::update(dsl::instance) + .filter(dsl::id.eq(instance)) + .set(( + dsl::state.eq(db::model::InstanceState::NoVmm), + dsl::active_propolis_id.eq(None::), + )) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) + .await + .unwrap(); + } + + async fn set_instance_state_running( + datastore: &DataStore, + instance: uuid::Uuid, + ) { + use db::schema::instance::dsl; + diesel::update(dsl::instance) + .filter(dsl::id.eq(instance)) + .set(( + dsl::state.eq(db::model::InstanceState::Vmm), + dsl::active_propolis_id.eq(uuid::Uuid::new_v4()), + )) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn affinity_groups_are_project_scoped() { + // Setup + let logctx = dev::test_setup_log("affinity_groups_are_project_scoped"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + let (authz_project, _) = + create_project(&opctx, &datastore, "my-project").await; + + let (authz_other_project, _) = + create_project(&opctx, &datastore, "my-other-project").await; + + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + + // To start: No groups exist + let groups = datastore + .affinity_group_list(&opctx, &authz_project, &pagbyid) + .await + .unwrap(); + assert!(groups.is_empty()); + + // Create a group + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + // Now when we list groups, we'll see the one we created. + let groups = datastore + .affinity_group_list(&opctx, &authz_project, &pagbyid) + .await + .unwrap(); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0], group); + + // This group won't appear in the other project + let groups = datastore + .affinity_group_list(&opctx, &authz_other_project, &pagbyid) + .await + .unwrap(); + assert!(groups.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_groups_are_project_scoped() { + // Setup + let logctx = + dev::test_setup_log("anti_affinity_groups_are_project_scoped"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + let (authz_project, _) = + create_project(&opctx, &datastore, "my-project").await; + + let (authz_other_project, _) = + create_project(&opctx, &datastore, "my-other-project").await; + + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + + // To start: No groups exist + let groups = datastore + .anti_affinity_group_list(&opctx, &authz_project, &pagbyid) + .await + .unwrap(); + assert!(groups.is_empty()); + + // Create a group + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + // Now when we list groups, we'll see the one we created. + let groups = datastore + .anti_affinity_group_list(&opctx, &authz_project, &pagbyid) + .await + .unwrap(); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0], group); + + // This group won't appear in the other project + let groups = datastore + .anti_affinity_group_list(&opctx, &authz_other_project, &pagbyid) + .await + .unwrap(); + assert!(groups.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_groups_prevent_project_deletion() { + // Setup + let logctx = + dev::test_setup_log("affinity_groups_prevent_project_deletion"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, mut project) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + // If we try to delete the project, we'll fail. + let err = datastore + .project_delete(&opctx, &authz_project, &project) + .await + .unwrap_err(); + assert!(matches!(err, Error::InvalidRequest { .. })); + assert!( + err.to_string() + .contains("project to be deleted contains an affinity group"), + "{err:?}" + ); + + // Delete the group, then try to delete the project again. + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + datastore.affinity_group_delete(&opctx, &authz_group).await.unwrap(); + + // When the group was created, it bumped the rcgen in the project. If we + // have an old view of the project, we expect a "concurrent + // modification" error. + let err = datastore + .project_delete(&opctx, &authz_project, &project) + .await + .unwrap_err(); + assert!(err.to_string().contains("concurrent modification"), "{err:?}"); + + // If we update rcgen, however, and the group has been deleted, + // we can successfully delete the project. + project.rcgen = project.rcgen.next().into(); + datastore + .project_delete(&opctx, &authz_project, &project) + .await + .unwrap(); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_groups_prevent_project_deletion() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_groups_prevent_project_deletion", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, mut project) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + // If we try to delete the project, we'll fail. + let err = datastore + .project_delete(&opctx, &authz_project, &project) + .await + .unwrap_err(); + assert!(matches!(err, Error::InvalidRequest { .. })); + assert!( + err.to_string().contains( + "project to be deleted contains an anti affinity group" + ), + "{err:?}" + ); + + // Delete the group, then try to delete the project again. + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + datastore + .anti_affinity_group_delete(&opctx, &authz_group) + .await + .unwrap(); + + // When the group was created, it bumped the rcgen in the project. If we + // have an old view of the project, we expect a "concurrent + // modification" error. + let err = datastore + .project_delete(&opctx, &authz_project, &project) + .await + .unwrap_err(); + assert!(err.to_string().contains("concurrent modification"), "{err:?}"); + + // If we update rcgen, however, and the group has been deleted, + // we can successfully delete the project. + project.rcgen = project.rcgen.next().into(); + datastore + .project_delete(&opctx, &authz_project, &project) + .await + .unwrap(); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_group_names_are_unique_in_project() { + // Setup + let logctx = + dev::test_setup_log("affinity_group_names_are_unique_in_project"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create two projects + let (authz_project1, _) = + create_project(&opctx, &datastore, "my-project-1").await; + let (authz_project2, _) = + create_project(&opctx, &datastore, "my-project-2").await; + + // We can create a group wiht the same name in different projects + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project1, + "my-group", + ) + .await + .unwrap(); + create_affinity_group(&opctx, &datastore, &authz_project2, "my-group") + .await + .unwrap(); + + // If we try to create a new group with the same name in the same + // project, we'll see an error. + let err = create_affinity_group( + &opctx, + &datastore, + &authz_project1, + "my-group", + ) + .await + .unwrap_err(); + assert!( + matches!(&err, Error::ObjectAlreadyExists { + type_name, + object_name, + } if *type_name == ResourceType::AffinityGroup && + *object_name == "my-group"), + "Unexpected error: {err:?}" + ); + + // If we delete the group from the project, we can re-use the name. + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + datastore.affinity_group_delete(&opctx, &authz_group).await.unwrap(); + + create_affinity_group(&opctx, &datastore, &authz_project1, "my-group") + .await + .expect("Should have been able to re-use name after deletion"); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_names_are_unique_in_project() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_group_names_are_unique_in_project", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create two projects + let (authz_project1, _) = + create_project(&opctx, &datastore, "my-project-1").await; + let (authz_project2, _) = + create_project(&opctx, &datastore, "my-project-2").await; + + // We can create a group wiht the same name in different projects + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project1, + "my-group", + ) + .await + .unwrap(); + create_anti_affinity_group( + &opctx, + &datastore, + &authz_project2, + "my-group", + ) + .await + .unwrap(); + + // If we try to create a new group with the same name in the same + // project, we'll see an error. + let err = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project1, + "my-group", + ) + .await + .unwrap_err(); + assert!( + matches!(&err, Error::ObjectAlreadyExists { + type_name, + object_name, + } if *type_name == ResourceType::AntiAffinityGroup && + *object_name == "my-group"), + "Unexpected error: {err:?}" + ); + + // If we delete the group from the project, we can re-use the name. + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + datastore + .anti_affinity_group_delete(&opctx, &authz_group) + .await + .unwrap(); + + create_anti_affinity_group( + &opctx, + &datastore, + &authz_project1, + "my-group", + ) + .await + .expect("Should have been able to re-use name after deletion"); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_group_membership_add_list_remove() { + // Setup + let logctx = + dev::test_setup_log("affinity_group_membership_add_list_remove"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance without a VMM. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + + // Add the instance as a member to the group + datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // We should now be able to list the new member + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert_eq!(members.len(), 1); + assert_eq!( + external::AffinityGroupMember::Instance(instance.id()), + members[0].clone().into() + ); + + // We can delete the member and observe an empty member list + datastore + .affinity_group_member_delete( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_membership_add_list_remove() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_group_membership_add_list_remove", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance without a VMM. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + + // Add the instance as a member to the group + datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // We should now be able to list the new member + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert_eq!(members.len(), 1); + assert_eq!( + external::AntiAffinityGroupMember::Instance(instance.id()), + members[0].clone().into() + ); + + // We can delete the member and observe an empty member list + datastore + .anti_affinity_group_member_delete( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_group_membership_add_remove_instance_with_vmm() { + // Setup + let logctx = dev::test_setup_log( + "affinity_group_membership_add_remove_instance_with_vmm", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance with a VMM. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + set_instance_state_running(&datastore, instance.id()).await; + + // Cannot add the instance to the group while it's running. + let err = datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .expect_err( + "Shouldn't be able to add running instances to affinity groups", + ); + assert!(matches!(err, Error::InvalidRequest { .. })); + assert!( + err.to_string().contains( + "Instance cannot be added to affinity group in state" + ), + "{err:?}" + ); + + // If we stop the instance, we can add it to the group. + set_instance_state_stopped(&datastore, instance.id()).await; + datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Now we can set the instance state to "running" once more. + set_instance_state_running(&datastore, instance.id()).await; + + // We should now be able to list the new member + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert_eq!(members.len(), 1); + assert_eq!( + external::AffinityGroupMember::Instance(instance.id()), + members[0].clone().into() + ); + + // We can delete the member and observe an empty member list -- even + // though it's running! + datastore + .affinity_group_member_delete( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_membership_add_remove_instance_with_vmm() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_group_membership_add_remove_instance_with_vmm", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance with a VMM. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + set_instance_state_running(&datastore, instance.id()).await; + + // Cannot add the instance to the group while it's running. + let err = datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .expect_err( + "Shouldn't be able to add running instances to anti-affinity groups", + ); + assert!(matches!(err, Error::InvalidRequest { .. })); + assert!( + err.to_string().contains( + "Instance cannot be added to anti-affinity group in state" + ), + "{err:?}" + ); + + // If we stop the instance, we can add it to the group. + set_instance_state_stopped(&datastore, instance.id()).await; + datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Now we can set the instance state to "running" once more. + set_instance_state_running(&datastore, instance.id()).await; + + // We should now be able to list the new member + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert_eq!(members.len(), 1); + assert_eq!( + external::AntiAffinityGroupMember::Instance(instance.id()), + members[0].clone().into() + ); + + // We can delete the member and observe an empty member list -- even + // though it's running! + datastore + .anti_affinity_group_member_delete( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_group_delete_group_deletes_members() { + // Setup + let logctx = + dev::test_setup_log("affinity_group_delete_group_deletes_members"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance without a VMM, add it to the group. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Delete the group + datastore.affinity_group_delete(&opctx, &authz_group).await.unwrap(); + + // Confirm that no instance members exist + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_delete_group_deletes_members() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_group_delete_group_deletes_members", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance without a VMM, add it to the group. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Delete the group + datastore + .anti_affinity_group_delete(&opctx, &authz_group) + .await + .unwrap(); + + // Confirm that no instance members exist + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_group_delete_instance_deletes_membership() { + // Setup + let logctx = dev::test_setup_log( + "affinity_group_delete_instance_deletes_membership", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance without a VMM, add it to the group. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Delete the instance + let (.., authz_instance) = LookupPath::new(opctx, datastore) + .instance_id(instance.id()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + datastore + .project_delete_instance(&opctx, &authz_instance) + .await + .unwrap(); + + // Confirm that no instance members exist + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_delete_instance_deletes_membership() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_group_delete_instance_deletes_membership", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // A new group should have no members + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Create an instance without a VMM, add it to the group. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Delete the instance + let (.., authz_instance) = LookupPath::new(opctx, datastore) + .instance_id(instance.id()) + .lookup_for(authz::Action::Delete) + .await + .unwrap(); + datastore + .project_delete_instance(&opctx, &authz_instance) + .await + .unwrap(); + + // Confirm that no instance members exist + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_group_membership_for_deleted_objects() { + // Setup + let logctx = dev::test_setup_log( + "affinity_group_membership_for_deleted_objects", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + + struct TestArgs { + // Does the group exist? + group: bool, + // Does the instance exist? + instance: bool, + } + + let args = [ + TestArgs { group: false, instance: false }, + TestArgs { group: true, instance: false }, + TestArgs { group: false, instance: true }, + ]; + + for arg in args { + // Create an affinity group, and maybe delete it. + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + if !arg.group { + datastore + .affinity_group_delete(&opctx, &authz_group) + .await + .unwrap(); + } + + // Create an instance, and maybe delete it. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + let (.., authz_instance) = LookupPath::new(opctx, datastore) + .instance_id(instance.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + if !arg.instance { + datastore + .project_delete_instance(&opctx, &authz_instance) + .await + .unwrap(); + } + + // Try to add the instance to the group. + // + // Expect to see specific errors, depending on whether or not the + // group/instance exist. + let err = datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .expect_err("Should have failed"); + + match (arg.group, arg.instance) { + (false, _) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::AffinityGroup), + "{err:?}" + ); + } + (true, false) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::Instance), + "{err:?}" + ); + } + (true, true) => { + panic!("If both exist, we won't throw an error") + } + } + + // Do the same thing, but for group membership removal. + let err = datastore + .affinity_group_member_delete( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .expect_err("Should have failed"); + match (arg.group, arg.instance) { + (false, _) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::AffinityGroup), + "{err:?}" + ); + } + (true, false) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::Instance), + "{err:?}" + ); + } + (true, true) => { + panic!("If both exist, we won't throw an error") + } + } + + // Cleanup, if we actually created anything. + if arg.instance { + datastore + .project_delete_instance(&opctx, &authz_instance) + .await + .unwrap(); + } + if arg.group { + datastore + .affinity_group_delete(&opctx, &authz_group) + .await + .unwrap(); + } + } + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_membership_for_deleted_objects() { + // Setup + let logctx = dev::test_setup_log( + "anti_affinity_group_membership_for_deleted_objects", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + + struct TestArgs { + // Does the group exist? + group: bool, + // Does the instance exist? + instance: bool, + } + + let args = [ + TestArgs { group: false, instance: false }, + TestArgs { group: true, instance: false }, + TestArgs { group: false, instance: true }, + ]; + + for arg in args { + // Create an anti-affinity group, and maybe delete it. + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + if !arg.group { + datastore + .anti_affinity_group_delete(&opctx, &authz_group) + .await + .unwrap(); + } + + // Create an instance, and maybe delete it. + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + let (.., authz_instance) = LookupPath::new(opctx, datastore) + .instance_id(instance.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + if !arg.instance { + datastore + .project_delete_instance(&opctx, &authz_instance) + .await + .unwrap(); + } + + // Try to add the instance to the group. + // + // Expect to see specific errors, depending on whether or not the + // group/instance exist. + let err = datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .expect_err("Should have failed"); + + match (arg.group, arg.instance) { + (false, _) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::AntiAffinityGroup), + "{err:?}" + ); + } + (true, false) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::Instance), + "{err:?}" + ); + } + (true, true) => { + panic!("If both exist, we won't throw an error") + } + } + + // Do the same thing, but for group membership removal. + let err = datastore + .anti_affinity_group_member_delete( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .expect_err("Should have failed"); + match (arg.group, arg.instance) { + (false, _) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::AntiAffinityGroup), + "{err:?}" + ); + } + (true, false) => { + assert!( + matches!(err, Error::ObjectNotFound { + type_name, .. + } if type_name == ResourceType::Instance), + "{err:?}" + ); + } + (true, true) => { + panic!("If both exist, we won't throw an error") + } + } + + // Cleanup, if we actually created anything. + if arg.instance { + datastore + .project_delete_instance(&opctx, &authz_instance) + .await + .unwrap(); + } + if arg.group { + datastore + .anti_affinity_group_delete(&opctx, &authz_group) + .await + .unwrap(); + } + } + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn affinity_group_membership_idempotency() { + // Setup + let logctx = + dev::test_setup_log("affinity_group_membership_idempotency"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + let (.., authz_group) = LookupPath::new(opctx, datastore) + .affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // Create an instance + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + + // Add the instance to the group + datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Add the instance to the group again + let err = datastore + .affinity_group_member_add( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap_err(); + assert!( + matches!( + err, + Error::ObjectAlreadyExists { + type_name: ResourceType::AffinityGroupMember, + .. + } + ), + "Error: {err:?}" + ); + + // We should still only observe a single member in the group. + // + // Two calls to "affinity_group_member_add" should be the same + // as a single call. + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert_eq!(members.len(), 1); + + // We should be able to delete the membership idempotently. + datastore + .affinity_group_member_delete( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + let err = datastore + .affinity_group_member_delete( + &opctx, + &authz_group, + external::AffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap_err(); + assert!( + matches!( + err, + Error::ObjectNotFound { + type_name: ResourceType::AffinityGroupMember, + .. + } + ), + "Error: {err:?}" + ); + + let members = datastore + .affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_membership_idempotency() { + // Setup + let logctx = + dev::test_setup_log("anti_affinity_group_membership_idempotency"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // Create a project and a group + let (authz_project, ..) = + create_project(&opctx, &datastore, "my-project").await; + let group = create_anti_affinity_group( + &opctx, + &datastore, + &authz_project, + "my-group", + ) + .await + .unwrap(); + let (.., authz_group) = LookupPath::new(opctx, datastore) + .anti_affinity_group_id(group.id()) + .lookup_for(authz::Action::Modify) + .await + .unwrap(); + + // Create an instance + let instance = create_instance_record( + &opctx, + &datastore, + &authz_project, + "my-instance", + ) + .await; + + // Add the instance to the group + datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + + // Add the instance to the group again + let err = datastore + .anti_affinity_group_member_add( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap_err(); + assert!( + matches!( + err, + Error::ObjectAlreadyExists { + type_name: ResourceType::AntiAffinityGroupMember, + .. + } + ), + "Error: {err:?}" + ); + + // We should still only observe a single member in the group. + // + // Two calls to "anti_affinity_group_member_add" should be the same + // as a single call. + let pagparams_id = DataPageParams { + marker: None, + limit: NonZeroU32::new(100).unwrap(), + direction: dropshot::PaginationOrder::Ascending, + }; + let pagbyid = PaginatedBy::Id(pagparams_id); + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert_eq!(members.len(), 1); + + // We should be able to delete the membership idempotently. + datastore + .anti_affinity_group_member_delete( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap(); + let err = datastore + .anti_affinity_group_member_delete( + &opctx, + &authz_group, + external::AntiAffinityGroupMember::Instance(instance.id()), + ) + .await + .unwrap_err(); + assert!( + matches!( + err, + Error::ObjectNotFound { + type_name: ResourceType::AntiAffinityGroupMember, + .. + } + ), + "Error: {err:?}" + ); + + let members = datastore + .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .await + .unwrap(); + assert!(members.is_empty()); + + // Clean up. + db.terminate().await; + logctx.cleanup_successful(); + } +} diff --git a/nexus/db-queries/src/db/datastore/instance.rs b/nexus/db-queries/src/db/datastore/instance.rs index 65d42fbe7c2..9e4bec81d05 100644 --- a/nexus/db-queries/src/db/datastore/instance.rs +++ b/nexus/db-queries/src/db/datastore/instance.rs @@ -1387,6 +1387,13 @@ impl DataStore { })?; let instance_id = InstanceUuid::from_untyped_uuid(authz_instance.id()); + self.instance_affinity_group_memberships_delete(opctx, instance_id) + .await?; + self.instance_anti_affinity_group_memberships_delete( + opctx, + instance_id, + ) + .await?; self.instance_ssh_keys_delete(opctx, instance_id).await?; self.instance_mark_migrations_deleted(opctx, instance_id).await?; diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index b2d9f8f2471..5c241578a50 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -49,6 +49,7 @@ use std::num::NonZeroU32; use std::sync::Arc; mod address_lot; +mod affinity; mod allow_list; mod auth; mod bfd; diff --git a/nexus/db-queries/src/db/datastore/project.rs b/nexus/db-queries/src/db/datastore/project.rs index 58b7b315c1c..367e094b2e5 100644 --- a/nexus/db-queries/src/db/datastore/project.rs +++ b/nexus/db-queries/src/db/datastore/project.rs @@ -225,6 +225,8 @@ impl DataStore { generate_fn_to_ensure_none_in_project!(project_image, name, String); generate_fn_to_ensure_none_in_project!(snapshot, name, String); generate_fn_to_ensure_none_in_project!(vpc, name, String); + generate_fn_to_ensure_none_in_project!(affinity_group, name, String); + generate_fn_to_ensure_none_in_project!(anti_affinity_group, name, String); /// Delete a project pub async fn project_delete( @@ -242,6 +244,9 @@ impl DataStore { self.ensure_no_project_images_in_project(opctx, authz_project).await?; self.ensure_no_snapshots_in_project(opctx, authz_project).await?; self.ensure_no_vpcs_in_project(opctx, authz_project).await?; + self.ensure_no_affinity_groups_in_project(opctx, authz_project).await?; + self.ensure_no_anti_affinity_groups_in_project(opctx, authz_project) + .await?; use db::schema::project::dsl; diff --git a/nexus/db-queries/src/db/lookup.rs b/nexus/db-queries/src/db/lookup.rs index c629dbfc425..d4591024e7a 100644 --- a/nexus/db-queries/src/db/lookup.rs +++ b/nexus/db-queries/src/db/lookup.rs @@ -175,6 +175,16 @@ impl<'a> LookupPath<'a> { Instance::PrimaryKey(Root { lookup_root: self }, id) } + /// Select a resource of type AffinityGroup, identified by its id + pub fn affinity_group_id(self, id: Uuid) -> AffinityGroup<'a> { + AffinityGroup::PrimaryKey(Root { lookup_root: self }, id) + } + + /// Select a resource of type AntiAffinityGroup, identified by its id + pub fn anti_affinity_group_id(self, id: Uuid) -> AntiAffinityGroup<'a> { + AntiAffinityGroup::PrimaryKey(Root { lookup_root: self }, id) + } + /// Select a resource of type IpPool, identified by its name pub fn ip_pool_name<'b, 'c>(self, name: &'b Name) -> IpPool<'c> where @@ -645,7 +655,7 @@ lookup_resource! { lookup_resource! { name = "Project", ancestors = [ "Silo" ], - children = [ "Disk", "Instance", "Vpc", "Snapshot", "ProjectImage", "FloatingIp" ], + children = [ "AffinityGroup", "AntiAffinityGroup", "Disk", "Instance", "Vpc", "Snapshot", "ProjectImage", "FloatingIp" ], lookup_by_name = true, soft_deletes = true, primary_key_columns = [ { column_name = "id", rust_type = Uuid } ] @@ -696,6 +706,24 @@ lookup_resource! { primary_key_columns = [ { column_name = "id", rust_type = Uuid } ] } +lookup_resource! { + name = "AffinityGroup", + ancestors = [ "Silo", "Project" ], + children = [], + lookup_by_name = true, + soft_deletes = true, + primary_key_columns = [ { column_name = "id", rust_type = Uuid } ] +} + +lookup_resource! { + name = "AntiAffinityGroup", + ancestors = [ "Silo", "Project" ], + children = [], + lookup_by_name = true, + soft_deletes = true, + primary_key_columns = [ { column_name = "id", rust_type = Uuid } ] +} + lookup_resource! { name = "InstanceNetworkInterface", ancestors = [ "Silo", "Project", "Instance" ], From 161f9d60759521080dcb84736e9077c631add27a Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jan 2025 13:04:02 -0800 Subject: [PATCH 04/14] fix policy tests --- .../src/policy_test/resource_builder.rs | 2 + nexus/db-queries/src/policy_test/resources.rs | 17 ++++ nexus/db-queries/tests/output/authz-roles.out | 84 +++++++++++++++++++ 3 files changed, 103 insertions(+) diff --git a/nexus/db-queries/src/policy_test/resource_builder.rs b/nexus/db-queries/src/policy_test/resource_builder.rs index b6d7d97553e..310c11adf3c 100644 --- a/nexus/db-queries/src/policy_test/resource_builder.rs +++ b/nexus/db-queries/src/policy_test/resource_builder.rs @@ -243,6 +243,8 @@ macro_rules! impl_dyn_authorized_resource_for_resource { } impl_dyn_authorized_resource_for_resource!(authz::AddressLot); +impl_dyn_authorized_resource_for_resource!(authz::AffinityGroup); +impl_dyn_authorized_resource_for_resource!(authz::AntiAffinityGroup); impl_dyn_authorized_resource_for_resource!(authz::Blueprint); impl_dyn_authorized_resource_for_resource!(authz::Certificate); impl_dyn_authorized_resource_for_resource!(authz::DeviceAccessToken); diff --git a/nexus/db-queries/src/policy_test/resources.rs b/nexus/db-queries/src/policy_test/resources.rs index 6ee92e167cf..04655410533 100644 --- a/nexus/db-queries/src/policy_test/resources.rs +++ b/nexus/db-queries/src/policy_test/resources.rs @@ -300,6 +300,21 @@ async fn make_project( LookupType::ByName(vpc1_name.clone()), ); + let affinity_group_name = format!("{}-affinity-group1", project_name); + let affinity_group = authz::AffinityGroup::new( + project.clone(), + Uuid::new_v4(), + LookupType::ByName(affinity_group_name.clone()), + ); + + let anti_affinity_group_name = + format!("{}-anti-affinity-group1", project_name); + let anti_affinity_group = authz::AntiAffinityGroup::new( + project.clone(), + Uuid::new_v4(), + LookupType::ByName(anti_affinity_group_name.clone()), + ); + let instance_name = format!("{}-instance1", project_name); let instance = authz::Instance::new( project.clone(), @@ -313,6 +328,8 @@ async fn make_project( Uuid::new_v4(), LookupType::ByName(disk_name.clone()), )); + builder.new_resource(affinity_group.clone()); + builder.new_resource(anti_affinity_group.clone()); builder.new_resource(instance.clone()); builder.new_resource(authz::InstanceNetworkInterface::new( instance, diff --git a/nexus/db-queries/tests/output/authz-roles.out b/nexus/db-queries/tests/output/authz-roles.out index 4b24e649ccb..e0d43250d13 100644 --- a/nexus/db-queries/tests/output/authz-roles.out +++ b/nexus/db-queries/tests/output/authz-roles.out @@ -306,6 +306,34 @@ resource: Disk "silo1-proj1-disk1" silo1-proj1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ unauthenticated ! ! ! ! ! ! ! ! +resource: AffinityGroup "silo1-proj1-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + +resource: AntiAffinityGroup "silo1-proj1-anti-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + resource: Instance "silo1-proj1-instance1" USER Q R LC RP M MP CC D @@ -474,6 +502,34 @@ resource: Disk "silo1-proj2-disk1" silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ unauthenticated ! ! ! ! ! ! ! ! +resource: AffinityGroup "silo1-proj2-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + +resource: AntiAffinityGroup "silo1-proj2-anti-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + resource: Instance "silo1-proj2-instance1" USER Q R LC RP M MP CC D @@ -810,6 +866,34 @@ resource: Disk "silo2-proj1-disk1" silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ unauthenticated ! ! ! ! ! ! ! ! +resource: AffinityGroup "silo2-proj1-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + +resource: AntiAffinityGroup "silo2-proj1-anti-affinity-group1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + resource: Instance "silo2-proj1-instance1" USER Q R LC RP M MP CC D From 8dc0825fb4834c9b25030b4083efc59e13820c24 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jan 2025 13:19:07 -0800 Subject: [PATCH 05/14] fmt --- nexus/src/external_api/http_entrypoints.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 906684e7825..887a6d35a65 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -13,9 +13,9 @@ use super::{ Utilization, Vpc, VpcRouter, VpcSubnet, }, }; -use crate::app::Unimpl; use crate::app::external_endpoints::authority_for_request; use crate::app::support_bundles::SupportBundleQueryType; +use crate::app::Unimpl; use crate::context::ApiContext; use crate::external_api::shared; use dropshot::Body; From 4e9cebcf72fb8d85c2319ca61930721b9eca5bdd Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jan 2025 14:58:56 -0800 Subject: [PATCH 06/14] tags --- nexus/external-api/src/lib.rs | 2 +- openapi/nexus.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index 7daa5514619..ad637c9ca41 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -75,7 +75,7 @@ const PUT_UPDATE_REPOSITORY_MAX_BYTES: usize = 4 * GIB; policy = EndpointTagPolicy::ExactlyOne, tags = { "affinity" = { - description = "Affinity groups give control over instance placement.", + description = "Affinity and anti-affinity groups give control over instance placement.", external_docs = { url = "http://docs.oxide.computer/api/affinity" } diff --git a/openapi/nexus.json b/openapi/nexus.json index 2e4ef3ac597..aac7aab1051 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -24558,7 +24558,7 @@ "tags": [ { "name": "affinity", - "description": "Affinity groups give control over instance placement.", + "description": "Affinity and anti-affinity groups give control over instance placement.", "externalDocs": { "url": "http://docs.oxide.computer/api/affinity" } From 6cfca2dc198d80cc08daaaab0785729de2f73ee9 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Thu, 30 Jan 2025 16:06:14 -0800 Subject: [PATCH 07/14] doc comments --- common/src/api/external/mod.rs | 13 +++++++++++++ openapi/nexus.json | 5 +++++ 2 files changed, 18 insertions(+) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 98832487d5a..210953d49eb 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1318,6 +1318,9 @@ pub enum InstanceAutoRestartPolicy { // AFFINITY GROUPS +/// Affinity policy used to describe "what to do when a request cannot be satisfied" +/// +/// Used for both Affinity and Anti-Affinity Groups #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] #[serde(rename_all = "snake_case")] pub enum AffinityPolicy { @@ -1338,9 +1341,14 @@ pub enum FailureDomain { Sled, } +/// A member of an Affinity Group +/// +/// Membership in a group is not exclusive - members may belong to multiple +/// affinity / anti-affinity groups. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] #[serde(tag = "type", content = "value", rename_all = "snake_case")] pub enum AffinityGroupMember { + /// An instance belonging to this group, identified by UUID. Instance(Uuid), } @@ -1352,9 +1360,14 @@ impl SimpleIdentity for AffinityGroupMember { } } +/// A member of an Anti-Affinity Group +/// +/// Membership in a group is not exclusive - members may belong to multiple +/// affinity / anti-affinity groups. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] #[serde(tag = "type", content = "value", rename_all = "snake_case")] pub enum AntiAffinityGroupMember { + /// An instance belonging to this group, identified by UUID. Instance(Uuid), } diff --git a/openapi/nexus.json b/openapi/nexus.json index aac7aab1051..bc8abb272c2 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -12205,8 +12205,10 @@ ] }, "AffinityGroupMember": { + "description": "A member of an Affinity Group\n\nMembership in a group is not exclusive - members may belong to multiple affinity / anti-affinity groups.", "oneOf": [ { + "description": "An instance belonging to this group, identified by UUID.", "type": "object", "properties": { "type": { @@ -12288,6 +12290,7 @@ } }, "AffinityPolicy": { + "description": "Affinity policy used to describe \"what to do when a request cannot be satisfied\"\n\nUsed for both Affinity and Anti-Affinity Groups", "oneOf": [ { "description": "If the affinity request cannot be satisfied, allow it anyway.\n\nThis enables a \"best-effort\" attempt to satisfy the affinity policy.", @@ -12482,8 +12485,10 @@ ] }, "AntiAffinityGroupMember": { + "description": "A member of an Anti-Affinity Group\n\nMembership in a group is not exclusive - members may belong to multiple affinity / anti-affinity groups.", "oneOf": [ { + "description": "An instance belonging to this group, identified by UUID.", "type": "object", "properties": { "type": { From 4b08032dc3c887cf4512518705d80f22b50b0a48 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 31 Jan 2025 14:25:53 -0800 Subject: [PATCH 08/14] typed UUID --- common/src/api/external/mod.rs | 10 ++++++---- openapi/nexus.json | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 210953d49eb..f3234d74b99 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -23,6 +23,8 @@ use dropshot::HttpError; pub use dropshot::PaginationOrder; pub use error::*; use futures::stream::BoxStream; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceUuid; use oxnet::IpNet; use oxnet::Ipv4Net; use parse_display::Display; @@ -1349,13 +1351,13 @@ pub enum FailureDomain { #[serde(tag = "type", content = "value", rename_all = "snake_case")] pub enum AffinityGroupMember { /// An instance belonging to this group, identified by UUID. - Instance(Uuid), + Instance(InstanceUuid), } impl SimpleIdentity for AffinityGroupMember { fn id(&self) -> Uuid { match self { - AffinityGroupMember::Instance(id) => *id, + AffinityGroupMember::Instance(id) => *id.as_untyped_uuid(), } } } @@ -1368,13 +1370,13 @@ impl SimpleIdentity for AffinityGroupMember { #[serde(tag = "type", content = "value", rename_all = "snake_case")] pub enum AntiAffinityGroupMember { /// An instance belonging to this group, identified by UUID. - Instance(Uuid), + Instance(InstanceUuid), } impl SimpleIdentity for AntiAffinityGroupMember { fn id(&self) -> Uuid { match self { - AntiAffinityGroupMember::Instance(id) => *id, + AntiAffinityGroupMember::Instance(id) => *id.as_untyped_uuid(), } } } diff --git a/openapi/nexus.json b/openapi/nexus.json index bc8abb272c2..49b3c682465 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -12218,8 +12218,7 @@ ] }, "value": { - "type": "string", - "format": "uuid" + "$ref": "#/components/schemas/TypedUuidForInstanceKind" } }, "required": [ @@ -12498,8 +12497,7 @@ ] }, "value": { - "type": "string", - "format": "uuid" + "$ref": "#/components/schemas/TypedUuidForInstanceKind" } }, "required": [ @@ -23055,6 +23053,10 @@ } } }, + "TypedUuidForInstanceKind": { + "type": "string", + "format": "uuid" + }, "TypedUuidForSupportBundleKind": { "type": "string", "format": "uuid" From 900f09c6a3acef149867adedff51fa0ab3324fb4 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 31 Jan 2025 14:29:07 -0800 Subject: [PATCH 09/14] Typed UUID --- nexus/db-model/src/affinity.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nexus/db-model/src/affinity.rs b/nexus/db-model/src/affinity.rs index df97e7d1633..c2c4d30e224 100644 --- a/nexus/db-model/src/affinity.rs +++ b/nexus/db-model/src/affinity.rs @@ -23,7 +23,6 @@ use omicron_uuid_kinds::AffinityGroupKind; use omicron_uuid_kinds::AffinityGroupUuid; use omicron_uuid_kinds::AntiAffinityGroupKind; use omicron_uuid_kinds::AntiAffinityGroupUuid; -use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceKind; use omicron_uuid_kinds::InstanceUuid; use uuid::Uuid; @@ -231,7 +230,7 @@ impl AffinityGroupInstanceMembership { impl From for external::AffinityGroupMember { fn from(member: AffinityGroupInstanceMembership) -> Self { - Self::Instance(member.instance_id.into_untyped_uuid()) + Self::Instance(member.instance_id.into()) } } @@ -255,6 +254,6 @@ impl From for external::AntiAffinityGroupMember { fn from(member: AntiAffinityGroupInstanceMembership) -> Self { - Self::Instance(member.instance_id.into_untyped_uuid()) + Self::Instance(member.instance_id.into()) } } From f2ebe31df00ef40a655d070288c467c2c23398a9 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 31 Jan 2025 14:45:01 -0800 Subject: [PATCH 10/14] Typed UUID --- nexus/db-queries/src/db/datastore/affinity.rs | 154 ++++++++++++------ 1 file changed, 106 insertions(+), 48 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index e09a0e43406..78690c746b5 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -409,7 +409,7 @@ impl DataStore { use db::schema::affinity_group_instance_membership::dsl; dsl::affinity_group_instance_membership .filter(dsl::group_id.eq(authz_affinity_group.id())) - .filter(dsl::instance_id.eq(instance_id)) + .filter(dsl::instance_id.eq(instance_id.into_untyped_uuid())) .select(AffinityGroupInstanceMembership::as_select()) .get_result_async(&*conn) .await @@ -419,7 +419,7 @@ impl DataStore { e, ErrorHandler::NotFoundByLookup( ResourceType::AffinityGroupMember, - LookupType::by_id(instance_id), + LookupType::by_id(instance_id.into_untyped_uuid()), ), ) }) @@ -441,7 +441,7 @@ impl DataStore { use db::schema::anti_affinity_group_instance_membership::dsl; dsl::anti_affinity_group_instance_membership .filter(dsl::group_id.eq(authz_anti_affinity_group.id())) - .filter(dsl::instance_id.eq(instance_id)) + .filter(dsl::instance_id.eq(instance_id.into_untyped_uuid())) .select(AntiAffinityGroupInstanceMembership::as_select()) .get_result_async(&*conn) .await @@ -451,7 +451,7 @@ impl DataStore { e, ErrorHandler::NotFoundByLookup( ResourceType::AntiAffinityGroupMember, - LookupType::by_id(instance_id), + LookupType::by_id(instance_id.into_untyped_uuid()), ), ) }) @@ -508,7 +508,7 @@ impl DataStore { // issues, so we do the lookup manually. let instance_state = instance_dsl::instance .filter(instance_dsl::time_deleted.is_null()) - .filter(instance_dsl::id.eq(instance_id)) + .filter(instance_dsl::id.eq(instance_id.into_untyped_uuid())) .select(instance_dsl::state) .first_async(&conn) .await @@ -518,7 +518,7 @@ impl DataStore { e, ErrorHandler::NotFoundByLookup( ResourceType::Instance, - LookupType::ById(instance_id) + LookupType::ById(instance_id.into_untyped_uuid()) ), ) }) @@ -543,7 +543,7 @@ impl DataStore { diesel::insert_into(membership_dsl::affinity_group_instance_membership) .values(AffinityGroupInstanceMembership::new( AffinityGroupUuid::from_untyped_uuid(authz_affinity_group.id()), - InstanceUuid::from_untyped_uuid(instance_id), + instance_id, )) .execute_async(&conn) .await @@ -616,7 +616,7 @@ impl DataStore { // Check that the instance exists, and has no VMM let instance_state = instance_dsl::instance .filter(instance_dsl::time_deleted.is_null()) - .filter(instance_dsl::id.eq(instance_id)) + .filter(instance_dsl::id.eq(instance_id.into_untyped_uuid())) .select(instance_dsl::state) .first_async(&conn) .await @@ -626,7 +626,7 @@ impl DataStore { e, ErrorHandler::NotFoundByLookup( ResourceType::Instance, - LookupType::ById(instance_id) + LookupType::ById(instance_id.into_untyped_uuid()) ), ) }) @@ -651,7 +651,7 @@ impl DataStore { diesel::insert_into(membership_dsl::anti_affinity_group_instance_membership) .values(AntiAffinityGroupInstanceMembership::new( AntiAffinityGroupUuid::from_untyped_uuid(authz_anti_affinity_group.id()), - InstanceUuid::from_untyped_uuid(instance_id), + instance_id, )) .execute_async(&conn) .await @@ -752,7 +752,7 @@ impl DataStore { // Check that the instance exists instance_dsl::instance .filter(instance_dsl::time_deleted.is_null()) - .filter(instance_dsl::id.eq(instance_id)) + .filter(instance_dsl::id.eq(instance_id.into_untyped_uuid())) .select(instance_dsl::id) .first_async::(&conn) .await @@ -762,7 +762,7 @@ impl DataStore { e, ErrorHandler::NotFoundByLookup( ResourceType::Instance, - LookupType::ById(instance_id) + LookupType::ById(instance_id.into_untyped_uuid()) ), ) }) @@ -770,7 +770,7 @@ impl DataStore { let rows = diesel::delete(membership_dsl::affinity_group_instance_membership) .filter(membership_dsl::group_id.eq(authz_affinity_group.id())) - .filter(membership_dsl::instance_id.eq(instance_id)) + .filter(membership_dsl::instance_id.eq(instance_id.into_untyped_uuid())) .execute_async(&conn) .await .map_err(|e| { @@ -779,7 +779,7 @@ impl DataStore { }) })?; if rows == 0 { - return Err(err.bail(LookupType::ById(instance_id).into_not_found( + return Err(err.bail(LookupType::ById(instance_id.into_untyped_uuid()).into_not_found( ResourceType::AffinityGroupMember, ))); } @@ -841,7 +841,7 @@ impl DataStore { // Check that the instance exists instance_dsl::instance .filter(instance_dsl::time_deleted.is_null()) - .filter(instance_dsl::id.eq(instance_id)) + .filter(instance_dsl::id.eq(instance_id.into_untyped_uuid())) .select(instance_dsl::id) .first_async::(&conn) .await @@ -851,7 +851,7 @@ impl DataStore { e, ErrorHandler::NotFoundByLookup( ResourceType::Instance, - LookupType::ById(instance_id) + LookupType::ById(instance_id.into_untyped_uuid()) ), ) }) @@ -859,7 +859,7 @@ impl DataStore { let rows = diesel::delete(membership_dsl::anti_affinity_group_instance_membership) .filter(membership_dsl::group_id.eq(authz_anti_affinity_group.id())) - .filter(membership_dsl::instance_id.eq(instance_id)) + .filter(membership_dsl::instance_id.eq(instance_id.into_untyped_uuid())) .execute_async(&conn) .await .map_err(|e| { @@ -868,7 +868,7 @@ impl DataStore { }) })?; if rows == 0 { - return Err(err.bail(LookupType::ById(instance_id).into_not_found( + return Err(err.bail(LookupType::ById(instance_id.into_untyped_uuid()).into_not_found( ResourceType::AntiAffinityGroupMember, ))); } @@ -1487,7 +1487,9 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -1499,7 +1501,9 @@ mod tests { .unwrap(); assert_eq!(members.len(), 1); assert_eq!( - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()) + ), members[0].clone().into() ); @@ -1508,7 +1512,9 @@ mod tests { .affinity_group_member_delete( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -1577,7 +1583,9 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -1589,7 +1597,9 @@ mod tests { .unwrap(); assert_eq!(members.len(), 1); assert_eq!( - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()) + ), members[0].clone().into() ); @@ -1598,7 +1608,9 @@ mod tests { .anti_affinity_group_member_delete( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -1668,7 +1680,9 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .expect_err( @@ -1688,7 +1702,9 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -1703,7 +1719,9 @@ mod tests { .unwrap(); assert_eq!(members.len(), 1); assert_eq!( - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()) + ), members[0].clone().into() ); @@ -1713,7 +1731,9 @@ mod tests { .affinity_group_member_delete( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -1783,7 +1803,7 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance(InstanceUuid::from_untyped_uuid(instance.id())), ) .await .expect_err( @@ -1803,7 +1823,9 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -1818,7 +1840,9 @@ mod tests { .unwrap(); assert_eq!(members.len(), 1); assert_eq!( - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()) + ), members[0].clone().into() ); @@ -1828,7 +1852,9 @@ mod tests { .anti_affinity_group_member_delete( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -1894,7 +1920,9 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -1966,7 +1994,9 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -2041,7 +2071,9 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -2121,7 +2153,9 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -2227,7 +2261,9 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .expect_err("Should have failed"); @@ -2259,7 +2295,9 @@ mod tests { .affinity_group_member_delete( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .expect_err("Should have failed"); @@ -2383,7 +2421,9 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .expect_err("Should have failed"); @@ -2415,7 +2455,9 @@ mod tests { .anti_affinity_group_member_delete( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .expect_err("Should have failed"); @@ -2500,7 +2542,9 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -2510,7 +2554,9 @@ mod tests { .affinity_group_member_add( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap_err(); @@ -2546,7 +2592,9 @@ mod tests { .affinity_group_member_delete( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -2554,7 +2602,9 @@ mod tests { .affinity_group_member_delete( &opctx, &authz_group, - external::AffinityGroupMember::Instance(instance.id()), + external::AffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap_err(); @@ -2619,7 +2669,9 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -2629,7 +2681,9 @@ mod tests { .anti_affinity_group_member_add( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap_err(); @@ -2665,7 +2719,9 @@ mod tests { .anti_affinity_group_member_delete( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap(); @@ -2673,7 +2729,9 @@ mod tests { .anti_affinity_group_member_delete( &opctx, &authz_group, - external::AntiAffinityGroupMember::Instance(instance.id()), + external::AntiAffinityGroupMember::Instance( + InstanceUuid::from_untyped_uuid(instance.id()), + ), ) .await .unwrap_err(); From aba9596a0d1ce624ce59f8583ca7c687bc35f7f9 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 31 Jan 2025 14:59:07 -0800 Subject: [PATCH 11/14] comments --- nexus/db-model/src/affinity.rs | 2 +- schema/crdb/dbinit.sql | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/nexus/db-model/src/affinity.rs b/nexus/db-model/src/affinity.rs index c2c4d30e224..5309ac95275 100644 --- a/nexus/db-model/src/affinity.rs +++ b/nexus/db-model/src/affinity.rs @@ -2,7 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/5.0/. -// Copyright 2024 Oxide Computer Company +// Copyright 2025 Oxide Computer Company //! Database representation of affinity and anti-affinity groups diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index a2fa3d48e23..b8e110788d6 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -245,10 +245,15 @@ CREATE TABLE IF NOT EXISTS omicron.public.sled_resource ( kind omicron.public.sled_resource_kind NOT NULL, -- The UUID of an instance, if this resource belongs to an instance. + -- + -- This should eventually become NOT NULL for all instances, but is + -- still nullable for backwards compatibility purposes. Specifically, + -- the "instance start" saga can create rows in this table before creating + -- rows for "omicron.public.vmm", which we would use for back-filling. + -- If we tried to backfill + make this column non-nullable while that saga + -- was mid-execution, we would still have some rows in this table with nullable + -- values that would be more complex to fix. instance_id UUID - - -- TODO Add constraint that if kind is instance, instance_id is not NULL? - -- Or will that break backwards compatibility? ); -- Allow looking up all resources which reside on a sled From 1ad01010bcd8e59581a267463b3efb671a748343 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 31 Jan 2025 16:00:03 -0800 Subject: [PATCH 12/14] review feedback --- common/src/api/external/mod.rs | 6 + nexus/db-queries/src/db/datastore/affinity.rs | 231 ++++++++++-------- 2 files changed, 139 insertions(+), 98 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index f3234d74b99..f4b657a1a59 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -294,6 +294,12 @@ impl<'a> From<&'a Name> for &'a str { } } +impl From for String { + fn from(name: Name) -> Self { + name.0 + } +} + /// `Name` instances are comparable like Strings, primarily so that they can /// be used as keys in trees. impl PartialEq for Name diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 78690c746b5..5cd7c579105 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -20,7 +20,6 @@ use crate::db::model::AffinityGroupUpdate; use crate::db::model::AntiAffinityGroup; use crate::db::model::AntiAffinityGroupInstanceMembership; use crate::db::model::AntiAffinityGroupUpdate; -use crate::db::model::InstanceState; use crate::db::model::Name; use crate::db::model::Project; use crate::db::pagination::paginated; @@ -477,6 +476,7 @@ impl DataStore { use db::schema::affinity_group::dsl as group_dsl; use db::schema::affinity_group_instance_membership::dsl as membership_dsl; use db::schema::instance::dsl as instance_dsl; + use db::schema::sled_resource::dsl as resource_dsl; async move { // Check that the group exists @@ -497,7 +497,8 @@ impl DataStore { }) })?; - // Check that the instance exists, and has no VMM + // Check that the instance exists, and has no sled + // reservation. // // NOTE: I'd prefer to use the "LookupPath" infrastructure // to look up the path, but that API does not give the @@ -506,11 +507,12 @@ impl DataStore { // Looking up the instance on a different database // connection than the transaction risks several concurrency // issues, so we do the lookup manually. - let instance_state = instance_dsl::instance + + let _check_instance_exists = instance_dsl::instance .filter(instance_dsl::time_deleted.is_null()) .filter(instance_dsl::id.eq(instance_id.into_untyped_uuid())) - .select(instance_dsl::state) - .first_async(&conn) + .select(instance_dsl::id) + .get_result_async::(&conn) .await .map_err(|e| { err.bail_retryable_or_else(e, |e| { @@ -523,21 +525,33 @@ impl DataStore { ) }) })?; + let has_reservation: bool = diesel::select( + diesel::dsl::exists( + resource_dsl::sled_resource + .filter(resource_dsl::instance_id.eq(instance_id.into_untyped_uuid())) + ) + ).get_result_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::Server, + ) + }) + })?; // NOTE: It may be possible to add non-stopped instances to // affinity groups, depending on where they have already // been placed. However, only operating on "stopped" // instances is much easier to work with, as it does not // require any understanding of the group policy. - match instance_state { - InstanceState::NoVmm => (), - other => { - return Err(err.bail(Error::invalid_request( - format!( - "Instance cannot be added to affinity group in state: {other}" - ) - ))); - }, + if has_reservation { + return Err(err.bail(Error::invalid_request( + format!( + "Instance cannot be added to affinity group with reservation" + ) + ))); } diesel::insert_into(membership_dsl::affinity_group_instance_membership) @@ -593,6 +607,7 @@ impl DataStore { use db::schema::anti_affinity_group::dsl as group_dsl; use db::schema::anti_affinity_group_instance_membership::dsl as membership_dsl; use db::schema::instance::dsl as instance_dsl; + use db::schema::sled_resource::dsl as resource_dsl; async move { // Check that the group exists @@ -613,12 +628,13 @@ impl DataStore { }) })?; - // Check that the instance exists, and has no VMM - let instance_state = instance_dsl::instance + // Check that the instance exists, and has no sled + // reservation. + let _check_instance_exists = instance_dsl::instance .filter(instance_dsl::time_deleted.is_null()) .filter(instance_dsl::id.eq(instance_id.into_untyped_uuid())) - .select(instance_dsl::state) - .first_async(&conn) + .select(instance_dsl::id) + .get_result_async::(&conn) .await .map_err(|e| { err.bail_retryable_or_else(e, |e| { @@ -631,21 +647,33 @@ impl DataStore { ) }) })?; + let has_reservation: bool = diesel::select( + diesel::dsl::exists( + resource_dsl::sled_resource + .filter(resource_dsl::instance_id.eq(instance_id.into_untyped_uuid())) + ) + ).get_result_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::Server, + ) + }) + })?; // NOTE: It may be possible to add non-stopped instances to - // anti affinity groups, depending on where they have already + // affinity groups, depending on where they have already // been placed. However, only operating on "stopped" // instances is much easier to work with, as it does not // require any understanding of the group policy. - match instance_state { - InstanceState::NoVmm => (), - other => { - return Err(err.bail(Error::invalid_request( - format!( - "Instance cannot be added to anti-affinity group in state: {other}" - ) - ))); - }, + if has_reservation { + return Err(err.bail(Error::invalid_request( + format!( + "Instance cannot be added to anti-affinity group with reservation" + ) + ))); } diesel::insert_into(membership_dsl::anti_affinity_group_instance_membership) @@ -728,7 +756,6 @@ impl DataStore { let err = err.clone(); use db::schema::affinity_group::dsl as group_dsl; use db::schema::affinity_group_instance_membership::dsl as membership_dsl; - use db::schema::instance::dsl as instance_dsl; async move { // Check that the group exists @@ -749,25 +776,6 @@ impl DataStore { }) })?; - // Check that the instance exists - instance_dsl::instance - .filter(instance_dsl::time_deleted.is_null()) - .filter(instance_dsl::id.eq(instance_id.into_untyped_uuid())) - .select(instance_dsl::id) - .first_async::(&conn) - .await - .map_err(|e| { - err.bail_retryable_or_else(e, |e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByLookup( - ResourceType::Instance, - LookupType::ById(instance_id.into_untyped_uuid()) - ), - ) - }) - })?; - let rows = diesel::delete(membership_dsl::affinity_group_instance_membership) .filter(membership_dsl::group_id.eq(authz_affinity_group.id())) .filter(membership_dsl::instance_id.eq(instance_id.into_untyped_uuid())) @@ -817,7 +825,6 @@ impl DataStore { let err = err.clone(); use db::schema::anti_affinity_group::dsl as group_dsl; use db::schema::anti_affinity_group_instance_membership::dsl as membership_dsl; - use db::schema::instance::dsl as instance_dsl; async move { // Check that the group exists @@ -838,25 +845,6 @@ impl DataStore { }) })?; - // Check that the instance exists - instance_dsl::instance - .filter(instance_dsl::time_deleted.is_null()) - .filter(instance_dsl::id.eq(instance_id.into_untyped_uuid())) - .select(instance_dsl::id) - .first_async::(&conn) - .await - .map_err(|e| { - err.bail_retryable_or_else(e, |e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByLookup( - ResourceType::Instance, - LookupType::ById(instance_id.into_untyped_uuid()) - ), - ) - }) - })?; - let rows = diesel::delete(membership_dsl::anti_affinity_group_instance_membership) .filter(membership_dsl::group_id.eq(authz_anti_affinity_group.id())) .filter(membership_dsl::instance_id.eq(instance_id.into_untyped_uuid())) @@ -893,12 +881,16 @@ mod tests { use crate::db::lookup::LookupPath; use crate::db::pub_test_utils::TestDatabase; use nexus_db_model::Instance; + use nexus_db_model::Resources; + use nexus_db_model::SledResource; use nexus_types::external_api::params; use omicron_common::api::external::{ self, ByteCount, DataPageParams, IdentityMetadataCreateParams, }; use omicron_test_utils::dev; use omicron_uuid_kinds::InstanceUuid; + use omicron_uuid_kinds::PropolisUuid; + use omicron_uuid_kinds::SledUuid; use std::num::NonZeroU32; // Helper function for creating a project @@ -1006,14 +998,6 @@ mod tests { instance } - // Helper for explicitly modifying instance state. - // - // The interaction we typically use to create and modify instance state - // is more complex in production, since it's the result of a back-and-forth - // between Nexus and Sled Agent, using carefully crafted rcgen values. - // - // Here, we just set the value of state explicitly. Be warned, there - // are no guardrails! async fn set_instance_state_stopped( datastore: &DataStore, instance: uuid::Uuid, @@ -1032,16 +1016,28 @@ mod tests { .unwrap(); } - async fn set_instance_state_running( + // Helper for explicitly modifying sled resource usage + // + // The interaction we typically use to create and modify instance state + // is more complex in production, using a real allocation algorithm. + // + // Here, we just set the value of state explicitly. Be warned, there + // are no guardrails! + async fn allocate_instance_reservation( datastore: &DataStore, - instance: uuid::Uuid, + instance: InstanceUuid, ) { - use db::schema::instance::dsl; - diesel::update(dsl::instance) - .filter(dsl::id.eq(instance)) - .set(( - dsl::state.eq(db::model::InstanceState::Vmm), - dsl::active_propolis_id.eq(uuid::Uuid::new_v4()), + use db::schema::sled_resource::dsl; + diesel::insert_into(dsl::sled_resource) + .values(SledResource::new_for_vmm( + PropolisUuid::new_v4(), + instance, + SledUuid::new_v4(), + Resources::new( + 1, + ByteCount::from_kibibytes_u32(1).into(), + ByteCount::from_kibibytes_u32(1).into(), + ), )) .execute_async( &*datastore.pool_connection_for_tests().await.unwrap(), @@ -1050,6 +1046,20 @@ mod tests { .unwrap(); } + async fn delete_instance_reservation( + datastore: &DataStore, + instance: InstanceUuid, + ) { + use db::schema::sled_resource::dsl; + diesel::delete(dsl::sled_resource) + .filter(dsl::instance_id.eq(instance.into_untyped_uuid())) + .execute_async( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) + .await + .unwrap(); + } + #[tokio::test] async fn affinity_groups_are_project_scoped() { // Setup @@ -1673,7 +1683,12 @@ mod tests { "my-instance", ) .await; - set_instance_state_running(&datastore, instance.id()).await; + + allocate_instance_reservation( + &datastore, + InstanceUuid::from_untyped_uuid(instance.id()), + ) + .await; // Cannot add the instance to the group while it's running. let err = datastore @@ -1691,13 +1706,17 @@ mod tests { assert!(matches!(err, Error::InvalidRequest { .. })); assert!( err.to_string().contains( - "Instance cannot be added to affinity group in state" + "Instance cannot be added to affinity group with reservation" ), "{err:?}" ); - // If we stop the instance, we can add it to the group. - set_instance_state_stopped(&datastore, instance.id()).await; + // If we have no reservation for the instance, we can add it to the group. + delete_instance_reservation( + &datastore, + InstanceUuid::from_untyped_uuid(instance.id()), + ) + .await; datastore .affinity_group_member_add( &opctx, @@ -1709,8 +1728,12 @@ mod tests { .await .unwrap(); - // Now we can set the instance state to "running" once more. - set_instance_state_running(&datastore, instance.id()).await; + // Now we can reserve a sled for the instance once more. + allocate_instance_reservation( + &datastore, + InstanceUuid::from_untyped_uuid(instance.id()), + ) + .await; // We should now be able to list the new member let members = datastore @@ -1796,7 +1819,11 @@ mod tests { "my-instance", ) .await; - set_instance_state_running(&datastore, instance.id()).await; + allocate_instance_reservation( + &datastore, + InstanceUuid::from_untyped_uuid(instance.id()), + ) + .await; // Cannot add the instance to the group while it's running. let err = datastore @@ -1812,13 +1839,17 @@ mod tests { assert!(matches!(err, Error::InvalidRequest { .. })); assert!( err.to_string().contains( - "Instance cannot be added to anti-affinity group in state" + "Instance cannot be added to anti-affinity group with reservation" ), "{err:?}" ); - // If we stop the instance, we can add it to the group. - set_instance_state_stopped(&datastore, instance.id()).await; + // If we have no reservation for the instance, we can add it to the group. + delete_instance_reservation( + &datastore, + InstanceUuid::from_untyped_uuid(instance.id()), + ) + .await; datastore .anti_affinity_group_member_add( &opctx, @@ -1830,8 +1861,12 @@ mod tests { .await .unwrap(); - // Now we can set the instance state to "running" once more. - set_instance_state_running(&datastore, instance.id()).await; + // Now we can reserve a sled for the instance once more. + allocate_instance_reservation( + &datastore, + InstanceUuid::from_untyped_uuid(instance.id()), + ) + .await; // We should now be able to list the new member let members = datastore @@ -2314,7 +2349,7 @@ mod tests { assert!( matches!(err, Error::ObjectNotFound { type_name, .. - } if type_name == ResourceType::Instance), + } if type_name == ResourceType::AffinityGroupMember), "{err:?}" ); } @@ -2474,7 +2509,7 @@ mod tests { assert!( matches!(err, Error::ObjectNotFound { type_name, .. - } if type_name == ResourceType::Instance), + } if type_name == ResourceType::AntiAffinityGroupMember), "{err:?}" ); } From 4d26262ab67d305d4c3052d1f95eef52f2cb7252 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 31 Jan 2025 16:03:08 -0800 Subject: [PATCH 13/14] comment --- nexus/db-queries/src/db/datastore/affinity.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 5cd7c579105..101af1910d2 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -664,7 +664,7 @@ impl DataStore { })?; // NOTE: It may be possible to add non-stopped instances to - // affinity groups, depending on where they have already + // anti-affinity groups, depending on where they have already // been placed. However, only operating on "stopped" // instances is much easier to work with, as it does not // require any understanding of the group policy. From 6ae191011d49b1adec0aba75b7f37b088d23b456 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 31 Jan 2025 16:03:59 -0800 Subject: [PATCH 14/14] clippy --- nexus/db-queries/src/db/datastore/affinity.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 101af1910d2..e55afe41eac 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -548,9 +548,7 @@ impl DataStore { // require any understanding of the group policy. if has_reservation { return Err(err.bail(Error::invalid_request( - format!( - "Instance cannot be added to affinity group with reservation" - ) + "Instance cannot be added to affinity group with reservation".to_string() ))); } @@ -670,9 +668,7 @@ impl DataStore { // require any understanding of the group policy. if has_reservation { return Err(err.bail(Error::invalid_request( - format!( - "Instance cannot be added to anti-affinity group with reservation" - ) + "Instance cannot be added to anti-affinity group with reservation".to_string() ))); }