diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 1ee3c778e26..a5af4647b7e 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -22,6 +22,7 @@ use dropshot::HttpError; pub use dropshot::PaginationOrder; pub use error::*; use futures::stream::BoxStream; +use omicron_uuid_kinds::AffinityGroupUuid; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceUuid; use oxnet::IpNet; @@ -1339,6 +1340,9 @@ impl SimpleIdentity for AffinityGroupMember { #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] #[serde(tag = "type", content = "value", rename_all = "snake_case")] pub enum AntiAffinityGroupMember { + /// An affinity group belonging to this group, identified by UUID. + AffinityGroup(AffinityGroupUuid), + /// An instance belonging to this group, identified by UUID. Instance(InstanceUuid), } @@ -1346,6 +1350,7 @@ pub enum AntiAffinityGroupMember { impl SimpleIdentity for AntiAffinityGroupMember { fn id(&self) -> Uuid { match self { + AntiAffinityGroupMember::AffinityGroup(id) => *id.as_untyped_uuid(), AntiAffinityGroupMember::Instance(id) => *id.as_untyped_uuid(), } } diff --git a/nexus/db-model/src/affinity.rs b/nexus/db-model/src/affinity.rs index 5309ac95275..89214f85778 100644 --- a/nexus/db-model/src/affinity.rs +++ b/nexus/db-model/src/affinity.rs @@ -11,6 +11,7 @@ use super::Name; use crate::schema::affinity_group; use crate::schema::affinity_group_instance_membership; use crate::schema::anti_affinity_group; +use crate::schema::anti_affinity_group_affinity_membership; use crate::schema::anti_affinity_group_instance_membership; use crate::typed_uuid::DbTypedUuid; use chrono::{DateTime, Utc}; @@ -257,3 +258,30 @@ impl From Self::Instance(member.instance_id.into()) } } + +#[derive(Queryable, Insertable, Clone, Debug, Selectable)] +#[diesel(table_name = anti_affinity_group_affinity_membership)] +pub struct AntiAffinityGroupAffinityMembership { + pub anti_affinity_group_id: DbTypedUuid, + pub affinity_group_id: DbTypedUuid, +} + +impl AntiAffinityGroupAffinityMembership { + pub fn new( + anti_affinity_group_id: AntiAffinityGroupUuid, + affinity_group_id: AffinityGroupUuid, + ) -> Self { + Self { + anti_affinity_group_id: anti_affinity_group_id.into(), + affinity_group_id: affinity_group_id.into(), + } + } +} + +impl From + for external::AntiAffinityGroupMember +{ + fn from(member: AntiAffinityGroupAffinityMembership) -> Self { + Self::AffinityGroup(member.affinity_group_id.into()) + } +} diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 55d7b1f9d59..b3c5ebd540c 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -510,6 +510,13 @@ table! { } } +table! { + anti_affinity_group_affinity_membership (anti_affinity_group_id, affinity_group_id) { + anti_affinity_group_id -> Uuid, + affinity_group_id -> Uuid, + } +} + table! { metric_producer (id) { id -> Uuid, @@ -2074,6 +2081,7 @@ allow_tables_to_appear_in_same_query!(hw_baseboard_id, inv_sled_agent,); allow_tables_to_appear_in_same_query!( anti_affinity_group, + anti_affinity_group_affinity_membership, anti_affinity_group_instance_membership, affinity_group, affinity_group_instance_membership, diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index b9d6ed0c217..b99f92d54e7 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -16,7 +16,7 @@ use std::{collections::BTreeMap, sync::LazyLock}; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: Version = Version::new(128, 0, 0); +pub const SCHEMA_VERSION: Version = Version::new(129, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -28,6 +28,7 @@ static KNOWN_VERSIONS: LazyLock> = LazyLock::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(129, "anti-affinity-group-affinity-member"), KnownVersion::new(128, "sled-resource-for-vmm"), KnownVersion::new(127, "bp-disk-disposition-expunged-cleanup"), KnownVersion::new(126, "affinity"), diff --git a/nexus/db-queries/src/db/datastore/affinity.rs b/nexus/db-queries/src/db/datastore/affinity.rs index 183c9caad30..3797560aca9 100644 --- a/nexus/db-queries/src/db/datastore/affinity.rs +++ b/nexus/db-queries/src/db/datastore/affinity.rs @@ -18,11 +18,13 @@ use crate::db::model::AffinityGroup; use crate::db::model::AffinityGroupInstanceMembership; use crate::db::model::AffinityGroupUpdate; use crate::db::model::AntiAffinityGroup; +use crate::db::model::AntiAffinityGroupAffinityMembership; use crate::db::model::AntiAffinityGroupInstanceMembership; use crate::db::model::AntiAffinityGroupUpdate; use crate::db::model::Name; use crate::db::model::Project; use crate::db::pagination::paginated; +use crate::db::raw_query_builder::QueryBuilder; use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; @@ -30,6 +32,7 @@ use diesel::prelude::*; use omicron_common::api::external; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; +use omicron_common::api::external::DataPageParams; use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; @@ -41,6 +44,7 @@ use omicron_uuid_kinds::AntiAffinityGroupUuid; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceUuid; use ref_cast::RefCast; +use uuid::Uuid; impl DataStore { pub async fn affinity_group_list( @@ -368,28 +372,72 @@ impl DataStore { &self, opctx: &OpContext, authz_anti_affinity_group: &authz::AntiAffinityGroup, - pagparams: &PaginatedBy<'_>, - ) -> ListResultVec { + pagparams: &DataPageParams<'_, Uuid>, + ) -> ListResultVec { opctx.authorize(authz::Action::Read, authz_anti_affinity_group).await?; - use db::schema::anti_affinity_group_instance_membership::dsl; - match pagparams { - PaginatedBy::Id(pagparams) => paginated( - dsl::anti_affinity_group_instance_membership, - dsl::instance_id, - &pagparams, - ), - PaginatedBy::Name(_) => { - return Err(Error::invalid_request( - "Cannot paginate group members by name", - )); + let mut query = QueryBuilder::new() + .sql( + " + SELECT instance_id as id, 'instance' as label + FROM anti_affinity_group_instance_membership + WHERE group_id = ", + ) + .param() + .bind::(authz_anti_affinity_group.id()) + .sql( + " + UNION + SELECT affinity_group_id as id, 'affinity_group' as label + FROM anti_affinity_group_affinity_membership + WHERE anti_affinity_group_id = ", + ) + .param() + .bind::(authz_anti_affinity_group.id()) + .sql(" "); + + let (sort, cmp) = match pagparams.direction { + dropshot::PaginationOrder::Ascending => (" ORDER BY id ASC ", ">"), + dropshot::PaginationOrder::Descending => { + (" ORDER BY id DESC ", "<") } - } - .filter(dsl::group_id.eq(authz_anti_affinity_group.id())) - .select(AntiAffinityGroupInstanceMembership::as_select()) - .load_async(&*self.pool_connection_authorized(opctx).await?) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + }; + if let Some(id) = pagparams.marker { + query = query + .sql("WHERE id ") + .sql(cmp) + .sql(" ") + .param() + .bind::(*id); + }; + + query = query.sql(sort); + query = + query.sql(" LIMIT ").param().bind::( + i64::from(pagparams.limit.get()), + ); + + Ok(query + .query::<(diesel::sql_types::Uuid, diesel::sql_types::Text)>() + .load_async::<(Uuid, String)>( + &*self.pool_connection_authorized(opctx).await?, + ) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? + .into_iter() + .map(|(id, label)| { + use external::AntiAffinityGroupMember as Member; + match label.as_str() { + "affinity_group" => Member::AffinityGroup( + AffinityGroupUuid::from_untyped_uuid(id), + ), + "instance" => { + Member::Instance(InstanceUuid::from_untyped_uuid(id)) + } + other => panic!("Unexpected label from query: {other}"), + } + }) + .collect()) } pub async fn affinity_group_member_view( @@ -433,27 +481,60 @@ impl DataStore { opctx.authorize(authz::Action::Read, authz_anti_affinity_group).await?; let conn = self.pool_connection_authorized(opctx).await?; - let instance_id = match member { - external::AntiAffinityGroupMember::Instance(id) => id, - }; - - use db::schema::anti_affinity_group_instance_membership::dsl; - dsl::anti_affinity_group_instance_membership - .filter(dsl::group_id.eq(authz_anti_affinity_group.id())) - .filter(dsl::instance_id.eq(instance_id.into_untyped_uuid())) - .select(AntiAffinityGroupInstanceMembership::as_select()) - .get_result_async(&*conn) - .await - .map(|m| m.into()) - .map_err(|e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByLookup( - ResourceType::AntiAffinityGroupMember, - LookupType::by_id(instance_id.into_untyped_uuid()), - ), - ) - }) + match member { + external::AntiAffinityGroupMember::Instance(instance_id) => { + use db::schema::anti_affinity_group_instance_membership::dsl; + dsl::anti_affinity_group_instance_membership + .filter(dsl::group_id.eq(authz_anti_affinity_group.id())) + .filter( + dsl::instance_id.eq(instance_id.into_untyped_uuid()), + ) + .select(AntiAffinityGroupInstanceMembership::as_select()) + .get_result_async(&*conn) + .await + .map(|m| m.into()) + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::AntiAffinityGroupMember, + LookupType::by_id( + instance_id.into_untyped_uuid(), + ), + ), + ) + }) + } + external::AntiAffinityGroupMember::AffinityGroup( + affinity_group_id, + ) => { + use db::schema::anti_affinity_group_affinity_membership::dsl; + dsl::anti_affinity_group_affinity_membership + .filter( + dsl::anti_affinity_group_id + .eq(authz_anti_affinity_group.id()), + ) + .filter( + dsl::affinity_group_id + .eq(affinity_group_id.into_untyped_uuid()), + ) + .select(AntiAffinityGroupAffinityMembership::as_select()) + .get_result_async(&*conn) + .await + .map(|m| m.into()) + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::AntiAffinityGroupMember, + LookupType::by_id( + affinity_group_id.into_untyped_uuid(), + ), + ), + ) + }) + } + } } pub async fn affinity_group_member_add( @@ -593,13 +674,35 @@ impl DataStore { .authorize(authz::Action::Modify, authz_anti_affinity_group) .await?; - let instance_id = match member { - external::AntiAffinityGroupMember::Instance(id) => id, - }; + match member { + external::AntiAffinityGroupMember::Instance(id) => { + self.anti_affinity_group_member_add_instance( + opctx, + authz_anti_affinity_group, + id, + ) + .await + } + external::AntiAffinityGroupMember::AffinityGroup(id) => { + self.anti_affinity_group_member_add_group( + opctx, + authz_anti_affinity_group, + id, + ) + .await + } + } + } + async fn anti_affinity_group_member_add_instance( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + instance_id: InstanceUuid, + ) -> Result<(), Error> { let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; - self.transaction_retry_wrapper("anti_affinity_group_member_add") + self.transaction_retry_wrapper("anti_affinity_group_member_add_instance") .transaction(&conn, |conn| { let err = err.clone(); use db::schema::anti_affinity_group::dsl as group_dsl; @@ -703,6 +806,95 @@ impl DataStore { Ok(()) } + async fn anti_affinity_group_member_add_group( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + affinity_group_id: AffinityGroupUuid, + ) -> Result<(), Error> { + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("anti_affinity_group_member_add_group") + .transaction(&conn, |conn| { + let err = err.clone(); + use db::schema::anti_affinity_group::dsl as anti_affinity_group_dsl; + use db::schema::affinity_group::dsl as affinity_group_dsl; + use db::schema::anti_affinity_group_affinity_membership::dsl as membership_dsl; + + async move { + // Check that the anti-affinity group exists + anti_affinity_group_dsl::anti_affinity_group + .filter(anti_affinity_group_dsl::time_deleted.is_null()) + .filter(anti_affinity_group_dsl::id.eq(authz_anti_affinity_group.id())) + .select(anti_affinity_group_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_anti_affinity_group, + ), + ) + }) + })?; + + // Check that the affinity group exists + affinity_group_dsl::affinity_group + .filter(affinity_group_dsl::time_deleted.is_null()) + .filter(affinity_group_dsl::id.eq(affinity_group_id.into_untyped_uuid())) + .select(affinity_group_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::AffinityGroup, + LookupType::ById(affinity_group_id.into_untyped_uuid()) + ), + ) + }) + })?; + + // TODO: It's possible that the affinity group has members + // which are already running. We should probably check this, + // and prevent it, otherwise we could circumvent "policy = + // fail" stances. + + diesel::insert_into(membership_dsl::anti_affinity_group_affinity_membership) + .values(AntiAffinityGroupAffinityMembership::new( + AntiAffinityGroupUuid::from_untyped_uuid(authz_anti_affinity_group.id()), + affinity_group_id, + )) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::AntiAffinityGroupMember, + &affinity_group_id.to_string(), + ), + ) + }) + })?; + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } + pub async fn instance_affinity_group_memberships_delete( &self, opctx: &OpContext, @@ -810,13 +1002,38 @@ impl DataStore { .authorize(authz::Action::Modify, authz_anti_affinity_group) .await?; - let instance_id = match member { - external::AntiAffinityGroupMember::Instance(id) => id, - }; + match member { + external::AntiAffinityGroupMember::Instance(id) => { + self.anti_affinity_group_instance_member_delete( + opctx, + authz_anti_affinity_group, + id, + ) + .await + } + external::AntiAffinityGroupMember::AffinityGroup(id) => { + self.anti_affinity_group_affinity_member_delete( + opctx, + authz_anti_affinity_group, + id, + ) + .await + } + } + } + // Deletes an anti-affinity member, when that member is an instance + // + // See: [`Self::anti_affinity_group_member_delete`] + async fn anti_affinity_group_instance_member_delete( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + instance_id: InstanceUuid, + ) -> Result<(), Error> { let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; - self.transaction_retry_wrapper("anti_affinity_group_member_delete") + self.transaction_retry_wrapper("anti_affinity_group_instance_member_delete") .transaction(&conn, |conn| { let err = err.clone(); use db::schema::anti_affinity_group::dsl as group_dsl; @@ -868,6 +1085,70 @@ impl DataStore { })?; Ok(()) } + + // Deletes an anti-affinity member, when that member is an affinity group + // + // See: [`Self::anti_affinity_group_member_delete`] + async fn anti_affinity_group_affinity_member_delete( + &self, + opctx: &OpContext, + authz_anti_affinity_group: &authz::AntiAffinityGroup, + affinity_group_id: AffinityGroupUuid, + ) -> Result<(), Error> { + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("anti_affinity_group_affinity_member_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + use db::schema::anti_affinity_group::dsl as group_dsl; + use db::schema::anti_affinity_group_affinity_membership::dsl as membership_dsl; + + async move { + // Check that the anti-affinity group exists + group_dsl::anti_affinity_group + .filter(group_dsl::time_deleted.is_null()) + .filter(group_dsl::id.eq(authz_anti_affinity_group.id())) + .select(group_dsl::id) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_anti_affinity_group, + ), + ) + }) + })?; + + let rows = diesel::delete(membership_dsl::anti_affinity_group_affinity_membership) + .filter(membership_dsl::anti_affinity_group_id.eq(authz_anti_affinity_group.id())) + .filter(membership_dsl::affinity_group_id.eq(affinity_group_id.into_untyped_uuid())) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + })?; + if rows == 0 { + return Err(err.bail(LookupType::ById(affinity_group_id.into_untyped_uuid()).into_not_found( + ResourceType::AntiAffinityGroupMember, + ))); + } + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + public_error_from_diesel(e, ErrorHandler::Server) + })?; + Ok(()) + } } #[cfg(test)] @@ -1482,14 +1763,13 @@ mod tests { .unwrap(); // A new group should have no members - let pagparams_id = DataPageParams { + let pagparams = DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, }; - let pagbyid = PaginatedBy::Id(pagparams_id); let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert!(members.is_empty()); @@ -1515,7 +1795,7 @@ mod tests { // We should now be able to list the new member let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert_eq!(members.len(), 1); @@ -1534,7 +1814,7 @@ mod tests { .await .unwrap(); let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert!(members.is_empty()); @@ -1688,14 +1968,13 @@ mod tests { .unwrap(); // A new group should have no members - let pagparams_id = DataPageParams { + let pagparams = DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, }; - let pagbyid = PaginatedBy::Id(pagparams_id); let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert!(members.is_empty()); @@ -1745,7 +2024,7 @@ mod tests { // We should now be able to list the new member let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert_eq!(members.len(), 1); @@ -1765,7 +2044,7 @@ mod tests { .await .unwrap(); let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert!(members.is_empty()); @@ -1874,14 +2153,13 @@ mod tests { .unwrap(); // A new group should have no members - let pagparams_id = DataPageParams { + let pagparams = DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, }; - let pagbyid = PaginatedBy::Id(pagparams_id); let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert!(members.is_empty()); @@ -1911,7 +2189,7 @@ mod tests { // Confirm that no instance members exist let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert!(members.is_empty()); @@ -2029,14 +2307,13 @@ mod tests { .unwrap(); // A new group should have no members - let pagparams_id = DataPageParams { + let pagparams = DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, }; - let pagbyid = PaginatedBy::Id(pagparams_id); let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert!(members.is_empty()); @@ -2071,7 +2348,7 @@ mod tests { // Confirm that no instance members exist let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert!(members.is_empty()); @@ -2580,14 +2857,13 @@ mod tests { // // Two calls to "anti_affinity_group_member_add" should be the same // as a single call. - let pagparams_id = DataPageParams { + let pagparams = DataPageParams { marker: None, limit: NonZeroU32::new(100).unwrap(), direction: dropshot::PaginationOrder::Ascending, }; - let pagbyid = PaginatedBy::Id(pagparams_id); let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert_eq!(members.len(), 1); @@ -2621,7 +2897,7 @@ mod tests { ); let members = datastore - .anti_affinity_group_member_list(&opctx, &authz_group, &pagbyid) + .anti_affinity_group_member_list(&opctx, &authz_group, &pagparams) .await .unwrap(); assert!(members.is_empty()); diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index 5231e9275de..ebc83d15ad2 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -1362,6 +1362,31 @@ pub(in crate::db::datastore) mod test { .unwrap(); } + async fn add_affinity_group_to_anti_affinity_group( + datastore: &DataStore, + aa_group_id: AntiAffinityGroupUuid, + a_group_id: AffinityGroupUuid, + ) { + use db::model::AntiAffinityGroupAffinityMembership; + use db::schema::anti_affinity_group_affinity_membership::dsl as membership_dsl; + + diesel::insert_into( + membership_dsl::anti_affinity_group_affinity_membership, + ) + .values(AntiAffinityGroupAffinityMembership::new( + aa_group_id, + a_group_id, + )) + .on_conflict(( + membership_dsl::anti_affinity_group_id, + membership_dsl::affinity_group_id, + )) + .do_nothing() + .execute_async(&*datastore.pool_connection_for_tests().await.unwrap()) + .await + .unwrap(); + } + // This short-circuits some of the logic and checks we normally have when // creating affinity groups, but makes testing easier. async fn add_instance_to_affinity_group( @@ -1465,6 +1490,42 @@ pub(in crate::db::datastore) mod test { Self { id_by_name } } + + async fn add_affinity_groups_to_anti_affinity_group( + &self, + datastore: &DataStore, + aa_group: &'static str, + a_groups: &[&'static str], + ) { + let (affinity, aa_group_id) = + self.id_by_name.get(aa_group).unwrap_or_else(|| { + panic!( + "Anti-affinity group {aa_group} not part of AllGroups" + ) + }); + assert!( + matches!(affinity, Affinity::Negative), + "{aa_group} should be an anti-affinity group, but is not" + ); + + for a_group in a_groups { + let (affinity, a_group_id) = + self.id_by_name.get(a_group).unwrap_or_else(|| { + panic!("Affinity group {a_group} not part of AllGroups") + }); + assert!( + matches!(affinity, Affinity::Positive), + "{a_group} should be an affinity group, but is not" + ); + + add_affinity_group_to_anti_affinity_group( + datastore, + AntiAffinityGroupUuid::from_untyped_uuid(*aa_group_id), + AffinityGroupUuid::from_untyped_uuid(*a_group_id), + ) + .await; + } + } } struct Instance { @@ -2179,6 +2240,331 @@ pub(in crate::db::datastore) mod test { logctx.cleanup_successful(); } + #[tokio::test] + async fn anti_affinity_group_containing_affinity_groups() { + let logctx = dev::test_setup_log( + "anti_affinity_group_containing_affinity_groups", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 4; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Fail, + }, + Group { + affinity: Affinity::Positive, + name: "affinity1", + policy: external::AffinityPolicy::Allow, + }, + Group { + affinity: Affinity::Positive, + name: "affinity2", + policy: external::AffinityPolicy::Allow, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + all_groups + .add_affinity_groups_to_anti_affinity_group( + &datastore, + "anti-affinity", + &["affinity1", "affinity2"], + ) + .await; + + // The instances on sled 0 and 1 belong to "affinity1" + // The instances on sled 2 and 3 belong to "affinity2" + // + // A new instance in "affinity2" can only be placed on sled 2. + let instances = [ + Instance::new().group("affinity1").sled(sleds[0].id()), + Instance::new().group("affinity1").sled(sleds[1].id()), + Instance::new().group("affinity2").sled(sleds[2].id()), + Instance::new() + .group("affinity2") + .sled(sleds[3].id()) + .use_many_resources(), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = Instance::new().group("affinity2"); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have succeeded allocation"); + + assert_eq!( + resource.sled_id.into_untyped_uuid(), + sleds[2].id(), + "All sleds: {sled_ids:#?}", + sled_ids = sleds.iter().map(|s| s.identity.id).collect::>(), + ); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_containing_affinity_groups_force_away_from_affine( + ) { + let logctx = dev::test_setup_log("anti_affinity_group_containing_affinity_groups_force_away_from_affine"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 5; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Fail, + }, + Group { + affinity: Affinity::Positive, + name: "affinity1", + policy: external::AffinityPolicy::Allow, + }, + Group { + affinity: Affinity::Positive, + name: "affinity2", + policy: external::AffinityPolicy::Allow, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + all_groups + .add_affinity_groups_to_anti_affinity_group( + &datastore, + "anti-affinity", + &["affinity1", "affinity2"], + ) + .await; + + // The instances on sled 0 and 1 belong to "affinity1" + // The instance on sled 2 belongs to "affinity2" + // The instance on sled 3 directly belongs to "anti-affinity" + // + // Even though the instance belongs to the affinity groups - and + // would "prefer" to be co-located with them - it's forced to be placed + // on sleds[4], since that's the only sled which doesn't violate + // the hard "anti-affinity" requirement. + let instances = [ + Instance::new().group("affinity1").sled(sleds[0].id()), + Instance::new().group("affinity1").sled(sleds[1].id()), + Instance::new().group("affinity2").sled(sleds[2].id()), + Instance::new().group("anti-affinity").sled(sleds[3].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + // This instance is not part of "anti-affinity" directly. However, it + // is indirectly part of the anti-affinity group, because of its + // affinity group memberships. + let test_instance = + Instance::new().group("affinity1").group("affinity2"); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have succeeded allocation"); + + assert_eq!( + resource.sled_id.into_untyped_uuid(), + sleds[4].id(), + "All sleds: {sled_ids:#?}", + sled_ids = sleds.iter().map(|s| s.identity.id).collect::>(), + ); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_containing_affinity_groups_multigroup() { + let logctx = dev::test_setup_log( + "anti_affinity_group_containing_affinity_groups_multigroup", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 3; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Negative, + name: "wolf-eat-goat", + policy: external::AffinityPolicy::Fail, + }, + Group { + affinity: Affinity::Negative, + name: "goat-eat-cabbage", + policy: external::AffinityPolicy::Fail, + }, + Group { + affinity: Affinity::Positive, + name: "wolf", + policy: external::AffinityPolicy::Allow, + }, + Group { + affinity: Affinity::Positive, + name: "goat", + policy: external::AffinityPolicy::Allow, + }, + Group { + affinity: Affinity::Positive, + name: "cabbage", + policy: external::AffinityPolicy::Allow, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + all_groups + .add_affinity_groups_to_anti_affinity_group( + &datastore, + "wolf-eat-goat", + &["wolf", "goat"], + ) + .await; + all_groups + .add_affinity_groups_to_anti_affinity_group( + &datastore, + "goat-eat-cabbage", + &["goat", "cabbage"], + ) + .await; + + let instances = [ + Instance::new().group("wolf").sled(sleds[0].id()), + Instance::new().group("cabbage").sled(sleds[1].id()), + Instance::new().group("goat").sled(sleds[2].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = Instance::new().group("wolf").group("cabbage"); + let resource = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Should have succeeded allocation"); + + // This instance can go on either sled[0] or sled[1], but not sled[2] + assert_ne!( + resource.sled_id.into_untyped_uuid(), + sleds[2].id(), + "All sleds: {sled_ids:#?}", + sled_ids = sleds.iter().map(|s| s.identity.id).collect::>(), + ); + + db.terminate().await; + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn anti_affinity_group_containing_overlapping_affinity_groups() { + let logctx = dev::test_setup_log( + "anti_affinity_group_containing_overlapping_affinity_groups", + ); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + let (authz_project, _project) = + create_project(&opctx, &datastore).await; + + const SLED_COUNT: usize = 4; + let sleds = create_sleds(&datastore, SLED_COUNT).await; + + let groups = [ + Group { + affinity: Affinity::Negative, + name: "anti-affinity", + policy: external::AffinityPolicy::Fail, + }, + Group { + affinity: Affinity::Positive, + name: "affinity1", + policy: external::AffinityPolicy::Allow, + }, + Group { + affinity: Affinity::Positive, + name: "affinity2", + policy: external::AffinityPolicy::Allow, + }, + ]; + let all_groups = + AllGroups::create(&opctx, &datastore, &authz_project, &groups) + .await; + all_groups + .add_affinity_groups_to_anti_affinity_group( + &datastore, + "anti-affinity", + &["affinity1", "affinity2"], + ) + .await; + + // The instances on sled 0 and 1 belong to "affinity1" + // The instances on sled 2 and 3 belong to "affinity2" + // + // If a new instance belongs to both affinity groups, it cannot satisfy + // the anti-affinity requirements. + let instances = [ + Instance::new().group("affinity1").sled(sleds[0].id()), + Instance::new().group("affinity1").sled(sleds[1].id()), + Instance::new().group("affinity2").sled(sleds[2].id()), + Instance::new().group("affinity2").sled(sleds[3].id()), + ]; + for instance in instances { + instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect("Failed to set up instances"); + } + + let test_instance = + Instance::new().group("affinity1").group("affinity2"); + let err = test_instance + .add_to_groups_and_reserve(&opctx, &datastore, &all_groups) + .await + .expect_err("Should have failed to place instance"); + + let SledReservationTransactionError::Reservation( + SledReservationError::NotFound, + ) = err + else { + panic!("Unexpected error: {err:?}"); + }; + + db.terminate().await; + logctx.cleanup_successful(); + } + #[tokio::test] async fn affinity_multi_group() { let logctx = dev::test_setup_log("affinity_multi_group"); diff --git a/nexus/db-queries/src/db/queries/sled_reservation.rs b/nexus/db-queries/src/db/queries/sled_reservation.rs index 912fd644da4..7fe28101a79 100644 --- a/nexus/db-queries/src/db/queries/sled_reservation.rs +++ b/nexus/db-queries/src/db/queries/sled_reservation.rs @@ -57,18 +57,70 @@ pub fn sled_find_targets_query( COALESCE(SUM(CAST(sled_resource_vmm.reservoir_ram AS INT8)), 0) + " ).param().sql(" <= sled.reservoir_size ), - our_aa_groups AS ( + our_a_groups AS ( + SELECT group_id + FROM affinity_group_instance_membership + WHERE instance_id = ").param().sql(" + ), + other_a_instances AS ( + SELECT affinity_group_instance_membership.group_id,instance_id + FROM affinity_group_instance_membership + JOIN our_a_groups + ON affinity_group_instance_membership.group_id = our_a_groups.group_id + WHERE instance_id != ").param().sql(" + ), + our_direct_aa_groups AS ( + -- Anti-affinity groups to which our instance belongs SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = ").param().sql(" ), - other_aa_instances AS ( + other_direct_aa_instances AS ( SELECT anti_affinity_group_instance_membership.group_id,instance_id FROM anti_affinity_group_instance_membership - JOIN our_aa_groups - ON anti_affinity_group_instance_membership.group_id = our_aa_groups.group_id + JOIN our_direct_aa_groups + ON anti_affinity_group_instance_membership.group_id = our_direct_aa_groups.group_id WHERE instance_id != ").param().sql(" ), + our_indirect_aa_groups AS ( + -- Anti-affinity groups to which our instance's affinity groups belong + SELECT + anti_affinity_group_id, + affinity_group_id, + CASE + WHEN COUNT(*) OVER (PARTITION BY anti_affinity_group_id) = 1 THEN TRUE + ELSE FALSE + END AS exactly_one_affinity_group + FROM anti_affinity_group_affinity_membership + WHERE affinity_group_id IN (SELECT group_id FROM our_a_groups) + ), + other_indirect_aa_instances_via_instances AS ( + SELECT anti_affinity_group_id AS group_id,instance_id + FROM anti_affinity_group_instance_membership + JOIN our_indirect_aa_groups + ON anti_affinity_group_instance_membership.group_id = our_indirect_aa_groups.anti_affinity_group_id + ), + other_indirect_aa_instances_via_groups AS ( + SELECT anti_affinity_group_id AS group_id,instance_id + FROM affinity_group_instance_membership + JOIN our_indirect_aa_groups + ON affinity_group_instance_membership.group_id = our_indirect_aa_groups.affinity_group_id + WHERE + -- If our instance belongs to exactly one of these groups... + CASE WHEN our_indirect_aa_groups.exactly_one_affinity_group + -- ... exclude that group from our anti-affinity + THEN affinity_group_instance_membership.group_id NOT IN (SELECT group_id FROM our_a_groups) + -- ... otherwise, consider all groups anti-affine + ELSE TRUE + END + ), + other_aa_instances AS ( + SELECT * FROM other_direct_aa_instances + UNION + SELECT * FROM other_indirect_aa_instances_via_instances + UNION + SELECT * FROM other_indirect_aa_instances_via_groups + ), other_aa_instances_by_policy AS ( SELECT policy,instance_id FROM other_aa_instances @@ -85,18 +137,6 @@ pub fn sled_find_targets_query( ON sled_resource_vmm.instance_id = other_aa_instances_by_policy.instance_id ), - our_a_groups AS ( - SELECT group_id - FROM affinity_group_instance_membership - WHERE instance_id = ").param().sql(" - ), - other_a_instances AS ( - SELECT affinity_group_instance_membership.group_id,instance_id - FROM affinity_group_instance_membership - JOIN our_a_groups - ON affinity_group_instance_membership.group_id = our_a_groups.group_id - WHERE instance_id != ").param().sql(" - ), other_a_instances_by_policy AS ( SELECT policy,instance_id FROM other_a_instances @@ -172,18 +212,70 @@ pub fn sled_insert_resource_query( COALESCE(SUM(CAST(sled_resource_vmm.reservoir_ram AS INT8)), 0) + " ).param().sql(" <= sled.reservoir_size ), - our_aa_groups AS ( + our_a_groups AS ( + SELECT group_id + FROM affinity_group_instance_membership + WHERE instance_id = ").param().sql(" + ), + other_a_instances AS ( + SELECT affinity_group_instance_membership.group_id,instance_id + FROM affinity_group_instance_membership + JOIN our_a_groups + ON affinity_group_instance_membership.group_id = our_a_groups.group_id + WHERE instance_id != ").param().sql(" + ), + our_direct_aa_groups AS ( + -- Anti-affinity groups to which our instance belongs SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = ").param().sql(" ), - other_aa_instances AS ( + other_direct_aa_instances AS ( SELECT anti_affinity_group_instance_membership.group_id,instance_id FROM anti_affinity_group_instance_membership - JOIN our_aa_groups - ON anti_affinity_group_instance_membership.group_id = our_aa_groups.group_id + JOIN our_direct_aa_groups + ON anti_affinity_group_instance_membership.group_id = our_direct_aa_groups.group_id WHERE instance_id != ").param().sql(" ), + our_indirect_aa_groups AS ( + -- Anti-affinity groups to which our instance's affinity groups belong + SELECT + anti_affinity_group_id, + affinity_group_id, + CASE + WHEN COUNT(*) OVER (PARTITION BY anti_affinity_group_id) = 1 THEN TRUE + ELSE FALSE + END AS only_one_affinity_group + FROM anti_affinity_group_affinity_membership + WHERE affinity_group_id IN (SELECT group_id FROM our_a_groups) + ), + other_indirect_aa_instances_via_instances AS ( + SELECT anti_affinity_group_id AS group_id,instance_id + FROM anti_affinity_group_instance_membership + JOIN our_indirect_aa_groups + ON anti_affinity_group_instance_membership.group_id = our_indirect_aa_groups.anti_affinity_group_id + ), + other_indirect_aa_instances_via_groups AS ( + SELECT anti_affinity_group_id AS group_id,instance_id + FROM affinity_group_instance_membership + JOIN our_indirect_aa_groups + ON affinity_group_instance_membership.group_id = our_indirect_aa_groups.affinity_group_id + WHERE + -- If our instance belongs to exactly one of these groups... + CASE WHEN our_indirect_aa_groups.only_one_affinity_group + -- ... exclude that group from our anti-affinity + THEN affinity_group_instance_membership.group_id NOT IN (SELECT group_id FROM our_a_groups) + -- ... otherwise, consider all groups anti-affine + ELSE TRUE + END + ), + other_aa_instances AS ( + SELECT * FROM other_direct_aa_instances + UNION + SELECT * FROM other_indirect_aa_instances_via_instances + UNION + SELECT * FROM other_indirect_aa_instances_via_groups + ), banned_instances AS ( SELECT instance_id FROM other_aa_instances @@ -201,18 +293,6 @@ pub fn sled_insert_resource_query( ON sled_resource_vmm.instance_id = banned_instances.instance_id ), - our_a_groups AS ( - SELECT group_id - FROM affinity_group_instance_membership - WHERE instance_id = ").param().sql(" - ), - other_a_instances AS ( - SELECT affinity_group_instance_membership.group_id,instance_id - FROM affinity_group_instance_membership - JOIN our_a_groups - ON affinity_group_instance_membership.group_id = our_a_groups.group_id - WHERE instance_id != ").param().sql(" - ), required_instances AS ( SELECT policy,instance_id FROM other_a_instances diff --git a/nexus/db-queries/tests/output/sled_find_targets_query.sql b/nexus/db-queries/tests/output/sled_find_targets_query.sql index 9de50f47f7f..f230a4b6dcb 100644 --- a/nexus/db-queries/tests/output/sled_find_targets_query.sql +++ b/nexus/db-queries/tests/output/sled_find_targets_query.sql @@ -17,18 +17,75 @@ WITH AND COALESCE(sum(CAST(sled_resource_vmm.reservoir_ram AS INT8)), 0) + $3 <= sled.reservoir_size ), - our_aa_groups - AS (SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = $4), - other_aa_instances + our_a_groups AS (SELECT group_id FROM affinity_group_instance_membership WHERE instance_id = $4), + other_a_instances + AS ( + SELECT + affinity_group_instance_membership.group_id, instance_id + FROM + affinity_group_instance_membership + JOIN our_a_groups ON affinity_group_instance_membership.group_id = our_a_groups.group_id + WHERE + instance_id != $5 + ), + our_direct_aa_groups + AS (SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = $6), + other_direct_aa_instances AS ( SELECT anti_affinity_group_instance_membership.group_id, instance_id FROM anti_affinity_group_instance_membership - JOIN our_aa_groups ON - anti_affinity_group_instance_membership.group_id = our_aa_groups.group_id + JOIN our_direct_aa_groups ON + anti_affinity_group_instance_membership.group_id = our_direct_aa_groups.group_id WHERE - instance_id != $5 + instance_id != $7 + ), + our_indirect_aa_groups + AS ( + SELECT + anti_affinity_group_id, + affinity_group_id, + CASE + WHEN count(*) OVER (PARTITION BY anti_affinity_group_id) = 1 THEN true + ELSE false + END + AS exactly_one_affinity_group + FROM + anti_affinity_group_affinity_membership + WHERE + affinity_group_id IN (SELECT group_id FROM our_a_groups) + ), + other_indirect_aa_instances_via_instances + AS ( + SELECT + anti_affinity_group_id AS group_id, instance_id + FROM + anti_affinity_group_instance_membership + JOIN our_indirect_aa_groups ON + anti_affinity_group_instance_membership.group_id + = our_indirect_aa_groups.anti_affinity_group_id + ), + other_indirect_aa_instances_via_groups + AS ( + SELECT + anti_affinity_group_id AS group_id, instance_id + FROM + affinity_group_instance_membership + JOIN our_indirect_aa_groups ON + affinity_group_instance_membership.group_id = our_indirect_aa_groups.affinity_group_id + WHERE + CASE + WHEN our_indirect_aa_groups.exactly_one_affinity_group + THEN affinity_group_instance_membership.group_id NOT IN (SELECT group_id FROM our_a_groups) + ELSE true + END + ), + other_aa_instances + AS ( + SELECT * FROM other_direct_aa_instances + UNION SELECT * FROM other_indirect_aa_instances_via_instances + UNION SELECT * FROM other_indirect_aa_instances_via_groups ), other_aa_instances_by_policy AS ( @@ -51,17 +108,6 @@ WITH JOIN sled_resource_vmm ON sled_resource_vmm.instance_id = other_aa_instances_by_policy.instance_id ), - our_a_groups AS (SELECT group_id FROM affinity_group_instance_membership WHERE instance_id = $6), - other_a_instances - AS ( - SELECT - affinity_group_instance_membership.group_id, instance_id - FROM - affinity_group_instance_membership - JOIN our_a_groups ON affinity_group_instance_membership.group_id = our_a_groups.group_id - WHERE - instance_id != $7 - ), other_a_instances_by_policy AS ( SELECT diff --git a/nexus/db-queries/tests/output/sled_insert_resource_query.sql b/nexus/db-queries/tests/output/sled_insert_resource_query.sql index 9cfb68e3008..a0ae17555dd 100644 --- a/nexus/db-queries/tests/output/sled_insert_resource_query.sql +++ b/nexus/db-queries/tests/output/sled_insert_resource_query.sql @@ -20,18 +20,75 @@ WITH AND COALESCE(sum(CAST(sled_resource_vmm.reservoir_ram AS INT8)), 0) + $4 <= sled.reservoir_size ), - our_aa_groups - AS (SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = $5), - other_aa_instances + our_a_groups AS (SELECT group_id FROM affinity_group_instance_membership WHERE instance_id = $5), + other_a_instances + AS ( + SELECT + affinity_group_instance_membership.group_id, instance_id + FROM + affinity_group_instance_membership + JOIN our_a_groups ON affinity_group_instance_membership.group_id = our_a_groups.group_id + WHERE + instance_id != $6 + ), + our_direct_aa_groups + AS (SELECT group_id FROM anti_affinity_group_instance_membership WHERE instance_id = $7), + other_direct_aa_instances AS ( SELECT anti_affinity_group_instance_membership.group_id, instance_id FROM anti_affinity_group_instance_membership - JOIN our_aa_groups ON - anti_affinity_group_instance_membership.group_id = our_aa_groups.group_id + JOIN our_direct_aa_groups ON + anti_affinity_group_instance_membership.group_id = our_direct_aa_groups.group_id WHERE - instance_id != $6 + instance_id != $8 + ), + our_indirect_aa_groups + AS ( + SELECT + anti_affinity_group_id, + affinity_group_id, + CASE + WHEN count(*) OVER (PARTITION BY anti_affinity_group_id) = 1 THEN true + ELSE false + END + AS only_one_affinity_group + FROM + anti_affinity_group_affinity_membership + WHERE + affinity_group_id IN (SELECT group_id FROM our_a_groups) + ), + other_indirect_aa_instances_via_instances + AS ( + SELECT + anti_affinity_group_id AS group_id, instance_id + FROM + anti_affinity_group_instance_membership + JOIN our_indirect_aa_groups ON + anti_affinity_group_instance_membership.group_id + = our_indirect_aa_groups.anti_affinity_group_id + ), + other_indirect_aa_instances_via_groups + AS ( + SELECT + anti_affinity_group_id AS group_id, instance_id + FROM + affinity_group_instance_membership + JOIN our_indirect_aa_groups ON + affinity_group_instance_membership.group_id = our_indirect_aa_groups.affinity_group_id + WHERE + CASE + WHEN our_indirect_aa_groups.only_one_affinity_group + THEN affinity_group_instance_membership.group_id NOT IN (SELECT group_id FROM our_a_groups) + ELSE true + END + ), + other_aa_instances + AS ( + SELECT * FROM other_direct_aa_instances + UNION SELECT * FROM other_indirect_aa_instances_via_instances + UNION SELECT * FROM other_indirect_aa_instances_via_groups ), banned_instances AS ( @@ -54,17 +111,6 @@ WITH banned_instances JOIN sled_resource_vmm ON sled_resource_vmm.instance_id = banned_instances.instance_id ), - our_a_groups AS (SELECT group_id FROM affinity_group_instance_membership WHERE instance_id = $7), - other_a_instances - AS ( - SELECT - affinity_group_instance_membership.group_id, instance_id - FROM - affinity_group_instance_membership - JOIN our_a_groups ON affinity_group_instance_membership.group_id = our_a_groups.group_id - WHERE - instance_id != $8 - ), required_instances AS ( SELECT diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index c843090eddc..0f6b23286dc 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -1409,7 +1409,7 @@ pub trait NexusExternalApi { path_params: Path, ) -> Result>, HttpError>; - /// Fetch an anti-affinity group member + /// Fetch an anti-affinity group member (where that member is an instance) #[endpoint { method = GET, path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}", @@ -1421,7 +1421,7 @@ pub trait NexusExternalApi { path_params: Path, ) -> Result, HttpError>; - /// Add a member to an anti-affinity group + /// Add a member to an anti-affinity group (where that member is an instance) #[endpoint { method = POST, path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}", @@ -1433,7 +1433,7 @@ pub trait NexusExternalApi { path_params: Path, ) -> Result, HttpError>; - /// Remove a member from an anti-affinity group + /// Remove a member from an anti-affinity group (where that member is an instance) #[endpoint { method = DELETE, path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/instance/{instance}", @@ -1445,6 +1445,42 @@ pub trait NexusExternalApi { path_params: Path, ) -> Result; + /// Fetch an anti-affinity group member (where that member is an affinity group) + #[endpoint { + method = GET, + path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/affinity-group/{affinity_group}", + tags = ["affinity"], + }] + async fn anti_affinity_group_member_affinity_group_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// Add a member to an anti-affinity group (where that member is an affinity group) + #[endpoint { + method = POST, + path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/affinity-group/{affinity_group}", + tags = ["affinity"], + }] + async fn anti_affinity_group_member_affinity_group_add( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// Remove a member from an anti-affinity group (where that member is an affinity group) + #[endpoint { + method = DELETE, + path = "/v1/anti-affinity-groups/{anti_affinity_group}/members/affinity-group/{affinity_group}", + tags = ["affinity"], + }] + async fn anti_affinity_group_member_affinity_group_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result; + /// Create an anti-affinity group #[endpoint { method = POST, diff --git a/nexus/src/app/affinity.rs b/nexus/src/app/affinity.rs index 5a1ebf7ab22..0930e74ece4 100644 --- a/nexus/src/app/affinity.rs +++ b/nexus/src/app/affinity.rs @@ -15,12 +15,14 @@ use nexus_types::external_api::views; use omicron_common::api::external; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; +use omicron_common::api::external::DataPageParams; use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_common::api::external::NameOrId; use omicron_common::api::external::UpdateResult; +use omicron_uuid_kinds::AffinityGroupUuid; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceUuid; @@ -253,22 +255,18 @@ impl super::Nexus { &self, opctx: &OpContext, anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, - pagparams: &PaginatedBy<'_>, + pagparams: &DataPageParams<'_, uuid::Uuid>, ) -> ListResultVec { let (.., authz_anti_affinity_group) = anti_affinity_group_lookup .lookup_for(authz::Action::ListChildren) .await?; - Ok(self - .db_datastore + self.db_datastore .anti_affinity_group_member_list( opctx, &authz_anti_affinity_group, pagparams, ) - .await? - .into_iter() - .map(Into::into) - .collect()) + .await } pub(crate) async fn affinity_group_member_view( @@ -290,7 +288,7 @@ impl super::Nexus { .await } - pub(crate) async fn anti_affinity_group_member_view( + pub(crate) async fn anti_affinity_group_member_instance_view( &self, opctx: &OpContext, anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, @@ -313,6 +311,29 @@ impl super::Nexus { .await } + pub(crate) async fn anti_affinity_group_member_affinity_group_view( + &self, + opctx: &OpContext, + anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, + affinity_group_lookup: &lookup::AffinityGroup<'_>, + ) -> Result { + let (.., authz_anti_affinity_group) = + anti_affinity_group_lookup.lookup_for(authz::Action::Read).await?; + let (.., authz_affinity_group) = + affinity_group_lookup.lookup_for(authz::Action::Read).await?; + let member = external::AntiAffinityGroupMember::AffinityGroup( + AffinityGroupUuid::from_untyped_uuid(authz_affinity_group.id()), + ); + + self.db_datastore + .anti_affinity_group_member_view( + opctx, + &authz_anti_affinity_group, + member, + ) + .await + } + pub(crate) async fn affinity_group_member_add( &self, opctx: &OpContext, @@ -337,7 +358,7 @@ impl super::Nexus { Ok(member) } - pub(crate) async fn anti_affinity_group_member_add( + pub(crate) async fn anti_affinity_group_member_instance_add( &self, opctx: &OpContext, anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, @@ -362,6 +383,31 @@ impl super::Nexus { Ok(member) } + pub(crate) async fn anti_affinity_group_member_affinity_group_add( + &self, + opctx: &OpContext, + anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, + affinity_group_lookup: &lookup::AffinityGroup<'_>, + ) -> Result { + let (.., authz_anti_affinity_group) = anti_affinity_group_lookup + .lookup_for(authz::Action::Modify) + .await?; + let (.., authz_affinity_group) = + affinity_group_lookup.lookup_for(authz::Action::Read).await?; + let member = external::AntiAffinityGroupMember::AffinityGroup( + AffinityGroupUuid::from_untyped_uuid(authz_affinity_group.id()), + ); + + self.db_datastore + .anti_affinity_group_member_add( + opctx, + &authz_anti_affinity_group, + member.clone(), + ) + .await?; + Ok(member) + } + pub(crate) async fn affinity_group_member_delete( &self, opctx: &OpContext, @@ -381,7 +427,7 @@ impl super::Nexus { .await } - pub(crate) async fn anti_affinity_group_member_delete( + pub(crate) async fn anti_affinity_group_member_instance_delete( &self, opctx: &OpContext, anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, @@ -404,4 +450,28 @@ impl super::Nexus { ) .await } + + pub(crate) async fn anti_affinity_group_member_affinity_group_delete( + &self, + opctx: &OpContext, + anti_affinity_group_lookup: &lookup::AntiAffinityGroup<'_>, + affinity_group_lookup: &lookup::AffinityGroup<'_>, + ) -> Result<(), Error> { + let (.., authz_anti_affinity_group) = anti_affinity_group_lookup + .lookup_for(authz::Action::Modify) + .await?; + let (.., authz_affinity_group) = + affinity_group_lookup.lookup_for(authz::Action::Read).await?; + let member = external::AntiAffinityGroupMember::AffinityGroup( + AffinityGroupUuid::from_untyped_uuid(authz_affinity_group.id()), + ); + + self.db_datastore + .anti_affinity_group_member_delete( + opctx, + &authz_anti_affinity_group, + member, + ) + .await + } } diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index b1cf5b59e5a..246cc86c551 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -2925,7 +2925,6 @@ impl NexusExternalApi for NexusExternalApiImpl { let query = query_params.into_inner(); let pag_params = data_page_params_for(&rqctx, &query)?; let scan_params = ScanById::from_query(&query)?; - let paginated_by = id_pagination(&pag_params, scan_params)?; let group_selector = params::AntiAffinityGroupSelector { project: scan_params.selector.project.clone(), @@ -2937,7 +2936,7 @@ impl NexusExternalApi for NexusExternalApiImpl { .anti_affinity_group_member_list( &opctx, &group_lookup, - &paginated_by, + &pag_params, ) .await?; Ok(HttpResponseOk(ScanById::results_page( @@ -2983,7 +2982,7 @@ impl NexusExternalApi for NexusExternalApiImpl { nexus.instance_lookup(&opctx, instance_selector)?; let group = nexus - .anti_affinity_group_member_view( + .anti_affinity_group_member_instance_view( &opctx, &group_lookup, &instance_lookup, @@ -3029,7 +3028,7 @@ impl NexusExternalApi for NexusExternalApiImpl { nexus.instance_lookup(&opctx, instance_selector)?; let member = nexus - .anti_affinity_group_member_add( + .anti_affinity_group_member_instance_add( &opctx, &group_lookup, &instance_lookup, @@ -3074,7 +3073,7 @@ impl NexusExternalApi for NexusExternalApiImpl { nexus.instance_lookup(&opctx, instance_selector)?; nexus - .anti_affinity_group_member_delete( + .anti_affinity_group_member_instance_delete( &opctx, &group_lookup, &instance_lookup, @@ -3089,6 +3088,142 @@ impl NexusExternalApi for NexusExternalApiImpl { .await } + async fn anti_affinity_group_member_affinity_group_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let query = query_params.into_inner(); + + // Select anti-affinity group + let group_selector = params::AntiAffinityGroupSelector { + anti_affinity_group: path.anti_affinity_group, + project: query.project.clone(), + }; + let group_lookup = + nexus.anti_affinity_group_lookup(&opctx, group_selector)?; + + // Select affinity group + let affinity_group_selector = params::AffinityGroupSelector { + project: query.project, + affinity_group: path.affinity_group, + }; + let affinity_group_lookup = + nexus.affinity_group_lookup(&opctx, affinity_group_selector)?; + + let group = nexus + .anti_affinity_group_member_affinity_group_view( + &opctx, + &group_lookup, + &affinity_group_lookup, + ) + .await?; + + Ok(HttpResponseOk(group)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_member_affinity_group_add( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + + // Select anti-affinity group + let group_selector = params::AntiAffinityGroupSelector { + anti_affinity_group: path.anti_affinity_group, + project: query.project.clone(), + }; + let group_lookup = + nexus.anti_affinity_group_lookup(&opctx, group_selector)?; + + // Select affinity group + let affinity_group_selector = params::AffinityGroupSelector { + project: query.project, + affinity_group: path.affinity_group, + }; + let affinity_group_lookup = + nexus.affinity_group_lookup(&opctx, affinity_group_selector)?; + + let member = nexus + .anti_affinity_group_member_affinity_group_add( + &opctx, + &group_lookup, + &affinity_group_lookup, + ) + .await?; + Ok(HttpResponseCreated(member)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn anti_affinity_group_member_affinity_group_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + + // Select anti-affinity group + let group_selector = params::AntiAffinityGroupSelector { + anti_affinity_group: path.anti_affinity_group, + project: query.project.clone(), + }; + let group_lookup = + nexus.anti_affinity_group_lookup(&opctx, group_selector)?; + + // Select affinity group + let affinity_group_selector = params::AffinityGroupSelector { + project: query.project, + affinity_group: path.affinity_group, + }; + let affinity_group_lookup = + nexus.affinity_group_lookup(&opctx, affinity_group_selector)?; + + nexus + .anti_affinity_group_member_affinity_group_delete( + &opctx, + &group_lookup, + &affinity_group_lookup, + ) + .await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + async fn anti_affinity_group_create( rqctx: RequestContext, query_params: Query, diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 100768e8b19..3def55d69db 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -840,6 +840,12 @@ pub struct AntiAffinityInstanceGroupMemberPath { pub instance: NameOrId, } +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct AntiAffinityAffinityGroupMemberPath { + pub anti_affinity_group: NameOrId, + pub affinity_group: NameOrId, +} + /// Create-time parameters for an `AntiAffinityGroup` #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct AntiAffinityGroupCreate { diff --git a/openapi/nexus.json b/openapi/nexus.json index 17088ed5c9c..ec00fdab967 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -12504,6 +12504,25 @@ "type", "value" ] + }, + { + "description": "An affinity group belonging to this group, identified by UUID.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "affinity_group" + ] + }, + "value": { + "$ref": "#/components/schemas/TypedUuidForAffinityGroupKind" + } + }, + "required": [ + "type", + "value" + ] } ] }, @@ -23053,6 +23072,10 @@ } } }, + "TypedUuidForAffinityGroupKind": { + "type": "string", + "format": "uuid" + }, "TypedUuidForInstanceKind": { "type": "string", "format": "uuid" diff --git a/schema/crdb/anti-affinity-group-affinity-member/up01.sql b/schema/crdb/anti-affinity-group-affinity-member/up01.sql new file mode 100644 index 00000000000..4e16d622466 --- /dev/null +++ b/schema/crdb/anti-affinity-group-affinity-member/up01.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS omicron.public.anti_affinity_group_affinity_membership ( + anti_affinity_group_id UUID NOT NULL, + affinity_group_id UUID NOT NULL, + + PRIMARY KEY (anti_affinity_group_id, affinity_group_id) +); diff --git a/schema/crdb/anti-affinity-group-affinity-member/up02.sql b/schema/crdb/anti-affinity-group-affinity-member/up02.sql new file mode 100644 index 00000000000..459c36df1bb --- /dev/null +++ b/schema/crdb/anti-affinity-group-affinity-member/up02.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_anti_affinity_group_affinity_membership_by_affinity_group ON omicron.public.anti_affinity_group_affinity_membership ( + affinity_group_id +); + diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 1039b634b52..0b44c0dc2a2 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4239,6 +4239,25 @@ CREATE INDEX IF NOT EXISTS lookup_anti_affinity_group_instance_membership_by_ins instance_id ); +-- Describes an affinity group's membership within an anti-affinity group +-- +-- Since the naming here is a little confusing: +-- This allows an anti-affinity group to contain affinity groups as members. +-- This is useful for saying "I want these groups of VMMs to be anti-affine from +-- one another", rather than "I want individual VMMs to be anti-affine from each other". +CREATE TABLE IF NOT EXISTS omicron.public.anti_affinity_group_affinity_membership ( + anti_affinity_group_id UUID NOT NULL, + affinity_group_id UUID NOT NULL, + + PRIMARY KEY (anti_affinity_group_id, affinity_group_id) +); + +-- We need to look up all memberships of an affinity group so we can revoke these +-- memberships efficiently when affinity groups are deleted +CREATE INDEX IF NOT EXISTS lookup_anti_affinity_group_affinity_membership_by_affinity_group ON omicron.public.anti_affinity_group_affinity_membership ( + affinity_group_id +); + -- Per-VMM state. CREATE TABLE IF NOT EXISTS omicron.public.vmm ( id UUID PRIMARY KEY, @@ -4971,7 +4990,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '128.0.0', NULL) + (TRUE, NOW(), NOW(), '129.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT;