Assert that the table invariant is preserved

Author: bkirwi

src/storage-client/src/storage_collections.rs

@@ -1286,6 +1286,35 @@ where
                 .send(BackgroundCmd::DowngradeSince(persist_compaction_commands))
                 .expect("cannot fail to send");
         }
+
+        // Assert table invariants
+        for (id, state) in &*collections {
+            match &state.description.data_source {
+                DataSource::Table {
+                    primary: Some(mut primary_id),
+                } => {
+                    let primary = loop {
+                        let primary = collections.get(&primary_id).expect("must still exist");
+                        if let DataSource::Table {
+                            primary: Some(other),
+                        } = &primary.description.data_source
+                        {
+                            primary_id = *other;
+                        } else {
+                            break primary;
+                        }
+                    };
+                    assert!(
+                        PartialOrder::less_equal(
+                            &primary.read_capabilities.frontier(),
+                            &state.read_capabilities.frontier(),
+                        ),
+                        "primary ({primary_id}) since must be held back at least as much as secondary ({id})"
+                    )
+                }
+                _ => {}
+            }
+        }
     }
 
     /// Remove any shards that we know are finalized