Support a new Durability::NEVER_CHANGE

Inputs with this durability are assumed to never change. This allows us to optimize their storage a lot and not store any data about them except for the value. That makes them very compact.

This is done by bit-tagging the memo and using it as one of two types.

The overhead of handling the possibility of a second type will be seen in synthetic benchmarks; in practice, if using this durability (for example, for library code), speed will increase, not decrease (in addition to improved memory usage), because we don't need to validate query dependencies if they are assumed to never change.

Values participating in cycles cannot have `Durability::NEVER_CHANGE`: while it is possible to support this (and in fact, an early revision of the code did) it introduces a lot of complications to the code, as we need to replace the type of a memo after it has been inserted.

Author: ChayimFriedman2

components/salsa-macro-rules/src/setup_input_struct.rs

@@ -392,7 +392,7 @@ macro_rules! setup_input_struct {
                 pub(super) fn new_builder($($field_id: $field_ty),*) -> $Builder {
                     $Builder {
                         fields: ($($field_id,)*),
-                        durabilities: [::salsa::Durability::default(); $N],
+                        durabilities: [::salsa::Durability::MIN; $N],
                     }
                 }
 

components/salsa-macro-rules/src/setup_tracked_fn.rs

@@ -391,7 +391,7 @@ macro_rules! setup_tracked_fn {
                             zalsa,
                             struct_index,
                             first_index,
-                            $zalsa::function::MemoEntryType::of::<$zalsa::function::Memo<$Configuration>>(),
+                            $zalsa::function::MemoEntryType::of::<$zalsa::function::AmbiguousMemo<$Configuration>>(),
                             intern_ingredient_memo_types,
                         )
                     };

src/accumulator/accumulated_map.rs

@@ -21,6 +21,10 @@ impl std::fmt::Debug for AccumulatedMap {
 }
 
 impl AccumulatedMap {
+    pub(crate) const EMPTY: AccumulatedMap = AccumulatedMap {
+        map: hashbrown::HashMap::with_hasher(FxBuildHasher),
+    };
+
     pub fn accumulate<A: Accumulator>(&mut self, index: IngredientIndex, value: A) {
         self.map
             .entry(index)

src/active_query.rs

@@ -106,7 +106,6 @@ impl ActiveQuery {
     ) {
         self.durability = self.durability.min(durability);
         self.changed_at = self.changed_at.max(changed_at);
-        self.input_outputs.insert(QueryEdge::input(input));
         self.cycle_heads.extend(cycle_heads);
         #[cfg(feature = "accumulator")]
         {
@@ -115,6 +114,10 @@ impl ActiveQuery {
                 false => accumulated_inputs.load(),
             });
         }
+        if !cycle_heads.is_empty() || durability != Durability::NEVER_CHANGE {
+            // During cycles we need to record dependencies.
+            self.input_outputs.insert(QueryEdge::input(input));
+        }
     }
 
     pub(super) fn add_read_simple(
@@ -125,7 +128,9 @@ impl ActiveQuery {
     ) {
         self.durability = self.durability.min(durability);
         self.changed_at = self.changed_at.max(revision);
-        self.input_outputs.insert(QueryEdge::input(input));
+        if durability != Durability::NEVER_CHANGE {
+            self.input_outputs.insert(QueryEdge::input(input));
+        }
     }
 
     pub(super) fn add_untracked_read(&mut self, changed_at: Revision) {
@@ -147,7 +152,9 @@ impl ActiveQuery {
 
     /// Adds a key to our list of outputs.
     pub(super) fn add_output(&mut self, key: DatabaseKeyIndex) {
-        self.input_outputs.insert(QueryEdge::output(key));
+        if self.durability != Durability::NEVER_CHANGE {
+            self.input_outputs.insert(QueryEdge::output(key));
+        }
     }
 
     /// True if the given key was output by this query.

src/cycle.rs

@@ -217,6 +217,7 @@ impl<'de> serde::Deserialize<'de> for AtomicIterationCount {
 pub struct CycleHeads(ThinVec<CycleHead>);
 
 impl CycleHeads {
+    #[inline]
     pub(crate) fn is_empty(&self) -> bool {
         self.0.is_empty()
     }
@@ -505,6 +506,7 @@ pub enum ProvisionalStatus<'db> {
         iteration: IterationCount,
         verified_at: Revision,
     },
+    FinalNeverChange,
     FallbackImmediate,
 }
 

src/durability.rs

@@ -46,6 +46,7 @@ impl std::fmt::Debug for Durability {
                 DurabilityVal::Low => f.write_str("Durability::LOW"),
                 DurabilityVal::Medium => f.write_str("Durability::MEDIUM"),
                 DurabilityVal::High => f.write_str("Durability::HIGH"),
+                DurabilityVal::NeverChange => f.write_str("Durability::NEVER_CHANGE"),
             }
         } else {
             f.debug_tuple("Durability")
@@ -61,14 +62,17 @@ enum DurabilityVal {
     Low = 0,
     Medium = 1,
     High = 2,
+    NeverChange = 3,
 }
 
 impl From<u8> for DurabilityVal {
+    #[inline]
     fn from(value: u8) -> Self {
         match value {
             0 => DurabilityVal::Low,
             1 => DurabilityVal::Medium,
             2 => DurabilityVal::High,
+            3 => DurabilityVal::NeverChange,
             _ => panic!("invalid durability"),
         }
     }
@@ -87,23 +91,28 @@ impl Durability {
 
     /// High durability: things that are not expected to change under
     /// common usage.
-    ///
-    /// Example: the standard library or something from crates.io
     pub const HIGH: Durability = Durability(DurabilityVal::High);
 
+    /// The input is guaranteed to never change. Queries calling it won't have
+    /// it as a dependency.
+    ///
+    /// Example: the standard library or something from crates.io.
+    pub const NEVER_CHANGE: Durability = Durability(DurabilityVal::NeverChange);
+
     /// The minimum possible durability; equivalent to LOW but
     /// "conceptually" distinct (i.e., if we add more durability
     /// levels, this could change).
-    pub(crate) const MIN: Durability = Self::LOW;
+    pub const MIN: Durability = Self::LOW;
 
-    /// The maximum possible durability; equivalent to HIGH but
+    /// The maximum possible durability; equivalent to NEVER_CHANGE but
     /// "conceptually" distinct (i.e., if we add more durability
     /// levels, this could change).
-    pub(crate) const MAX: Durability = Self::HIGH;
+    pub(crate) const MAX: Durability = Self::NEVER_CHANGE;
 
     /// Number of durability levels.
-    pub(crate) const LEN: usize = Self::HIGH.0 as usize + 1;
+    pub(crate) const LEN: usize = Self::MAX.0 as usize + 1;
 
+    #[inline]
     pub(crate) fn index(self) -> usize {
         self.0 as usize
     }

src/function.rs

@@ -16,13 +16,15 @@ use crate::key::DatabaseKeyIndex;
 use crate::plumbing::{self, MemoIngredientMap};
 use crate::salsa_struct::SalsaStructInDb;
 use crate::sync::Arc;
-use crate::table::memo::MemoTableTypes;
+use crate::table::memo::{Either, MemoTableTypes};
 use crate::table::Table;
 use crate::views::DatabaseDownCaster;
 use crate::zalsa::{IngredientIndex, JarKind, MemoIngredientIndex, Zalsa};
 use crate::zalsa_local::{QueryEdge, QueryOriginRef};
 use crate::{Cycle, Durability, Id, Revision};
 
+pub use crate::function::memo::AmbiguousMemo;
+
 #[cfg(feature = "accumulator")]
 mod accumulated;
 mod backdate;
@@ -37,7 +39,9 @@ mod memo;
 mod specify;
 mod sync;
 
-pub type Memo<C> = memo::Memo<'static, C>;
+type EitherMemoNonNull<'db, C> =
+    Either<NonNull<memo::Memo<'db, C>>, NonNull<memo::NeverChangeMemo<'db, C>>>;
+type EitherMemoRef<'a, 'db, C> = Either<&'a memo::Memo<'db, C>, &'a memo::NeverChangeMemo<'db, C>>;
 
 pub trait Configuration: Any {
     const DEBUG_NAME: &'static str;
@@ -246,6 +250,7 @@ where
     /// when this function is called and (b) ensuring that any entries
     /// removed from the memo-map are added to `deleted_entries`, which is
     /// only cleared with `&mut self`.
+    #[inline]
     unsafe fn extend_memo_lifetime<'this>(
         &'this self,
         memo: &memo::Memo<'this, C>,
@@ -254,6 +259,15 @@ where
         unsafe { std::mem::transmute(memo) }
     }
 
+    #[inline]
+    unsafe fn extend_either_memo_lifetime<'this>(
+        &'this self,
+        memo: EitherMemoRef<'_, 'this, C>,
+    ) -> EitherMemoRef<'this, 'this, C> {
+        // SAFETY: the caller must guarantee that the memo will not be released before `&self`
+        unsafe { std::mem::transmute(memo) }
+    }
+
     fn insert_memo<'db>(
         &'db self,
         zalsa: &'db Zalsa,
@@ -284,6 +298,39 @@ where
         unsafe { self.extend_memo_lifetime(memo.as_ref()) }
     }
 
+    fn insert_never_change_memo<'db>(
+        &'db self,
+        zalsa: &'db Zalsa,
+        id: Id,
+        memo: memo::NeverChangeMemo<'db, C>,
+        memo_ingredient_index: MemoIngredientIndex,
+    ) -> EitherMemoRef<'db, 'db, C> {
+        // We convert to a `NonNull` here as soon as possible because we are going to alias
+        // into the `Box`, which is a `noalias` type.
+        // SAFETY: memo is not null
+        let memo = unsafe { NonNull::new_unchecked(Box::into_raw(Box::new(memo))) };
+
+        // SAFETY: memo must be in the map (it's not yet, but it will be by the time this
+        // value is returned) and anything removed from map is added to deleted entries (ensured elsewhere).
+        let db_memo = unsafe { self.extend_either_memo_lifetime(Either::Right(memo.as_ref())) };
+
+        if let Some(old_value) =
+            // SAFETY: We delay the drop of `old_value` until a new revision starts which ensures no
+            // references will exist for the memo contents.
+            unsafe {
+                self.insert_never_change_memo_into_table_for(zalsa, id, memo, memo_ingredient_index)
+            }
+        {
+            // In case there is a reference to the old memo out there, we have to store it
+            // in the deleted entries. This will get cleared when a new revision starts.
+            //
+            // SAFETY: Once the revision starts, there will be no outstanding borrows to the
+            // memo contents, and so it will be safe to free.
+            unsafe { self.deleted_entries.push(old_value) };
+        }
+        db_memo
+    }
+
     #[inline]
     fn memo_ingredient_index(&self, zalsa: &Zalsa, id: Id) -> MemoIngredientIndex {
         self.memo_ingredient_indices.get_zalsa_id(zalsa, id)
@@ -335,6 +382,7 @@ where
         else {
             return;
         };
+        let Either::Left(memo) = memo else { todo!() };
 
         let origin = memo.revisions.origin.as_ref();
 
@@ -371,8 +419,11 @@ where
         zalsa: &'db Zalsa,
         input: Id,
     ) -> Option<ProvisionalStatus<'db>> {
-        let memo =
-            self.get_memo_from_table_for(zalsa, input, self.memo_ingredient_index(zalsa, input))?;
+        let Either::Left(memo) =
+            self.get_memo_from_table_for(zalsa, input, self.memo_ingredient_index(zalsa, input))?
+        else {
+            return Some(ProvisionalStatus::FinalNeverChange);
+        };
 
         let iteration = memo.revisions.iteration();
         let verified_final = memo.revisions.verified_final.load(Ordering::Relaxed);
@@ -396,7 +447,7 @@ where
     }
 
     fn set_cycle_iteration_count(&self, zalsa: &Zalsa, input: Id, iteration_count: IterationCount) {
-        let Some(memo) =
+        let Some(Either::Left(memo)) =
             self.get_memo_from_table_for(zalsa, input, self.memo_ingredient_index(zalsa, input))
         else {
             return;
@@ -407,7 +458,7 @@ where
     }
 
     fn finalize_cycle_head(&self, zalsa: &Zalsa, input: Id) {
-        let Some(memo) =
+        let Some(Either::Left(memo)) =
             self.get_memo_from_table_for(zalsa, input, self.memo_ingredient_index(zalsa, input))
         else {
             return;
@@ -417,7 +468,7 @@ where
     }
 
     fn cycle_converged(&self, zalsa: &Zalsa, input: Id) -> bool {
-        let Some(memo) =
+        let Some(Either::Left(memo)) =
             self.get_memo_from_table_for(zalsa, input, self.memo_ingredient_index(zalsa, input))
         else {
             return true;
@@ -534,7 +585,12 @@ where
             let memo =
                 self.get_memo_from_table_for(zalsa, entry.key_index(), memo_ingredient_index);
 
-            if memo.is_some_and(|memo| memo.should_serialize()) {
+            let should_serialize = match memo {
+                Some(Either::Left(memo)) => memo.should_serialize(),
+                Some(Either::Right(memo)) => memo.should_serialize(),
+                None => false,
+            };
+            if should_serialize {
                 return true;
             }
         }

src/function/accumulated.rs

@@ -2,6 +2,7 @@ use crate::accumulator::accumulated_map::{AccumulatedMap, InputAccumulatedValues
 use crate::accumulator::{self};
 use crate::function::{Configuration, IngredientImpl};
 use crate::hash::FxHashSet;
+use crate::table::memo::Either;
 use crate::zalsa::ZalsaDatabase;
 use crate::zalsa_local::QueryOriginRef;
 use crate::{DatabaseKeyIndex, Id};
@@ -100,9 +101,15 @@ where
         let (zalsa, zalsa_local) = db.zalsas();
         // NEXT STEP: stash and refactor `fetch` to return an `&Memo` so we can make this work
         let memo = self.refresh_memo(db, zalsa, zalsa_local, key);
-        (
-            memo.revisions.accumulated(),
-            memo.revisions.accumulated_inputs.load(),
-        )
+        match memo {
+            Either::Left(memo) => (
+                memo.revisions.accumulated(),
+                memo.revisions.accumulated_inputs.load(),
+            ),
+            Either::Right(_) => (
+                Some(const { &AccumulatedMap::EMPTY }),
+                InputAccumulatedValues::Empty,
+            ),
+        }
     }
 }

src/function/delete.rs

@@ -1,13 +1,14 @@
 use std::ptr::NonNull;
 
-use crate::function::memo::Memo;
-use crate::function::Configuration;
+use crate::function::memo::{Memo, NeverChangeMemo};
+use crate::function::{Configuration, EitherMemoNonNull};
+use crate::table::memo::Either;
 
 /// Stores the list of memos that have been deleted so they can be freed
 /// once the next revision starts. See the comment on the field
 /// `deleted_entries` of [`FunctionIngredient`][] for more details.
 pub(super) struct DeletedEntries<C: Configuration> {
-    memos: boxcar::Vec<SharedBox<Memo<'static, C>>>,
+    memos: boxcar::Vec<Either<SharedBox<Memo<'static, C>>, SharedBox<NeverChangeMemo<'static, C>>>>,
 }
 
 #[allow(clippy::undocumented_unsafe_blocks)] // TODO(#697) document safety
@@ -27,12 +28,17 @@ impl<C: Configuration> DeletedEntries<C> {
     /// # Safety
     ///
     /// The memo must be valid and safe to free when the `DeletedEntries` list is cleared or dropped.
-    pub(super) unsafe fn push(&self, memo: NonNull<Memo<'_, C>>) {
+    pub(super) unsafe fn push(&self, memo: EitherMemoNonNull<'_, C>) {
         // Safety: The memo must be valid and safe to free when the `DeletedEntries` list is cleared or dropped.
-        let memo =
-            unsafe { std::mem::transmute::<NonNull<Memo<'_, C>>, NonNull<Memo<'static, C>>>(memo) };
-
-        self.memos.push(SharedBox(memo));
+        let memo = unsafe {
+            std::mem::transmute::<EitherMemoNonNull<'_, C>, EitherMemoNonNull<'static, C>>(memo)
+        };
+
+        let memo = match memo {
+            Either::Left(it) => Either::Left(SharedBox(it)),
+            Either::Right(it) => Either::Right(SharedBox(it)),
+        };
+        self.memos.push(memo);
     }
 
     /// Free all deleted memos, keeping the list available for reuse.