added precompilation for lowering

Author: TomerStarkware

Cargo.lock

@@ -126,6 +126,7 @@ schemars = { version = "0.8.21", features = ["preserve_order"] }
 semver = { version = "1.0.25", features = ["serde"] }
 serde = { version = "1.0.217", default-features = false, features = ["derive"] }
 serde_json = "1.0.138"
+bincode = "1.3.3"
 sha2 = "0.10.8"
 sha3 = "0.10.8"
 smol_str = { version = "0.3.2", features = ["serde"] }

Cargo.toml

@@ -19,3 +19,4 @@ cairo-lang-lowering = { path = "../../cairo-lang-lowering", version = "~2.9.2" }
 cairo-lang-starknet = { path = "../../cairo-lang-starknet", version = "~2.9.2" }
 cairo-lang-semantic = { path = "../../cairo-lang-semantic", version = "~2.9.2" }
 cairo-lang-utils = { path = "../../cairo-lang-utils", version = "~2.9.2" }
+bincode.workspace = true

crates/bin/get-lowering/Cargo.toml

@@ -85,7 +85,7 @@ pub fn update_crate_root(
     let (crate_id, crate_settings) = get_crate_id_and_settings(db, crate_identifier, config);
     db.set_crate_config(
         crate_id,
-        Some(CrateConfiguration { root, settings: crate_settings.clone() }),
+        Some(CrateConfiguration { root, settings: crate_settings.clone(), precompute_file: None }),
     );
 }
 

crates/cairo-lang-compiler/src/project.rs

@@ -11,7 +11,7 @@ use crate::db::DefsGroup;
 
 /// A stable location of a real, concrete syntax.
 #[derive(Copy, Clone, Debug, Eq, Hash, PartialEq)]
-pub struct StableLocation(SyntaxStablePtrId);
+pub struct StableLocation(pub SyntaxStablePtrId);
 impl StableLocation {
     pub fn new(stable_ptr: SyntaxStablePtrId) -> Self {
         Self(stable_ptr)

crates/cairo-lang-defs/src/diagnostic_utils.rs

@@ -13,8 +13,8 @@ use smol_str::{SmolStr, ToSmolStr};
 use crate::cfg::CfgSet;
 use crate::flag::Flag;
 use crate::ids::{
-    CodeMapping, CodeOrigin, CrateId, CrateLongId, Directory, FileId, FileLongId, FlagId,
-    FlagLongId, VirtualFile,
+    BlobId, BlobLongId, CodeMapping, CodeOrigin, CrateId, CrateLongId, Directory, FileId,
+    FileLongId, FlagId, FlagLongId, VirtualFile,
 };
 use crate::span::{FileSummary, TextOffset, TextSpan, TextWidth};
 
@@ -50,11 +50,12 @@ pub struct CrateConfiguration {
     /// The root directory of the crate.
     pub root: Directory,
     pub settings: CrateSettings,
+    pub precompute_file: Option<BlobId>,
 }
 impl CrateConfiguration {
     /// Returns a new configuration.
     pub fn default_for_root(root: Directory) -> Self {
-        Self { root, settings: CrateSettings::default() }
+        Self { root, settings: CrateSettings::default(), precompute_file: None }
     }
 }
 
@@ -182,6 +183,8 @@ pub trait FilesGroup: ExternalFiles {
     #[salsa::interned]
     fn intern_file(&self, file: FileLongId) -> FileId;
     #[salsa::interned]
+    fn intern_blob(&self, blob: BlobLongId) -> BlobId;
+    #[salsa::interned]
     fn intern_flag(&self, flag: FlagLongId) -> FlagId;
 
     /// Main input of the project. Lists all the crates configurations.
@@ -215,6 +218,8 @@ pub trait FilesGroup: ExternalFiles {
     fn file_content(&self, file_id: FileId) -> Option<Arc<str>>;
     fn file_summary(&self, file_id: FileId) -> Option<Arc<FileSummary>>;
 
+    /// Query for the blob content.
+    fn blob_content(&self, blob_id: BlobId) -> Option<Arc<[u8]>>;
     /// Query to get a compilation flag by its ID.
     fn get_flag(&self, id: FlagId) -> Option<Arc<Flag>>;
 }
@@ -244,6 +249,7 @@ pub fn init_dev_corelib(db: &mut (dyn FilesGroup + 'static), core_lib_dir: PathB
                     coupons: true,
                 },
             },
+            precompute_file: None,
         }),
     );
 }
@@ -302,13 +308,17 @@ fn crates(db: &dyn FilesGroup) -> Vec<CrateId> {
 fn crate_config(db: &dyn FilesGroup, crt: CrateId) -> Option<CrateConfiguration> {
     match crt.lookup_intern(db) {
         CrateLongId::Real { .. } => db.crate_configs().get(&crt).cloned(),
-        CrateLongId::Virtual { name: _, file_id, settings } => Some(CrateConfiguration {
-            root: Directory::Virtual {
-                files: BTreeMap::from([("lib.cairo".into(), file_id)]),
-                dirs: Default::default(),
-            },
-            settings: toml::from_str(&settings).expect("Failed to parse virtual crate settings."),
-        }),
+        CrateLongId::Virtual { name: _, file_id, settings, precompute_file } => {
+            Some(CrateConfiguration {
+                root: Directory::Virtual {
+                    files: BTreeMap::from([("lib.cairo".into(), file_id)]),
+                    dirs: Default::default(),
+                },
+                settings: toml::from_str(&settings)
+                    .expect("Failed to parse virtual crate settings."),
+                precompute_file,
+            })
+        }
     }
 }
 
@@ -348,6 +358,22 @@ fn get_flag(db: &dyn FilesGroup, id: FlagId) -> Option<Arc<Flag>> {
     db.flags().get(&id).cloned()
 }
 
+fn blob_content(db: &dyn FilesGroup, blob: BlobId) -> Option<Arc<[u8]>> {
+    match blob.lookup_intern(db) {
+        BlobLongId::OnDisk(path) => {
+            // This does not result in performance cost due to OS caching and the fact that salsa
+            // will re-execute only this single query if the file content did not change.
+            db.salsa_runtime().report_synthetic_read(Durability::LOW);
+
+            match fs::read(path) {
+                Ok(content) => Some(content.into()),
+                Err(_) => None,
+            }
+        }
+        BlobLongId::Virtual(content) => Some(content),
+    }
+}
+
 /// Returns the location of the originating user code.
 pub fn get_originating_location(
     db: &dyn FilesGroup,

crates/cairo-lang-filesystem/src/db.rs

@@ -4,6 +4,7 @@ use std::sync::Arc;
 
 use cairo_lang_utils::{Intern, LookupIntern, define_short_id};
 use path_clean::PathClean;
+use serde::{Deserialize, Serialize};
 use smol_str::SmolStr;
 
 use crate::db::{CORELIB_CRATE_NAME, FilesGroup};
@@ -17,7 +18,7 @@ pub enum CrateLongId {
     /// A crate that appears in crate_roots(), and on the filesystem.
     Real { name: SmolStr, discriminator: Option<SmolStr> },
     /// A virtual crate, not a part of the crate_roots(). Used mainly for tests.
-    Virtual { name: SmolStr, file_id: FileId, settings: String },
+    Virtual { name: SmolStr, file_id: FileId, settings: String, precompute_file: Option<BlobId> },
 }
 impl CrateLongId {
     pub fn name(&self) -> SmolStr {
@@ -78,14 +79,14 @@ pub enum FileLongId {
     External(salsa::InternId),
 }
 /// Whether the file holds syntax for a module or for an expression.
-#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)]
 pub enum FileKind {
     Module,
     Expr,
 }
 
 /// A mapping for a code rewrite.
-#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)]
 pub struct CodeMapping {
     pub span: TextSpan,
     pub origin: CodeOrigin,
@@ -108,7 +109,7 @@ impl CodeMapping {
 }
 
 /// The origin of a code mapping.
-#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)]
 pub enum CodeOrigin {
     /// The origin is a copied node staring at the given offset.
     Start(TextOffset),
@@ -214,3 +215,17 @@ impl Directory {
         }
     }
 }
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+pub enum BlobLongId {
+    OnDisk(PathBuf),
+    Virtual(Arc<[u8]>),
+}
+
+define_short_id!(BlobId, BlobLongId, FilesGroup, lookup_intern_blob, intern_blob);
+
+impl BlobId {
+    pub fn new(db: &dyn FilesGroup, path: PathBuf) -> BlobId {
+        BlobLongId::OnDisk(path.clean()).intern(db)
+    }
+}

crates/cairo-lang-filesystem/src/ids.rs

@@ -23,6 +23,9 @@ num-bigint = { workspace = true, default-features = true }
 num-integer = { workspace = true, default-features = true }
 num-traits = { workspace = true, default-features = true }
 salsa.workspace = true
+bincode.workspace = true
+serde = { workspace = true, default-features = true }
+smol_str.workspace = true
 
 [dev-dependencies]
 cairo-lang-plugins = { path = "../cairo-lang-plugins" }

crates/cairo-lang-lowering/Cargo.toml

@@ -1,12 +1,14 @@
 use std::sync::Arc;
 
+use cairo_lang_debug::DebugWithDb;
 use cairo_lang_defs as defs;
 use cairo_lang_defs::ids::{LanguageElementId, ModuleId, ModuleItemId, NamedLanguageElementLongId};
 use cairo_lang_diagnostics::{Diagnostics, DiagnosticsBuilder, Maybe};
 use cairo_lang_filesystem::ids::FileId;
 use cairo_lang_semantic::db::SemanticGroup;
 use cairo_lang_semantic::items::enm::SemanticEnumEx;
 use cairo_lang_semantic::{self as semantic, ConcreteTypeId, TypeId, TypeLongId, corelib};
+use cairo_lang_utils::ordered_hash_map::OrderedHashMap;
 use cairo_lang_utils::ordered_hash_set::OrderedHashSet;
 use cairo_lang_utils::unordered_hash_map::UnorderedHashMap;
 use cairo_lang_utils::unordered_hash_set::UnorderedHashSet;
@@ -28,6 +30,10 @@ use crate::optimizations::config::OptimizationConfig;
 use crate::optimizations::scrub_units::scrub_units;
 use crate::optimizations::strategy::{OptimizationStrategy, OptimizationStrategyId};
 use crate::panic::lower_panics;
+use crate::precompute::{
+    DeSerializationContext, DefsFunctionWithBodyIdSerializable, MultiLoweringSerializable,
+    SemanticSerializationLookups, SerializationLookups,
+};
 use crate::utils::InliningStrategy;
 use crate::{
     BlockId, DependencyType, FlatBlockEnd, FlatLowered, Location, MatchInfo, Statement, ids,
@@ -62,6 +68,11 @@ pub trait LoweringGroup: SemanticGroup + Upcast<dyn SemanticGroup> {
         function_id: defs::ids::FunctionWithBodyId,
     ) -> Maybe<Arc<MultiLowering>>;
 
+    fn load_serielized_multi_lowerings(
+        &self,
+        file_id: cairo_lang_filesystem::ids::CrateId,
+    ) -> Option<Arc<OrderedHashMap<defs::ids::FunctionWithBodyId, MultiLowering>>>;
+
     /// Computes the lowered representation of a function with a body before borrow checking.
     fn priv_function_with_body_lowering(
         &self,
@@ -382,9 +393,46 @@ fn priv_function_with_body_multi_lowering(
     db: &dyn LoweringGroup,
     function_id: defs::ids::FunctionWithBodyId,
 ) -> Maybe<Arc<MultiLowering>> {
+    let crate_id = function_id.module_file_id(db.upcast()).0.owning_crate(db.upcast());
+    if let Some(map) = db.load_serielized_multi_lowerings(crate_id) {
+        if let Some(multi_lowering) = map.get(&function_id) {
+            return Ok(Arc::new(multi_lowering.clone()));
+        } else {
+            panic!("function not found in precomputed lowering {:?}", function_id.debug(db));
+        }
+    };
+
     let multi_lowering = lower_semantic_function(db.upcast(), function_id)?;
     Ok(Arc::new(multi_lowering))
 }
+fn load_serielized_multi_lowerings(
+    db: &dyn LoweringGroup,
+    self_crate_id: cairo_lang_filesystem::ids::CrateId,
+) -> Option<Arc<OrderedHashMap<defs::ids::FunctionWithBodyId, MultiLowering>>> {
+    let blob_id = db.crate_config(self_crate_id)?.precompute_file?;
+    let Some(content) = db.blob_content(blob_id) else {
+        return Default::default();
+    };
+    let (lookups, semantic_lookups, lowerings): (
+        SerializationLookups,
+        SemanticSerializationLookups,
+        Vec<(DefsFunctionWithBodyIdSerializable, MultiLoweringSerializable)>,
+    ) = bincode::deserialize(&content).unwrap_or_default();
+    let mut ctx = DeSerializationContext::new(db, lookups, semantic_lookups, self_crate_id);
+
+    Some(
+        lowerings
+            .into_iter()
+            .map(|(function_id, lowering)| {
+                let function_id = function_id.embed(&mut ctx.semantic_ctx);
+
+                let lowering = lowering.embed(&mut ctx);
+                (function_id, lowering)
+            })
+            .collect::<OrderedHashMap<_, _>>()
+            .into(),
+    )
+}
 
 // * Borrow checking.
 fn priv_function_with_body_lowering(

crates/cairo-lang-lowering/src/db.rs

@@ -16,6 +16,7 @@ pub mod lower;
 pub mod objects;
 pub mod optimizations;
 pub mod panic;
+pub mod precompute;
 pub mod reorganize_blocks;
 pub mod scc;
 pub mod utils;

crates/cairo-lang-lowering/src/lib.rs

@@ -381,7 +381,7 @@ pub struct StatementStructDestructure {
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct StatementSnapshot {
     pub input: VarUsage,
-    outputs: [VariableId; 2],
+    pub outputs: [VariableId; 2],
 }
 impl StatementSnapshot {
     pub fn new(input: VarUsage, output_original: VariableId, output_snapshot: VariableId) -> Self {