Auto merge of #39109 - michaelwoerister:incr-comp-cache-cleanup, r=nikomatsakis

incr.comp.: Delete orphaned work-products.

The new partitioning scheme uncovered a hole in our incr. comp. cache directory garbage collection. So far, we relied on unneeded work products being deleted during the initial cache invalidation phase. However, we the new scheme, we get object files/work products that only contain code from upstream crates. Sometimes this code is not needed anymore (because all callers have been removed from the source) but because nothing that actually influences the contents of these work products had changed, we never deleted them from disk.

r? @nikomatsakis

Author: bors

src/librustc/dep_graph/graph.rs

@@ -126,6 +126,12 @@ impl DepGraph {
     pub fn work_products(&self) -> Ref<FxHashMap<Arc<WorkProductId>, WorkProduct>> {
         self.data.work_products.borrow()
     }
+
+    /// Access the map of work-products created during the cached run. Only
+    /// used during saving of the dep-graph.
+    pub fn previous_work_products(&self) -> Ref<FxHashMap<Arc<WorkProductId>, WorkProduct>> {
+        self.data.previous_work_products.borrow()
+    }
 }
 
 /// A "work product" is an intermediate result that we save into the

src/librustc_incremental/lib.rs

@@ -57,3 +57,4 @@ pub use persist::save_trans_partition;
 pub use persist::save_work_products;
 pub use persist::in_incr_comp_dir;
 pub use persist::finalize_session_directory;
+pub use persist::delete_workproduct_files;

src/librustc_incremental/persist/load.rs

@@ -18,7 +18,6 @@ use rustc::ty::TyCtxt;
 use rustc_data_structures::fx::{FxHashSet, FxHashMap};
 use rustc_serialize::Decodable as RustcDecodable;
 use rustc_serialize::opaque::Decoder;
-use std::fs;
 use std::path::{Path};
 
 use IncrementalHashesMap;
@@ -29,6 +28,7 @@ use super::dirty_clean;
 use super::hash::*;
 use super::fs::*;
 use super::file_format;
+use super::work_product;
 
 pub type DirtyNodes = FxHashSet<DepNode<DefPathIndex>>;
 
@@ -322,17 +322,7 @@ fn reconcile_work_products<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
 fn delete_dirty_work_product(tcx: TyCtxt,
                              swp: SerializedWorkProduct) {
     debug!("delete_dirty_work_product({:?})", swp);
-    for &(_, ref file_name) in &swp.work_product.saved_files {
-        let path = in_incr_comp_dir_sess(tcx.sess, file_name);
-        match fs::remove_file(&path) {
-            Ok(()) => { }
-            Err(err) => {
-                tcx.sess.warn(
-                    &format!("file-system error deleting outdated file `{}`: {}",
-                             path.display(), err));
-            }
-        }
-    }
+    work_product::delete_workproduct_files(tcx.sess, &swp.work_product);
 }
 
 fn load_prev_metadata_hashes(tcx: TyCtxt,

src/librustc_incremental/persist/mod.rs

@@ -29,3 +29,4 @@ pub use self::load::load_dep_graph;
 pub use self::save::save_dep_graph;
 pub use self::save::save_work_products;
 pub use self::work_product::save_trans_partition;
+pub use self::work_product::delete_workproduct_files;

src/librustc_incremental/persist/save.rs

@@ -30,6 +30,7 @@ use super::preds::*;
 use super::fs::*;
 use super::dirty_clean;
 use super::file_format;
+use super::work_product;
 use calculate_svh::IchHasher;
 
 pub fn save_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
@@ -84,6 +85,31 @@ pub fn save_work_products(sess: &Session) {
     let _ignore = sess.dep_graph.in_ignore();
     let path = work_products_path(sess);
     save_in(sess, path, |e| encode_work_products(sess, e));
+
+    // We also need to clean out old work-products, as not all of them are
+    // deleted during invalidation. Some object files don't change their
+    // content, they are just not needed anymore.
+    let new_work_products = sess.dep_graph.work_products();
+    let previous_work_products = sess.dep_graph.previous_work_products();
+
+    for (id, wp) in previous_work_products.iter() {
+        if !new_work_products.contains_key(id) {
+            work_product::delete_workproduct_files(sess, wp);
+            debug_assert!(wp.saved_files.iter().all(|&(_, ref file_name)| {
+                !in_incr_comp_dir_sess(sess, file_name).exists()
+            }));
+        }
+    }
+
+    // Check that we did not delete one of the current work-products:
+    debug_assert!({
+        new_work_products.iter()
+                         .flat_map(|(_, wp)| wp.saved_files
+                                               .iter()
+                                               .map(|&(_, ref name)| name))
+                         .map(|name| in_incr_comp_dir_sess(sess, name))
+                         .all(|path| path.exists())
+    });
 }
 
 fn save_in<F>(sess: &Session, path_buf: PathBuf, encode: F)

src/librustc_incremental/persist/work_product.rs

@@ -17,6 +17,7 @@ use rustc::session::config::OutputType;
 use rustc::util::fs::link_or_copy;
 use std::path::PathBuf;
 use std::sync::Arc;
+use std::fs as std_fs;
 
 pub fn save_trans_partition(sess: &Session,
                             cgu_name: &str,
@@ -61,3 +62,17 @@ pub fn save_trans_partition(sess: &Session,
 
     sess.dep_graph.insert_work_product(&work_product_id, work_product);
 }
+
+pub fn delete_workproduct_files(sess: &Session, work_product: &WorkProduct) {
+    for &(_, ref file_name) in &work_product.saved_files {
+        let path = in_incr_comp_dir_sess(sess, file_name);
+        match std_fs::remove_file(&path) {
+            Ok(()) => { }
+            Err(err) => {
+                sess.warn(
+                    &format!("file-system error deleting outdated file `{}`: {}",
+                             path.display(), err));
+            }
+        }
+    }
+}