Pass --emit=invocation-specific for crate builds

This avoids copying shared files for crates other than the empty
library. This
a) Allows removing `copy_doc_dir`, which allows crates to add their own custom CSS (with `--theme` or otherwise), and
b) Avoids unnecessary disk IO.

Here are the files generated for a build of regex 1.4.1:

```
cratesfyi=# select path from files where path like '%regex/1.4.1%' and path not like '%regex/1.4.1/%/%';
                                 path
-----------------------------------------------------------------------
 sources/regex/1.4.1/PERFORMANCE.md
 rustdoc/regex/1.4.1/.lock
 sources/regex/1.4.1/Cargo.toml
 sources/regex/1.4.1/.cargo_vcs_info.json
 sources/regex/1.4.1/LICENSE-APACHE
 rustdoc/regex/1.4.1/source-files-20210402-1.53.0-nightly-138fd56cf.js
 rustdoc/regex/1.4.1/settings.html
 rustdoc/regex/1.4.1/search-index-20210402-1.53.0-nightly-138fd56cf.js
 rustdoc/regex/1.4.1/crates-20210402-1.53.0-nightly-138fd56cf.js
 sources/regex/1.4.1/CHANGELOG.md
 sources/regex/1.4.1/LICENSE-MIT
 sources/regex/1.4.1/rustfmt.toml
 sources/regex/1.4.1/HACKING.md
 sources/regex/1.4.1/test
 sources/regex/1.4.1/Cargo.toml.orig
 sources/regex/1.4.1/UNICODE.md
 sources/regex/1.4.1/.gitignore
 sources/regex/1.4.1/Cargo.lock
 sources/regex/1.4.1/README.md
(19 rows)
```

Note in particular that search-index.js and crates.js are included, but
not shared files like `storage.js` or `rustdoc.css`.

Author: jyn514

src/docbuilder/rustwide_builder.rs

@@ -7,7 +7,7 @@ use crate::docbuilder::{crates::crates_from_path, Limits};
 use crate::error::Result;
 use crate::index::api::ReleaseData;
 use crate::storage::CompressionAlgorithms;
-use crate::utils::{copy_doc_dir, parse_rustc_version, CargoMetadata, GithubUpdater};
+use crate::utils::{copy_dir_all, parse_rustc_version, CargoMetadata, GithubUpdater};
 use crate::{db::blacklist::is_blacklisted, utils::MetadataPackage};
 use crate::{Config, Context, Index, Metrics, Storage};
 use docsrs_metadata::{Metadata, DEFAULT_TARGETS, HOST_TARGET};
@@ -187,17 +187,7 @@ impl RustwideBuilder {
             .run(|build| {
                 let metadata = Metadata::from_crate_root(&build.host_source_dir())?;
 
-                let rustdoc_flags = vec![
-                    "--emit=unversioned-shared-resources,toolchain-shared-resources".to_string(),
-                ];
-                let res = self.execute_build(
-                    HOST_TARGET,
-                    true,
-                    build,
-                    &limits,
-                    &metadata,
-                    rustdoc_flags,
-                )?;
+                let res = self.execute_build(HOST_TARGET, true, build, &limits, &metadata, true)?;
                 if !res.result.successful {
                     failure::bail!("failed to build dummy crate for {}", self.rustc_version);
                 }
@@ -207,7 +197,7 @@ impl RustwideBuilder {
                 let dest = tempfile::Builder::new()
                     .prefix("essential-files")
                     .tempdir()?;
-                crate::utils::copy_dir_all(source, &dest, |_| true)?;
+                copy_dir_all(source, &dest)?;
                 add_path_into_database(&self.storage, "", &dest)?;
                 conn.query(
                     "INSERT INTO config (name, value) VALUES ('rustc_version', $1) \
@@ -317,7 +307,7 @@ impl RustwideBuilder {
 
                 // Perform an initial build
                 let res =
-                    self.execute_build(default_target, true, &build, &limits, &metadata, vec![])?;
+                    self.execute_build(default_target, true, &build, &limits, &metadata, false)?;
                 if res.result.successful {
                     if let Some(name) = res.cargo_metadata.root().library_name() {
                         let host_target = build.host_target_dir();
@@ -423,7 +413,7 @@ impl RustwideBuilder {
         successful_targets: &mut Vec<String>,
         metadata: &Metadata,
     ) -> Result<()> {
-        let target_res = self.execute_build(target, false, build, limits, metadata, Vec::new())?;
+        let target_res = self.execute_build(target, false, build, limits, metadata, false)?;
         if target_res.result.successful {
             // Cargo is not giving any error and not generating documentation of some crates
             // when we use a target compile options. Check documentation exists before
@@ -499,11 +489,17 @@ impl RustwideBuilder {
         build: &Build,
         limits: &Limits,
         metadata: &Metadata,
-        mut rustdoc_flags: Vec<String>,
+        create_essential_files: bool,
     ) -> Result<FullBuildResult> {
         let cargo_metadata =
             CargoMetadata::load(&self.workspace, &self.toolchain, &build.host_source_dir())?;
 
+        let mut rustdoc_flags = vec![if create_essential_files {
+            "--emit=unversioned-shared-resources,toolchain-shared-resources"
+        } else {
+            "--emit=invocation-specific"
+        }
+        .to_string()];
         rustdoc_flags.extend(vec![
             "--resource-suffix".to_string(),
             format!("-{}", parse_rustc_version(&self.rustc_version)?),
@@ -620,7 +616,7 @@ impl RustwideBuilder {
         }
 
         info!("{} {}", source.display(), dest.display());
-        copy_doc_dir(source, dest)
+        copy_dir_all(source, dest).map_err(Into::into)
     }
 
     fn upload_docs(

src/utils/copy.rs

@@ -1,41 +1,17 @@
-use std::ffi::OsStr;
 use std::fs;
 use std::io;
 use std::path::Path;
 
-use crate::error::Result;
-use regex::Regex;
-
-/// Copies documentation from a crate's target directory to destination.
-///
-/// Target directory must have doc directory.
-///
-/// This function is designed to avoid file duplications.
-pub(crate) fn copy_doc_dir(source: impl AsRef<Path>, destination: impl AsRef<Path>) -> Result<()> {
-    // Avoid copying common files
-    let dup_regex = Regex::new(
-        r"(\.lock|\.txt|\.woff|\.svg|\.css|main-.*\.css|main-.*\.js|normalize-.*\.js|rustdoc-.*\.css|storage-.*\.js|theme-.*\.js)$")
-        .unwrap();
-
-    copy_dir_all(source, destination, |filename| {
-        dup_regex.is_match(filename.to_str().unwrap())
-    })
-    .map_err(Into::into)
-}
-
-pub(crate) fn copy_dir_all(
-    src: impl AsRef<Path>,
-    dst: impl AsRef<Path>,
-    should_copy: impl Copy + Fn(&OsStr) -> bool,
-) -> io::Result<()> {
+/// cp -r src dst
+pub(crate) fn copy_dir_all(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> io::Result<()> {
     let dst = dst.as_ref();
     fs::create_dir_all(dst)?;
     for entry in fs::read_dir(src)? {
         let entry = entry?;
         let filename = entry.file_name();
         if entry.file_type()?.is_dir() {
-            copy_dir_all(entry.path(), dst.join(filename), should_copy)?;
-        } else if should_copy(&filename) {
+            copy_dir_all(entry.path(), dst.join(filename))?;
+        } else {
             fs::copy(entry.path(), dst.join(filename))?;
         }
     }
@@ -62,21 +38,11 @@ mod test {
         fs::create_dir(doc.join("inner")).unwrap();
 
         fs::write(doc.join("index.html"), "<html>spooky</html>").unwrap();
-        fs::write(doc.join("index.txt"), "spooky").unwrap();
         fs::write(doc.join("inner").join("index.html"), "<html>spooky</html>").unwrap();
-        fs::write(doc.join("inner").join("index.txt"), "spooky").unwrap();
-        fs::write(doc.join("inner").join("important.svg"), "<svg></svg>").unwrap();
 
         // lets try to copy a src directory to tempdir
-        copy_doc_dir(source.path().join("doc"), destination.path()).unwrap();
+        copy_dir_all(source.path().join("doc"), destination.path()).unwrap();
         assert!(destination.path().join("index.html").exists());
-        assert!(!destination.path().join("index.txt").exists());
         assert!(destination.path().join("inner").join("index.html").exists());
-        assert!(!destination.path().join("inner").join("index.txt").exists());
-        assert!(!destination
-            .path()
-            .join("inner")
-            .join("important.svg")
-            .exists());
     }
 }

src/utils/mod.rs

@@ -1,7 +1,7 @@
 //! Various utilities for docs.rs
 
 pub(crate) use self::cargo_metadata::{CargoMetadata, Package as MetadataPackage};
-pub(crate) use self::copy::{copy_dir_all, copy_doc_dir};
+pub(crate) use self::copy::copy_dir_all;
 pub use self::daemon::start_daemon;
 pub use self::github_updater::GithubUpdater;
 pub(crate) use self::html::rewrite_lol;