Rewrite from proc-macro to build.rs (#15)

* Implemented build.rs flow, for single directory
* Improve logging, require environment variable
* Updated README
* Add force-embed feature, support multiple directories (WIP)
* Macro and/or build script
* Always use OUT_DIR
* Fix windows paths

---------

Co-authored-by: cikzh <[email protected]>

Author: marlonbaeten

.gitignore

@@ -1,11 +1,4 @@
 .vscode/
 target/
-/Cargo.lock
+Cargo.lock
 
-
-# Added by cargo
-#
-# already existing elements were commented out
-
-/target
-#/Cargo.lock

Cargo.toml

@@ -1,9 +1,10 @@
 [workspace]
-members = ["memory-serve", "memory-serve-macros", "examples/test"]
+members = ["memory-serve", "memory-serve-macros", "memory-serve-core"]
+exclude = ["example"]
 resolver = "2"
 
 [workspace.package]
-version = "0.6.0"
+version = "1.0.0-beta.0"
 edition = "2021"
 license = "Apache-2.0 OR MIT"
 repository = "https://github.com/tweedegolf/memory-serve"

README.md

@@ -33,24 +33,48 @@ memory-serve is designed to work with [axum](https://github.com/tokio-rs/axum)
 
 ## Usage
 
-Provide a relative path to the directory containing your static assets
-to the [`load_assets!`] macro. This macro creates a data structure intended to
-be consumed by [`MemoryServe::new`]. Calling [`MemoryServe::into_router()`] on
-the resulting instance produces a axum
+There are two mechanisms to include assets at compile time.
+
+1. Specify the path using a enviroment variable `ASSET_PATH` and call: `MemoryServe::from_env()` (best-practice)
+2. Call the `load_assets!` macro, and pass this to the constructor: `MemoryServe::new(load_assets!("/foo/bar"))`
+
+The environment variable is handled by a build script and instructs cargo to re-evaluate when an asset in the directory changes.
+The output of the macro might be cached between build.
+
+Both options try to be smart in resolving absolute and relative paths.
+
+When an instance of `MemoryServe` is created, we can bind these to your axum instance.
+Calling [`MemoryServe::into_router()`] on the `MemoryServe` instance produces an axum
 [`Router`](https://docs.rs/axum/latest/axum/routing/struct.Router.html) that
 can either be merged in another `Router` or used directly in a server by
 calling [`Router::into_make_service()`](https://docs.rs/axum/latest/axum/routing/struct.Router.html#method.into_make_service).
 
+### Named directories
+
+Multiple directories can be included using different environment variables, all prefixed by `ASSET_PATH_`.
+For example: if you specify `ASSET_PATH_FOO` and `ASSET_PATH_BAR` the memory serve instances can be loaded
+using `MemoryServe::from_env_name("FOO")` and `MemoryServe::from_env_name("BAR")` respectively.
+
+### Features
+
+Use the `force-embed` feature flag to always include assets in the binary - also in debug builds.
+
+### Environment variables
+
+Use `MEMORY_SERVE_ROOT` to specify a root directory for relative paths provided to the `load_assets!` macro (or th `ASSET_PATH` variable).
+
+Uee `MEMORY_SERVE_QUIET=1` to not print log messages at compile time.
+
 ## Example
 
 ```rust,no_run
 use axum::{response::Html, routing::get, Router};
-use memory_serve::{load_assets, MemoryServe};
+use memory_serve::{MemoryServe, load_assets};
 use std::net::SocketAddr;
 
 #[tokio::main]
 async fn main() {
-    let memory_router = MemoryServe::new(load_assets!("static"))
+    let memory_router = MemoryServe::new(load_assets!("../static"))
         .index_file(Some("/index.html"))
         .into_router();
 

example/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "memory-serve-test"
+edition = "2021"
+
+[dependencies]
+memory-serve = { path = "../memory-serve" }
+axum = "0.7"
+tokio = { version = "1.0", features = ["full"] }
+tracing-subscriber = "0.3"
+tracing = "0.1"

examples/test/src/main.rs renamed to example/src/main.rs

@@ -1,13 +1,15 @@
 use axum::{response::Html, routing::get, Router};
-use memory_serve::{load_assets, MemoryServe};
+use memory_serve::{MemoryServe, load_assets};
 use std::net::SocketAddr;
-use tracing::info;
+use tracing::{info, Level};
 
 #[tokio::main]
 async fn main() {
-    tracing_subscriber::fmt().init();
+    tracing_subscriber::fmt()
+        .with_max_level(Level::TRACE)
+        .init();
 
-    let memory_router = MemoryServe::new(load_assets!("../../static"))
+    let memory_router = MemoryServe::new(load_assets!("../static"))
         .index_file(Some("/index.html"))
         .into_router();
 

examples/test/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "memory-serve-core"
+description = "Shared code for memory-serve and memory-serve-macros"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+publish.workspace = true
+
+[dependencies]
+sha256 = "1.4"
+brotli = "7.0"
+mime_guess = "2.0"
+walkdir = "2"

memory-serve-core/Cargo.toml

@@ -0,0 +1 @@
+../README.md

memory-serve-core/README.md

@@ -1,13 +1,12 @@
-use syn::LitByteStr;
+use std::path::PathBuf;
 
 /// Internal data structure
-pub(crate) struct Asset {
-    pub(crate) route: String,
-    pub(crate) path: String,
-    pub(crate) etag: String,
-    pub(crate) content_type: String,
-    pub(crate) bytes: LitByteStr,
-    pub(crate) brotli_bytes: LitByteStr,
+pub struct Asset {
+    pub route: String,
+    pub path: PathBuf,
+    pub etag: String,
+    pub content_type: String,
+    pub compressed_bytes: Option<Vec<u8>>,
 }
 
 impl PartialEq for Asset {

memory-serve-macros/src/asset.rs renamed to memory-serve-core/src/asset.rs

@@ -0,0 +1,62 @@
+use std::path::{Path, PathBuf};
+
+use crate::{asset::Asset, list::list_assets};
+
+/// Generate code with metadata and contents for the assets
+pub fn assets_to_code(asset_dir: &str, path: &Path, embed: bool, log: fn(&str)) -> String {
+    let out_dir: String = std::env::var("OUT_DIR").expect("OUT_DIR environment variable not set.");
+    let out_dir = PathBuf::from(&out_dir);
+
+    log(&format!("Loading static assets from {asset_dir}"));
+
+    if embed {
+        log("Embedding assets into binary");
+    } else {
+        log("Not embedding assets into binary, assets will load dynamically");
+    }
+
+    let assets = list_assets(path, embed, log);
+
+    // using a string is faster than using quote ;)
+    let mut code = "&[".to_string();
+
+    for asset in assets {
+        let Asset {
+            route,
+            path,
+            etag,
+            content_type,
+            compressed_bytes,
+        } = asset;
+
+        let bytes = if !embed {
+            "None".to_string()
+        } else if let Some(compressed_bytes) = &compressed_bytes {
+            let file_name = path.file_name().expect("Unable to get file name.");
+            let file_path = Path::new(&out_dir).join(file_name);
+            std::fs::write(&file_path, compressed_bytes).expect("Unable to write file to out dir.");
+
+            format!("Some(include_bytes!(r\"{}\"))", file_path.to_string_lossy())
+        } else {
+            format!("Some(include_bytes!(r\"{}\"))", path.to_string_lossy())
+        };
+
+        let is_compressed = compressed_bytes.is_some();
+
+        code.push_str(&format!(
+            "
+            memory_serve::Asset {{
+                route: r\"{route}\",
+                path: r{path:?},
+                content_type: \"{content_type}\",
+                etag: \"{etag}\",
+                bytes: {bytes},
+                is_compressed: {is_compressed},
+            }},"
+        ));
+    }
+
+    code.push(']');
+
+    code
+}

memory-serve-core/src/code.rs

@@ -0,0 +1,20 @@
+mod asset;
+mod code;
+mod list;
+mod util;
+
+pub use asset::Asset;
+pub use code::assets_to_code;
+
+/// File mime types that can possibly be compressed
+pub const COMPRESS_TYPES: &[&str] = &[
+    "text/html",
+    "text/css",
+    "application/json",
+    "text/javascript",
+    "application/javascript",
+    "application/xml",
+    "text/xml",
+    "image/svg+xml",
+    "application/wasm",
+];

memory-serve-core/src/lib.rs

@@ -0,0 +1,112 @@
+use std::path::Path;
+
+use walkdir::WalkDir;
+
+use crate::{
+    asset::Asset,
+    util::{compress_brotli, path_to_content_type, path_to_route},
+    COMPRESS_TYPES,
+};
+
+/// List all assets in the given directory (recursively) and return a list of assets with metadata
+pub fn list_assets(base_path: &Path, embed: bool, log: fn(&str)) -> Vec<Asset> {
+    let mut assets: Vec<Asset> = WalkDir::new(base_path)
+        .into_iter()
+        .filter_map(|entry| entry.ok())
+        .filter_map(|entry| {
+            let path = entry.path().to_owned();
+            let route = path_to_route(base_path, entry.path());
+
+            let Ok(metadata) = entry.metadata() else {
+                log(&format!(
+                    "skipping file {route}, could not get file metadata"
+                ));
+                return None;
+            };
+
+            // skip directories
+            if !metadata.is_file() {
+                return None;
+            };
+
+            // skip empty
+            if metadata.len() == 0 {
+                log(&format!("skipping file {route}: file empty"));
+                return None;
+            }
+
+            let Some(content_type) = path_to_content_type(entry.path()) else {
+                log(&format!(
+                    "skipping file {route}, could not determine file extension"
+                ));
+                return None;
+            };
+
+            // do not load assets into the binary in debug / development mode
+            if !embed {
+                log(&format!("including {route} (dynamically)"));
+
+                return Some(Asset {
+                    route,
+                    path: path.to_owned(),
+                    content_type,
+                    etag: Default::default(),
+                    compressed_bytes: None,
+                });
+            }
+
+            let Ok(bytes) = std::fs::read(entry.path()) else {
+                log(&format!("skipping file {route}: file is not readable"));
+                return None;
+            };
+
+            let etag: String = sha256::digest(&bytes);
+            let original_size = bytes.len();
+            let is_compress_type = COMPRESS_TYPES.contains(&content_type.as_str());
+            let brotli_bytes = if is_compress_type {
+                compress_brotli(&bytes)
+            } else {
+                None
+            };
+
+            let mut asset = Asset {
+                route: route.clone(),
+                path: path.to_owned(),
+                content_type,
+                etag,
+                compressed_bytes: None,
+            };
+
+            if is_compress_type {
+                match brotli_bytes {
+                    Some(brotli_bytes) if brotli_bytes.len() >= original_size => {
+                        log(&format!(
+                            "including {route} {original_size} bytes (compression unnecessary)"
+                        ));
+                    }
+                    Some(brotli_bytes) => {
+                        log(&format!(
+                            "including {route} {original_size} -> {} bytes (compressed)",
+                            brotli_bytes.len()
+                        ));
+
+                        asset.compressed_bytes = Some(brotli_bytes);
+                    }
+                    None => {
+                        log(&format!(
+                            "including {route} {original_size} bytes (compression failed)"
+                        ));
+                    }
+                }
+            } else {
+                log(&format!("including {route} {original_size} bytes"));
+            }
+
+            Some(asset)
+        })
+        .collect();
+
+    assets.sort();
+
+    assets
+}