feat(mater-cli): add support for stdin input (#761)

mater/cli/Cargo.toml

@@ -17,7 +17,7 @@ mater.workspace = true
 clap = { workspace = true, features = ["derive"] }
 futures = { workspace = true }
 thiserror.workspace = true
-tokio = { workspace = true, features = ["fs", "macros", "rt-multi-thread"] }
+tokio = { workspace = true, features = ["fs", "io-std", "macros", "rt-multi-thread"] }
 
 [dev-dependencies]
 anyhow.workspace = true

mater/cli/src/convert.rs

@@ -9,15 +9,18 @@ pub(crate) async fn convert_file_to_car(
     output_path: &PathBuf,
     overwrite: bool,
 ) -> Result<Cid, Error> {
-    let source_file = File::open(input_path).await?;
     let output_file = if overwrite {
         File::create(output_path).await
     } else {
         File::create_new(output_path).await
     }?;
-    let cid = create_filestore(source_file, output_file, Config::default()).await?;
 
-    Ok(cid)
+    if input_path.as_os_str() == "-" {
+        create_filestore(tokio::io::stdin(), output_file, Config::default()).await
+    } else {
+        let source_file = File::open(input_path).await?;
+        create_filestore(source_file, output_file, Config::default()).await
+    }
 }
 
 /// Tests for file conversion.

mater/cli/src/extract.rs

@@ -1,7 +1,13 @@
 use std::path::PathBuf;
 
 use mater::{CarExtractor, Error};
-use tokio::fs::File;
+use tokio::{
+    fs::File,
+    io::{AsyncReadExt, BufReader},
+};
+
+/// Arbitrary value, it's just so the buffer doesn't constantly resize while loading the first bytes
+const STDIN_BUFFER_START_CAPACITY: usize = 16 * 1024;
 
 /// Extracts a file to `output_path` from the CARv2 file at `input_path`
 pub(crate) async fn extract_file_from_car(
@@ -15,10 +21,27 @@ pub(crate) async fn extract_file_from_car(
         File::create_new(&output_path).await?
     };
 
-    CarExtractor::from_path(input_path)
-        .await?
-        .copy_to_writer(output_file)
-        .await
+    if input_path.as_os_str() == "-" {
+        // NOTE(@jmg-duarte,17/02/2025): this approach is bound to be inefficient for large inputs
+        // given that we're forced to load the whole input into memory upfront
+        // a possible alternative could be implementing a Reader that takes an AsyncRead
+        // and makes it AsyncSeek by keeping read contents in memory and forward seeks (where
+        // forward means that said part of the stream hasn't been loaded into memory yet) start
+        // loading file as needed before returning the new position
+
+        let mut buffer = Vec::with_capacity(STDIN_BUFFER_START_CAPACITY);
+        let mut buffered_stdin = BufReader::new(tokio::io::stdin());
+        buffered_stdin.read_to_end(&mut buffer).await?;
+        CarExtractor::from_vec(buffer)
+            .await?
+            .copy_to_writer(output_file)
+            .await
+    } else {
+        CarExtractor::from_path(input_path)
+            .await?
+            .copy_to_writer(output_file)
+            .await
+    }
 }
 
 /// Tests for file extraction.

mater/cli/src/main.rs

@@ -52,9 +52,18 @@ async fn main() -> Result<(), Error> {
             overwrite,
         } => {
             let output_path = output_path.unwrap_or_else(|| {
-                let mut new_path = input_path.clone();
-                new_path.set_extension("car");
-                new_path
+                // If we let the output become `-.car` it isn't only weird
+                // terminals are annoying with it
+                if input_path.as_os_str() == "-" {
+                    let file_name = "stdin.car";
+                    let mut path = PathBuf::with_capacity(file_name.len());
+                    path.set_file_name(file_name);
+                    path
+                } else {
+                    let mut new_path = input_path.clone();
+                    new_path.set_extension("car");
+                    new_path
+                }
             });
             let cid = convert_file_to_car(&input_path, &output_path, overwrite).await?;
 
@@ -74,9 +83,26 @@ async fn main() -> Result<(), Error> {
             overwrite,
         } => {
             let output_path = output_path.unwrap_or_else(|| {
-                let mut new_path = input_path.clone();
-                new_path.set_extension("");
-                new_path
+                // If we let the output become `-` it isn't only weird
+                // terminals are annoying with it
+                if input_path.as_os_str() == "-" {
+                    let file_name = "stdin";
+                    let mut path = PathBuf::with_capacity(file_name.len());
+                    path.set_file_name(file_name);
+                    path
+                } else {
+                    let mut new_path = input_path.clone();
+                    if let Some(_) = new_path.extension() {
+                        // We don't check if the ext is `.car` because we don't care about the ext
+                        // being "right", we only care about the bytes, however, we remove the ext
+                        // if there is one
+                        new_path.set_extension("");
+                    } else {
+                        // If no extension existed, we must add one or we risk overwriting the file
+                        new_path.set_extension("out");
+                    }
+                    new_path
+                }
             });
             extract_file_from_car(&input_path, &output_path, overwrite).await?;
 

mater/lib/src/lib.rs

@@ -9,7 +9,7 @@
 #![deny(rustdoc::broken_intra_doc_links)]
 #![deny(rustdoc::private_intra_doc_links)]
 #![deny(unsafe_code)]
-#![deny(clippy::unwrap_used)]
+#![cfg_attr(not(test), deny(clippy::unwrap_used))]
 
 mod async_varint;
 mod cid;