From 3a87307288759237368d2a4298bb66941b7d9dcf Mon Sep 17 00:00:00 2001 From: Gavin Peacock Date: Mon, 17 Feb 2025 12:11:47 -0800 Subject: [PATCH] feat: Add validation_status to manifest reports chore: remove Store and ManifestStore from the non-v1 public SDK refactoring and code removal --- .../src/identity_assertion/assertion.rs | 6 +- make_test_images/src/compare_manifests.rs | 23 +- sdk/src/builder.rs | 7 +- sdk/src/claim.rs | 1 + sdk/src/ingredient.rs | 4 +- sdk/src/jumbf_io.rs | 3 +- sdk/src/lib.rs | 1 + sdk/src/manifest_store.rs | 10 +- sdk/src/manifest_store_report.rs | 14 +- sdk/src/reader.rs | 293 ++++++++++++++---- sdk/src/store.rs | 84 +++-- sdk/src/validation_results.rs | 76 ++++- sdk/src/validation_status.rs | 104 ++----- sdk/tests/common/compare_readers.rs | 6 +- 14 files changed, 426 insertions(+), 206 deletions(-) diff --git a/cawg_identity/src/identity_assertion/assertion.rs b/cawg_identity/src/identity_assertion/assertion.rs index 51fce7ee7..5b47cad0c 100644 --- a/cawg_identity/src/identity_assertion/assertion.rs +++ b/cawg_identity/src/identity_assertion/assertion.rs @@ -161,8 +161,8 @@ impl IdentityAssertion { /// /// [`ManifestStore`]: c2pa::ManifestStore #[cfg(feature = "v1_api")] - pub async fn summarize_manifest_store( - store: &c2pa::ManifestStore, + pub async fn summarize_reader( + reader: &c2pa::Reader, verifier: &SV, ) -> impl Serialize { // NOTE: We can't write this using .map(...).collect() because there are async @@ -174,7 +174,7 @@ impl IdentityAssertion { >, > = BTreeMap::new(); - for (id, manifest) in store.manifests() { + for (id, manifest) in reader.manifests() { let report = Self::summarize_all_impl(manifest, verifier).await; reports.insert(id.clone(), report); } diff --git a/make_test_images/src/compare_manifests.rs b/make_test_images/src/compare_manifests.rs index 2f1747577..d70bf9391 100644 --- a/make_test_images/src/compare_manifests.rs +++ b/make_test_images/src/compare_manifests.rs @@ -15,7 +15,7 @@ use std::collections::HashMap; use std::{fs, path::Path}; -use c2pa::{Error, Reader as ManifestStore, Result}; +use c2pa::{Error, Reader, Result}; /// Compares all the files in two directories and returns a list of issues pub fn compare_folders, Q: AsRef>(folder1: P, folder2: Q) -> Result<()> { @@ -79,17 +79,16 @@ pub fn compare_image_manifests, Q: AsRef>( ) -> Result> { let manifest_store1 = match m1.as_ref().extension() { Some(ext) if ext == "json" => { - ManifestStore::from_json(&fs::read_to_string(m1)?) + Reader::from_json(&fs::read_to_string(m1)?) //serde_json::from_str(&fs::read_to_string(m1)?).map_err(Error::JsonError) } - _ => ManifestStore::from_file(m1.as_ref()), + _ => Reader::from_file(m1.as_ref()), }; let manifest_store2 = match m2.as_ref().extension() { - Some(ext) if ext == "json" => ManifestStore::from_json(&fs::read_to_string(m2)?), - _ => ManifestStore::from_file(m2.as_ref()), + Some(ext) if ext == "json" => Reader::from_json(&fs::read_to_string(m2)?), + _ => Reader::from_file(m2.as_ref()), }; - // let manifest_store1 = ManifestStore::from_file(m1); - // let manifest_store2 = ManifestStore::from_file(m2); + match (manifest_store1, manifest_store2) { (Ok(manifest_store1), Ok(manifest_store2)) => { compare_manifests(&manifest_store1, &manifest_store2) @@ -102,8 +101,8 @@ pub fn compare_image_manifests, Q: AsRef>( /// Compares two manifest stores and returns a list of issues. pub fn compare_manifests( - manifest_store1: &ManifestStore, - manifest_store2: &ManifestStore, + manifest_store1: &Reader, + manifest_store2: &Reader, ) -> Result> { // first we need to gather all the manifests in the order they are first seen recursively let mut labels1 = Vec::new(); @@ -136,11 +135,7 @@ pub fn compare_manifests( } // creates list of manifests in the order they are first seen from the active manifest -fn gather_manifests( - manifest_store: &ManifestStore, - manifest_label: &str, - labels: &mut Vec, -) { +fn gather_manifests(manifest_store: &Reader, manifest_label: &str, labels: &mut Vec) { if !labels.contains(&manifest_label.to_string()) { labels.push(manifest_label.to_string()); } diff --git a/sdk/src/builder.rs b/sdk/src/builder.rs index b71a200c6..a551110a9 100644 --- a/sdk/src/builder.rs +++ b/sdk/src/builder.rs @@ -1469,7 +1469,7 @@ mod tests { .add("thumbnail.jpg", TEST_THUMBNAIL.to_vec()) .unwrap(); - // sign the ManifestStoreBuilder and write it to the output stream + // sign the Builder and write it to the output stream let signer = crate::utils::test::temp_async_remote_signer(); builder .sign_async(signer.as_ref(), format, &mut source, &mut dest) @@ -1504,7 +1504,7 @@ mod tests { .add_resource("thumbnail.jpg", Cursor::new(TEST_THUMBNAIL)) .unwrap(); - // sign the ManifestStoreBuilder and write it to the output stream + // sign the Builder and write it to the output stream let signer = test_signer(SigningAlg::Ps256); let manifest_data = builder .sign(signer.as_ref(), "image/jpeg", &mut source, &mut dest) @@ -1669,7 +1669,7 @@ mod tests { zipped.rewind().unwrap(); let mut builder = Builder::from_archive(&mut zipped).unwrap(); - // sign the ManifestStoreBuilder and write it to the output stream + // sign the Builder and write it to the output stream let signer = test_signer(SigningAlg::Ps256); let _manifest_data = builder .sign(signer.as_ref(), "image/jpeg", &mut source, &mut dest) @@ -1855,6 +1855,7 @@ mod tests { .expect("builder sign"); output.set_position(0); + println!("output len: {}", output.get_ref().len()); let reader = Reader::from_stream("jpeg", &mut output).expect("from_bytes"); println!("reader = {reader}"); let m = reader.active_manifest().unwrap(); diff --git a/sdk/src/claim.rs b/sdk/src/claim.rs index 5e5c181f9..0659734f3 100644 --- a/sdk/src/claim.rs +++ b/sdk/src/claim.rs @@ -81,6 +81,7 @@ static _V2_SPEC_DEPRECATED_ASSERTIONS: [&str; 4] = [ // Enum to encapsulate the data type of the source asset. This simplifies // having different implementations for functions as a single entry point can be // used to handle different data types. +#[allow(dead_code)] // Bytes and third param in StreamFragment not used without v1_api feature pub enum ClaimAssetData<'a> { #[cfg(feature = "file_io")] Path(&'a Path), diff --git a/sdk/src/ingredient.rs b/sdk/src/ingredient.rs index 947a6d8e0..c1d89855a 100644 --- a/sdk/src/ingredient.rs +++ b/sdk/src/ingredient.rs @@ -45,7 +45,7 @@ use crate::{ store::Store, utils::xmp_inmemory_utils::XmpInfo, validation_results::ValidationResults, - validation_status::{self, validation_results_for_store, ValidationStatus}, + validation_status::{self, ValidationStatus}, }; #[derive(Debug, Default, Deserialize, Serialize)] @@ -589,7 +589,7 @@ impl Ingredient { match result { Ok(store) => { // generate validation results from the store - let validation_results = validation_results_for_store(&store, validation_log); + let validation_results = ValidationResults::from_store(&store, validation_log); if let Some(claim) = store.provenance_claim() { // if the parent claim is valid and has a thumbnail, use it diff --git a/sdk/src/jumbf_io.rs b/sdk/src/jumbf_io.rs index 163d66750..bfe1ddd8a 100644 --- a/sdk/src/jumbf_io.rs +++ b/sdk/src/jumbf_io.rs @@ -329,14 +329,13 @@ where } } -#[cfg(feature = "file_io")] /// removes the C2PA JUMBF from an asset /// Note: Use with caution since this deletes C2PA data /// It is useful when creating remote manifests from embedded manifests /// /// path - path to file to be updated /// returns Unsupported type or errors from remove_cai_store -#[allow(dead_code)] +#[cfg(feature = "file_io")] pub fn remove_jumbf_from_file>(path: P) -> Result<()> { let ext = get_file_extension(path.as_ref()).ok_or(Error::UnsupportedType)?; match get_assetio_handler(&ext) { diff --git a/sdk/src/lib.rs b/sdk/src/lib.rs index b7fd0966c..1da994a50 100644 --- a/sdk/src/lib.rs +++ b/sdk/src/lib.rs @@ -148,6 +148,7 @@ pub(crate) mod jumbf; pub(crate) mod manifest; pub(crate) mod manifest_assertion; +#[cfg(feature = "v1_api")] pub(crate) mod manifest_store; pub(crate) mod manifest_store_report; #[allow(dead_code)] diff --git a/sdk/src/manifest_store.rs b/sdk/src/manifest_store.rs index 15d2882d3..f6cdc0c1d 100644 --- a/sdk/src/manifest_store.rs +++ b/sdk/src/manifest_store.rs @@ -30,7 +30,7 @@ use crate::{ jumbf::labels::{manifest_label_from_uri, to_absolute_uri, to_relative_uri}, store::Store, validation_results::ValidationResults, - validation_status::{validation_results_for_store, ValidationStatus}, + validation_status::ValidationStatus, Error, Manifest, Result, }; @@ -183,7 +183,7 @@ impl ManifestStore { validation_log: &impl StatusTracker, #[cfg(feature = "file_io")] resource_path: Option<&Path>, ) -> ManifestStore { - let mut validation_results = validation_results_for_store(&store, validation_log); + let mut validation_results = ValidationResults::from_store(&store, validation_log); let mut manifest_store = ManifestStore::new(); manifest_store.active_manifest = store.provenance_label(); @@ -220,7 +220,7 @@ impl ManifestStore { validation_log: &impl StatusTracker, #[cfg(feature = "file_io")] resource_path: Option<&Path>, ) -> ManifestStore { - let mut validation_results = validation_results_for_store(&store, validation_log); + let mut validation_results = ValidationResults::from_store(&store, validation_log); let mut manifest_store = ManifestStore::new(); manifest_store.active_manifest = store.provenance_label(); @@ -252,10 +252,6 @@ impl ManifestStore { manifest_store } - pub(crate) fn store(&self) -> &Store { - &self.store - } - /// Creates a new Manifest Store from a Manifest #[allow(dead_code)] #[deprecated(since = "0.38.0", note = "Please use Reader::from_json() instead")] diff --git a/sdk/src/manifest_store_report.rs b/sdk/src/manifest_store_report.rs index 5eb27e374..14ff70de4 100644 --- a/sdk/src/manifest_store_report.rs +++ b/sdk/src/manifest_store_report.rs @@ -26,7 +26,7 @@ use serde_json::Value; use crate::{ assertion::AssertionData, claim::Claim, store::Store, validation_results::ValidationResults, - validation_status::ValidationStatus, Result, + validation_status::ValidationStatus, Result, ValidationState, }; /// Low level JSON based representation of Manifest Store - used for debugging @@ -39,6 +39,7 @@ pub struct ManifestStoreReport { #[serde(skip_serializing_if = "Option::is_none")] validation_status: Option>, pub(crate) validation_results: Option, + pub(crate) validation_state: Option, } impl ManifestStoreReport { @@ -54,9 +55,20 @@ impl ManifestStoreReport { manifests, validation_status: None, validation_results: None, + validation_state: None, }) } + pub(crate) fn from_store_with_results( + store: &Store, + validation_results: &ValidationResults, + ) -> Result { + let mut report = Self::from_store(store)?; + report.validation_results = Some(validation_results.clone()); + report.validation_state = Some(validation_results.validation_state()); + Ok(report) + } + /// Prints tree view of manifest store #[cfg(feature = "file_io")] #[cfg(feature = "v1_api")] diff --git a/sdk/src/reader.rs b/sdk/src/reader.rs index fa650a884..a50c98731 100644 --- a/sdk/src/reader.rs +++ b/sdk/src/reader.rs @@ -16,20 +16,23 @@ #[cfg(feature = "file_io")] use std::fs::{read, File}; -use std::io::{Read, Seek, Write}; +use std::{ + collections::HashMap, + io::{Read, Seek, Write}, +}; use async_generic::async_generic; +use c2pa_crypto::base64; use c2pa_status_tracker::DetailedStatusTracker; #[cfg(feature = "json_schema")] use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use serde_with::skip_serializing_none; -#[cfg(feature = "file_io")] -use crate::error::Error; use crate::{ claim::ClaimAssetData, - error::Result, - manifest_store::ManifestStore, + error::{Error, Result}, + jumbf::labels::{manifest_label_from_uri, to_absolute_uri, to_relative_uri}, settings::get_settings_value, store::Store, validation_results::{ValidationResults, ValidationState}, @@ -38,10 +41,28 @@ use crate::{ }; /// A reader for the manifest store. +#[skip_serializing_none] #[derive(Serialize, Deserialize)] #[cfg_attr(feature = "json_schema", derive(JsonSchema))] pub struct Reader { - pub(crate) manifest_store: ManifestStore, + /// A label for the active (most recent) manifest in the store + active_manifest: Option, + + /// A HashMap of Manifests + manifests: HashMap, + + /// ValidationStatus generated when loading the ManifestStore from an asset + validation_status: Option>, + + /// ValidationStatus generated when loading the ManifestStore from an asset + validation_results: Option, + + /// The validation state of the manifest store + validation_state: Option, + + #[serde(skip)] + /// We keep this around so we can generate a detailed report if needed + store: Store, } impl Reader { @@ -65,16 +86,13 @@ impl Reader { /// ``` #[async_generic()] pub fn from_stream(format: &str, mut stream: impl Read + Seek + Send) -> Result { - let verify = get_settings_value::("verify.verify_after_reading")?; // defaults to true - #[allow(deprecated)] - let reader = if _sync { - ManifestStore::from_stream(format, &mut stream, verify) + let manifest_bytes = Store::load_jumbf_from_stream(format, &mut stream)?; + + if _sync { + Self::from_manifest_data_and_stream(&manifest_bytes, format, &mut stream) } else { - ManifestStore::from_stream_async(format, &mut stream, verify).await - }?; - Ok(Reader { - manifest_store: reader, - }) + Self::from_manifest_data_and_stream_async(&manifest_bytes, format, &mut stream).await + } } #[cfg(feature = "file_io")] @@ -135,8 +153,7 @@ impl Reader { /// # WARNING /// This function is intended for use in testing. Don't use it in an implementation. pub fn from_json(json: &str) -> Result { - let manifest_store = serde_json::from_str(json)?; - Ok(Reader { manifest_store }) + serde_json::from_str(json).map_err(crate::Error::JsonError) } /// Create a manifest store [`Reader`] from existing `c2pa_data` and a stream. @@ -163,28 +180,16 @@ impl Reader { let verify = get_settings_value::("verify.verify_after_reading")?; // defaults to true - if _sync { - if verify { - Store::verify_store( - &store, - &mut ClaimAssetData::Stream(&mut stream, format), - &mut validation_log, - )?; - } - } else { - if verify { - Store::verify_store_async( - &store, - &mut ClaimAssetData::Stream(&mut stream, format), - &mut validation_log, - ) - .await?; - } + if verify { + let mut asset_data = ClaimAssetData::Stream(&mut stream, format); + if _sync { + Store::verify_store(&store, &mut asset_data, &mut validation_log) + } else { + Store::verify_store_async(&store, &mut asset_data, &mut validation_log).await + }?; } - Ok(Reader { - manifest_store: ManifestStore::from_store(store, &validation_log), - }) + Ok(Self::from_store(store, &validation_log)) } /// Create a [`Reader`] from an initial segment and a fragment stream. @@ -213,17 +218,13 @@ impl Reader { if verify { let mut fragment = ClaimAssetData::StreamFragment(&mut stream, &mut fragment, format); if _sync { - // verify store and claims Store::verify_store(&store, &mut fragment, &mut validation_log) } else { - // verify store and claims Store::verify_store_async(&store, &mut fragment, &mut validation_log).await }?; }; - Ok(Self { - manifest_store: ManifestStore::from_store(store, &validation_log), - }) + Ok(Self::from_store(store, &validation_log)) } #[cfg(feature = "file_io")] @@ -235,15 +236,70 @@ impl Reader { fragments: &Vec, ) -> Result { let verify = get_settings_value::("verify.verify_after_reading")?; // defaults to true - #[allow(deprecated)] - Ok(Reader { - manifest_store: ManifestStore::from_fragments(path, fragments, verify)?, - }) + + let mut validation_log = DetailedStatusTracker::default(); + + let asset_type = crate::jumbf_io::get_supported_file_extension(path.as_ref()) + .ok_or(crate::Error::UnsupportedType)?; + + let mut init_segment = std::fs::File::open(path.as_ref())?; + + match Store::load_from_file_and_fragments( + &asset_type, + &mut init_segment, + fragments, + verify, + &mut validation_log, + ) { + Ok(store) => Ok(Self::from_store(store, &validation_log)), + Err(e) => Err(e), + } } /// Get the manifest store as a JSON string pub fn json(&self) -> String { - self.manifest_store.to_string() + let mut json = serde_json::to_string_pretty(self).unwrap_or_default(); + + fn omit_tag(mut json: String, tag: &str) -> String { + while let Some(index) = json.find(&format!("\"{tag}\": [")) { + if let Some(idx2) = json[index..].find(']') { + json = format!( + "{}\"{}\": \"\"{}", + &json[..index], + tag, + &json[index + idx2 + 1..] + ); + } + } + json + } + + // Make a base64 hash from Vec values. + fn b64_tag(mut json: String, tag: &str) -> String { + while let Some(index) = json.find(&format!("\"{tag}\": [")) { + if let Some(idx2) = json[index..].find(']') { + let idx3 = json[index..].find('[').unwrap_or_default(); + + let bytes: Vec = + serde_json::from_slice(json[index + idx3..index + idx2 + 1].as_bytes()) + .unwrap_or_default(); + + json = format!( + "{}\"{}\": \"{}\"{}", + &json[..index], + tag, + base64::encode(&bytes), + &json[index + idx2 + 1..] + ); + } + } + + json + } + + json = b64_tag(json, "hash"); + json = omit_tag(json, "pad"); + json } /// Get the [`ValidationStatus`] array of the manifest store if it exists. @@ -260,7 +316,7 @@ impl Reader { /// let status = reader.validation_status(); /// ``` pub fn validation_status(&self) -> Option<&[ValidationStatus]> { - self.manifest_store.validation_status() + self.validation_status.as_deref() } /// Get the [`ValidationResults`] map of an asset if it exists. @@ -281,12 +337,12 @@ impl Reader { /// let status = reader.validation_results(); /// ``` pub fn validation_results(&self) -> Option<&ValidationResults> { - self.manifest_store.validation_results() + self.validation_results.as_ref() } /// Get the [`ValidationState`] of the manifest store. pub fn validation_state(&self) -> ValidationState { - if let Some(validation_results) = self.manifest_store.validation_results() { + if let Some(validation_results) = self.validation_results() { return validation_results.validation_state(); } @@ -319,24 +375,33 @@ impl Reader { /// Return the active [`Manifest`], or `None` if there's no active manifest. pub fn active_manifest(&self) -> Option<&Manifest> { - self.manifest_store.get_active() + if let Some(label) = self.active_manifest.as_ref() { + self.manifests.get(label) + } else { + None + } } /// Return the active [`Manifest`], or `None` if there's no active manifest. pub fn active_label(&self) -> Option<&str> { - self.manifest_store.active_label() + self.active_manifest.as_deref() } /// Returns an iterator over a collection of [`Manifest`] structs. pub fn iter_manifests(&self) -> impl Iterator + '_ { - self.manifest_store.manifests().values() + self.manifests.values() + } + + /// Returns a reference to the [`Manifest`] collection. + pub fn manifests(&self) -> &HashMap { + &self.manifests } /// Given a label, return the associated [`Manifest`], if it exists. /// # Arguments /// * `label` - The label of the requested [`Manifest`]. pub fn get_manifest(&self, label: &str) -> Option<&Manifest> { - self.manifest_store.get(label) + self.manifests.get(label) } /// Write a resource identified by URI to the given stream. @@ -362,11 +427,48 @@ impl Reader { pub fn resource_to_stream( &self, uri: &str, - mut stream: impl Write + Read + Seek + Send, + stream: impl Write + Read + Seek + Send, ) -> Result { - self.manifest_store - .get_resource(uri, &mut stream) - .map(|size| size as usize) + // get the manifest referenced by the uri, or the active one if None + // add logic to search for local or absolute uri identifiers + let (manifest, label) = match manifest_label_from_uri(uri) { + Some(label) => (self.manifests.get(&label), label), + None => ( + self.active_manifest(), + self.active_label().unwrap_or_default().to_string(), + ), + }; + let relative_uri = to_relative_uri(uri); + let absolute_uri = to_absolute_uri(&label, uri); + + if let Some(manifest) = manifest { + let find_resource = |uri: &str| -> Result<&crate::ResourceStore> { + let mut resources = manifest.resources(); + if !resources.exists(uri) { + // also search ingredients resources to support Reader model + for ingredient in manifest.ingredients() { + if ingredient.resources().exists(uri) { + resources = ingredient.resources(); + return Ok(resources); + } + } + } else { + return Ok(resources); + } + Err(Error::ResourceNotFound(uri.to_owned())) + }; + let result = find_resource(&relative_uri); + match result { + Ok(resource) => resource.write_stream(&relative_uri, stream), + Err(_) => match find_resource(&absolute_uri) { + Ok(resource) => resource.write_stream(&absolute_uri, stream), + Err(e) => Err(e), + }, + } + } else { + Err(Error::ResourceNotFound(uri.to_owned())) + } + .map(|size| size as usize) } /// Convert a URI to a file path. (todo: move this to utils) @@ -406,7 +508,7 @@ impl Reader { pub fn to_folder>(&self, path: P) -> Result<()> { std::fs::create_dir_all(&path)?; std::fs::write(path.as_ref().join("manifest.json"), self.json())?; - for manifest in self.manifest_store.manifests().values() { + for manifest in self.manifests.values() { let resources = manifest.resources(); for (uri, data) in resources.resources() { let id_path = Self::uri_to_path(uri, manifest.label().unwrap_or("unknown")); @@ -420,16 +522,75 @@ impl Reader { } Ok(()) } + + #[async_generic()] + fn from_store(store: Store, validation_log: &DetailedStatusTracker) -> Self { + let mut validation_results = ValidationResults::from_store(&store, validation_log); + + let active_manifest = store.provenance_label(); + let mut manifests = HashMap::new(); + + for claim in store.claims() { + let manifest_label = claim.label(); + let result = if _sync { + #[cfg(feature = "file_io")] + { + Manifest::from_store(&store, manifest_label, None) + } + #[cfg(not(feature = "file_io"))] + Manifest::from_store(&store, manifest_label) + } else { + #[cfg(feature = "file_io")] + { + Manifest::from_store_async(&store, manifest_label, None).await + } + #[cfg(not(feature = "file_io"))] + Manifest::from_store_async(&store, manifest_label).await + }; + match result { + Ok(manifest) => { + manifests.insert(manifest_label.to_owned(), manifest); + } + Err(e) => { + validation_results.add_status(manifest_label, ValidationStatus::from_error(&e)); + } + }; + } + + let validation_state = validation_results.validation_state(); + Self { + active_manifest, + manifests, + validation_status: validation_results.validation_errors(), + validation_results: Some(validation_results), + validation_state: Some(validation_state), + store, + } + } } impl Default for Reader { fn default() -> Self { Self { - manifest_store: ManifestStore::new(), + active_manifest: None, + manifests: HashMap::::new(), + validation_status: None, + validation_results: None, + validation_state: None, + store: Store::new(), } } } +/// Convert the Reader to a JSON value. +impl TryInto for Reader { + type Error = Error; + + fn try_into(self) -> Result { + serde_json::to_value(self).map_err(Error::JsonError) + } +} + /// Prints the JSON of the manifest data. impl std::fmt::Display for Reader { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -440,9 +601,11 @@ impl std::fmt::Display for Reader { /// Prints the full debug details of the manifest data. impl std::fmt::Debug for Reader { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut report = ManifestStoreReport::from_store(self.manifest_store.store()) - .map_err(|_| std::fmt::Error)?; - report.validation_results = self.manifest_store.validation_results().cloned(); + let report = match self.validation_results() { + Some(results) => ManifestStoreReport::from_store_with_results(&self.store, results), + None => ManifestStoreReport::from_store(&self.store), + } + .map_err(|_| std::fmt::Error)?; f.write_str(&report.to_string()) } } @@ -496,7 +659,7 @@ pub mod tests { println!("{reader}"); assert_eq!(reader.validation_status(), None); assert_eq!(reader.validation_state(), ValidationState::Valid); - assert_eq!(reader.manifest_store.manifests().len(), 3); + assert_eq!(reader.manifests.len(), 3); Ok(()) } @@ -506,7 +669,7 @@ pub mod tests { let reader = Reader::from_stream("image/jpeg", std::io::Cursor::new(IMAGE_COMPLEX_MANIFEST))?; assert_eq!(reader.validation_status(), None); - assert_eq!(reader.manifest_store.manifests().len(), 3); + assert_eq!(reader.manifests.len(), 3); let manifest = reader.active_manifest().unwrap(); let ingredient = manifest.ingredients().iter().next().unwrap(); let uri = ingredient.thumbnail_ref().unwrap().identifier.clone(); diff --git a/sdk/src/store.rs b/sdk/src/store.rs index a6f8ae239..5800b8744 100644 --- a/sdk/src/store.rs +++ b/sdk/src/store.rs @@ -27,16 +27,19 @@ use c2pa_crypto::{ cose::{parse_cose_sign1, CertificateTrustPolicy, TimeStampStorage}, hash::sha256, }; -use c2pa_status_tracker::{log_item, DetailedStatusTracker, OneShotStatusTracker, StatusTracker}; +#[cfg(feature = "v1_api")] +use c2pa_status_tracker::DetailedStatusTracker; +use c2pa_status_tracker::{log_item, OneShotStatusTracker, StatusTracker}; use log::error; #[cfg(feature = "v1_api")] use crate::jumbf_io::save_jumbf_to_memory; #[cfg(feature = "file_io")] use crate::jumbf_io::{ - get_file_extension, get_supported_file_extension, load_jumbf_from_file, object_locations, - remove_jumbf_from_file, save_jumbf_to_file, + get_file_extension, get_supported_file_extension, load_jumbf_from_file, save_jumbf_to_file, }; +#[cfg(feature = "file_io")] +use crate::jumbf_io::{object_locations, remove_jumbf_from_file}; use crate::{ assertion::{ Assertion, AssertionBase, AssertionData, AssertionDecodeError, AssertionDecodeErrorCause, @@ -59,7 +62,6 @@ use crate::{ AsyncDynamicAssertion, DynamicAssertion, DynamicAssertionContent, PreliminaryClaim, }, error::{Error, Result}, - external_manifest::ManifestPatchCallback, hash_utils::{hash_by_alg, vec_compare, verify_by_alg}, hashed_uri::HashedUri, jumbf::{ @@ -75,8 +77,10 @@ use crate::{ salt::DefaultSalt, settings::get_settings_value, utils::{hash_utils::HashRange, io_utils::stream_len, patch::patch_bytes}, - validation_status, AsyncSigner, RemoteSigner, Signer, + validation_status, AsyncSigner, Signer, }; +#[cfg(feature = "v1_api")] +use crate::{external_manifest::ManifestPatchCallback, RemoteSigner}; const MANIFEST_STORE_EXT: &str = "c2pa"; // file extension for external manifests @@ -172,6 +176,7 @@ impl Store { } /// Return label for the store + #[allow(dead_code)] // doesn't harm to have this pub fn label(&self) -> &str { &self.label } @@ -199,6 +204,7 @@ impl Store { } /// Clear all existing trust anchors + #[cfg(feature = "v1_api")] pub fn clear_trust_anchors(&mut self) { self.ctp.clear(); } @@ -1925,6 +1931,7 @@ impl Store { /// It is an error if `get_data_hashed_manifest_placeholder` was not called first /// as this call inserts the DataHash placeholder assertion to reserve space for the /// actual hash values not required when using BoxHashes. + #[cfg(feature = "v1_api")] pub async fn get_data_hashed_embeddable_manifest_remote( &mut self, dh: &DataHash, @@ -2330,6 +2337,7 @@ impl Store { /// the Signer you plan to use. This function is not needed when using Box Hash. This function is used /// in conjunction with `embed_placed_manifest`. `embed_placed_manifest` will accept the manifest to sign and place /// in the output. + #[cfg(feature = "v1_api")] pub fn get_placed_manifest( &mut self, reserve_size: usize, @@ -2353,7 +2361,7 @@ impl Store { /// 'format' shoould match the type of the input stream.. /// Upon return, the output stream will contain the new manifest signed with signer /// This directly modifies the asset in stream, backup stream first if you need to preserve it. - + #[cfg(feature = "v1_api")] #[async_generic( async_signature( manifest_bytes: &[u8], @@ -2691,6 +2699,7 @@ impl Store { /// Embed the claims store as jumbf into an asset using an CoseSign box generated remotely. Updates XMP with provenance record. #[cfg(feature = "file_io")] + #[cfg(feature = "v1_api")] pub async fn save_to_asset_remote_signed( &mut self, asset_path: &Path, @@ -3187,6 +3196,7 @@ impl Store { } // verify from a buffer without file i/o + #[cfg(feature = "v1_api")] pub fn verify_from_buffer( &mut self, buf: &[u8], @@ -3482,6 +3492,7 @@ impl Store { /// data: reference to bytes of the file /// verify: if true will run verification checks when loading /// validation_log: If present all found errors are logged and returned, otherwise first error causes exit and is returned + #[cfg(feature = "v1_api")] pub async fn load_from_memory_async( asset_type: &str, data: &[u8], @@ -3536,11 +3547,41 @@ impl Store { }) } + /// Load Store from a stream and fragment stream + /// + /// asset_type: asset extension or mime type + /// stream: reference to initial segment asset + /// fragment: reference to fragment asset + /// validation_log: If present all found errors are logged and returned, otherwise first error causes exit and is returned + #[async_generic()] + pub fn load_fragment_from_stream( + format: &str, + mut stream: impl Read + Seek + Send, + mut fragment: impl Read + Seek + Send, + validation_log: &mut impl StatusTracker, + ) -> Result { + let manifest_bytes = Store::load_jumbf_from_stream(format, &mut stream)?; + let store = Store::from_jumbf(&manifest_bytes, validation_log)?; + + let verify = get_settings_value::("verify.verify_after_reading")?; // defaults to true + + if verify { + let mut fragment = ClaimAssetData::StreamFragment(&mut stream, &mut fragment, format); + if _sync { + Store::verify_store(&store, &mut fragment, validation_log) + } else { + Store::verify_store_async(&store, &mut fragment, validation_log).await + }?; + }; + Ok(store) + } + /// Load Store from a in-memory asset /// asset_type: asset extension or mime type /// data: reference to bytes of the the file /// verify: if true will run verification checks when loading /// validation_log: If present all found errors are logged and returned, otherwise first error causes exit and is returned + #[cfg(feature = "v1_api")] pub fn load_fragment_from_memory( asset_type: &str, init_segment: &[u8], @@ -3715,10 +3756,10 @@ pub mod tests { #![allow(clippy::panic)] #![allow(clippy::unwrap_used)] - use std::io::Write; + use std::{fs, io::Write}; use c2pa_crypto::raw_signature::SigningAlg; - use c2pa_status_tracker::StatusTracker; + use c2pa_status_tracker::{DetailedStatusTracker, StatusTracker}; use memchr::memmem; use serde::Serialize; use sha2::{Digest, Sha256}; @@ -3768,6 +3809,7 @@ pub mod tests { #[test] #[cfg(feature = "file_io")] + #[cfg(feature = "v1_api")] fn test_jumbf_generation() { // test adding to actual image let ap = fixture_path("earth_apollo17.jpg"); @@ -4234,6 +4276,7 @@ pub mod tests { assert!(errors.is_empty()); } + #[cfg(feature = "v1_api")] #[actix::test] async fn test_jumbf_generation_remote() { // test adding to actual image @@ -4301,9 +4344,9 @@ pub mod tests { store.commit_claim(claim1).unwrap(); store.save_to_asset(&ap, signer.as_ref(), &op).unwrap(); store.commit_claim(claim_capture).unwrap(); - store.save_to_asset(&op, signer.as_ref(), &op).unwrap(); + store.save_to_asset(&ap, signer.as_ref(), &op).unwrap(); store.commit_claim(claim2).unwrap(); - store.save_to_asset(&op, signer.as_ref(), &op).unwrap(); + store.save_to_asset(&ap, signer.as_ref(), &op).unwrap(); // write to new file println!("Provenance: {}\n", store.provenance_path().unwrap()); @@ -5948,6 +5991,7 @@ pub mod tests { #[test] #[cfg(feature = "file_io")] + #[cfg(feature = "v1_api")] fn test_datahash_embeddable_manifest_user_hashed() { // test adding to actual image @@ -6013,10 +6057,12 @@ pub mod tests { assert!(errors.is_empty()); } + #[cfg(feature = "v1_api")] struct PlacedCallback { path: String, } + #[cfg(feature = "v1_api")] impl ManifestPatchCallback for PlacedCallback { fn patch_manifest(&self, manifest_store: &[u8]) -> Result> { use ::jumbf::parser::SuperBox; @@ -6060,6 +6106,7 @@ pub mod tests { #[test] #[cfg(feature = "file_io")] + #[cfg(feature = "v1_api")] fn test_placed_manifest() { use crate::jumbf::labels::to_normalized_uri; @@ -6446,23 +6493,22 @@ pub mod tests { // verify the fragments let output_init = new_output_path.join(p.file_name().unwrap()); - let init_stream = std::fs::read(output_init).unwrap(); + let mut init_stream = std::fs::File::open(&output_init).unwrap(); for entry in &fragments { let file_path = new_output_path.join(entry.file_name().unwrap()); let mut validation_log = DetailedStatusTracker::default(); - let fragment_stream = std::fs::read(&file_path).unwrap(); - let _manifest = Store::load_fragment_from_memory( + let mut fragment_stream = std::fs::File::open(&file_path).unwrap(); + let _manifest = Store::load_fragment_from_stream( "mp4", - &init_stream, - &fragment_stream, - true, + &mut init_stream, + &mut fragment_stream, &mut validation_log, ) .unwrap(); - + init_stream.seek(std::io::SeekFrom::Start(0)).unwrap(); let errors = validation_log.take_errors(); assert!(errors.is_empty()); } @@ -6473,11 +6519,11 @@ pub mod tests { output_fragments.push(new_output_path.join(entry.file_name().unwrap())); } - let mut reader = Cursor::new(init_stream); + //let mut reader = Cursor::new(init_stream); let mut validation_log = DetailedStatusTracker::default(); let _manifest = Store::load_from_file_and_fragments( "mp4", - &mut reader, + &mut init_stream, &output_fragments, true, &mut validation_log, diff --git a/sdk/src/validation_results.rs b/sdk/src/validation_results.rs index b20f88a4b..8e6203cfa 100644 --- a/sdk/src/validation_results.rs +++ b/sdk/src/validation_results.rs @@ -12,12 +12,15 @@ // each license. pub use c2pa_status_tracker::validation_codes::*; -use c2pa_status_tracker::LogKind; +use c2pa_status_tracker::{LogKind, StatusTracker}; #[cfg(feature = "json_schema")] use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::validation_status::ValidationStatus; +use crate::{ + assertion::AssertionBase, assertions::Ingredient, jumbf::labels::manifest_label_from_uri, + store::Store, validation_status::ValidationStatus, +}; #[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[cfg_attr(feature = "json_schema", derive(JsonSchema))] @@ -95,6 +98,75 @@ pub struct ValidationResults { } impl ValidationResults { + pub(crate) fn from_store(store: &Store, validation_log: &impl StatusTracker) -> Self { + let mut results = ValidationResults::default(); + + let mut statuses: Vec = validation_log + .logged_items() + .iter() + .filter_map(ValidationStatus::from_log_item) + .collect(); + + // Filter out any status that is already captured in an ingredient assertion. + if let Some(claim) = store.provenance_claim() { + let active_manifest = Some(claim.label().to_string()); + + // This closure returns true if the URI references the store's active manifest. + let is_active_manifest = |uri: Option<&str>| { + uri.is_some_and(|uri| manifest_label_from_uri(uri) == active_manifest) + }; + + let make_absolute = |i: Ingredient| { + // Get a flat list of validation statuses from the ingredient. + let validation_status = match i.validation_results { + Some(v) => Some(v.validation_status()), + None => i.validation_status.map(|s| s.to_owned()), + }; + + // Convert any relative manifest urls found in ingredient validation statuses to absolute. + validation_status.map(|mut statuses| { + if let Some(label) = i + .active_manifest + .as_ref() + .or(i.c2pa_manifest.as_ref()) + .map(|m| m.url()) + .and_then(|uri| manifest_label_from_uri(&uri)) + { + for status in &mut statuses { + status.make_absolute(&label) + } + } + statuses + }) + }; + + // We only need to do the more detailed filtering if there are any status + // reports that reference ingredients. + if statuses.iter().any(|s| !is_active_manifest(s.url())) { + // Collect all the ValidationStatus records from all the ingredients in the store. + // Since we need to process v1,v2 and v3 ingredients, we process all in the same format. + let ingredient_statuses: Vec = store + .claims() + .iter() + .flat_map(|c| c.ingredient_assertions()) + .filter_map(|a| Ingredient::from_assertion(a).ok()) + .filter_map(make_absolute) + .flatten() + .collect(); + + // Filter statuses to only contain those from the active manifest and those not found in any ingredient. + statuses.retain(|s| { + is_active_manifest(s.url()) || !ingredient_statuses.iter().any(|i| i == s) + }) + } + let active_manifest_label = claim.label().to_string(); + for status in statuses { + results.add_status(&active_manifest_label, status); + } + } + results + } + /// Returns the [ValidationState] of the manifest store based on the validation results. pub fn validation_state(&self) -> ValidationState { let mut is_trusted = true; // Assume the state is trusted until proven otherwise diff --git a/sdk/src/validation_status.rs b/sdk/src/validation_status.rs index 47e71fe74..89a3fdb53 100644 --- a/sdk/src/validation_status.rs +++ b/sdk/src/validation_status.rs @@ -18,13 +18,17 @@ #![deny(missing_docs)] pub use c2pa_status_tracker::validation_codes::*; -use c2pa_status_tracker::{LogItem, LogKind, StatusTracker}; +#[cfg(feature = "v1_api")] +use c2pa_status_tracker::StatusTracker; +use c2pa_status_tracker::{LogItem, LogKind}; use log::debug; #[cfg(feature = "json_schema")] use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::{assertion::AssertionBase, assertions::Ingredient, error::Error, jumbf, store::Store}; +#[cfg(feature = "v1_api")] +use crate::store::Store; +use crate::{error::Error, jumbf}; /// A `ValidationStatus` struct describes the validation status of a /// specific part of a manifest. @@ -190,6 +194,16 @@ impl ValidationStatus { }), } } + + // converts a validation status url into and absolute URI given the manifest label. + pub(crate) fn make_absolute(&mut self, manifest_label: &str) { + if let Some(url) = &self.url { + if url.starts_with("self#jumbf") { + // Some are just labels (i.e. "Cose_Sign1") + self.url = Some(jumbf::labels::to_absolute_uri(manifest_label, url)); + } + } + } } impl PartialEq for ValidationStatus { @@ -198,88 +212,6 @@ impl PartialEq for ValidationStatus { } } -use crate::validation_results::ValidationResults; -/// Given a `Store` and a `StatusTracker`, return `ValidationResultsMap -pub fn validation_results_for_store( - store: &Store, - validation_log: &impl StatusTracker, -) -> ValidationResults { - let mut results = ValidationResults::default(); - - let mut statuses: Vec = validation_log - .logged_items() - .iter() - .filter_map(ValidationStatus::from_log_item) - .collect(); - - // Filter out any status that is already captured in an ingredient assertion. - if let Some(claim) = store.provenance_claim() { - let active_manifest = Some(claim.label().to_string()); - - // This closure returns true if the URI references the store's active manifest. - let is_active_manifest = |uri: Option<&str>| { - uri.is_some_and(|uri| jumbf::labels::manifest_label_from_uri(uri) == active_manifest) - }; - - let make_absolute = |i: Ingredient| { - // Get a flat list of validation statuses from the ingredient. - let validation_status = match i.validation_results { - Some(v) => Some(v.validation_status()), - None => i.validation_status, - }; - - // Convert any relative manifest urls found in ingredient validation statuses to absolute. - validation_status.map(|mut statuses| { - if let Some(label) = i - .active_manifest - .as_ref() - .or(i.c2pa_manifest.as_ref()) - .map(|m| m.url()) - .and_then(|uri| jumbf::labels::manifest_label_from_uri(&uri)) - { - for status in &mut statuses { - if let Some(url) = &status.url { - if url.starts_with("self#jumbf") { - // Some are just labels (i.e. "Cose_Sign1") - status.url = Some(jumbf::labels::to_absolute_uri(&label, url)); - } - } - } - } - statuses - }) - }; - - // We only need to do the more detailed filtering if there are any status - // reports that reference ingredients. - if statuses - .iter() - .any(|s| !is_active_manifest(s.url.as_deref())) - { - // Collect all the ValidationStatus records from all the ingredients in the store. - // Since we need to process v1,v2 and v3 ingredients, we process all in the same format. - let ingredient_statuses: Vec = store - .claims() - .iter() - .flat_map(|c| c.ingredient_assertions()) - .filter_map(|a| Ingredient::from_assertion(a).ok()) - .filter_map(make_absolute) - .flatten() - .collect(); - - // Filter statuses to only contain those from the active manifest and those not found in any ingredient. - statuses.retain(|s| { - is_active_manifest(s.url.as_deref()) || !ingredient_statuses.iter().any(|i| i == s) - }) - } - let active_manifest_label = claim.label().to_string(); - for status in statuses { - results.add_status(&active_manifest_label, status); - } - } - results -} - // TODO: Does this still need to be public? (I do see one reference in the JS SDK.) /// Get the validation status for a store. @@ -287,11 +219,13 @@ pub fn validation_results_for_store( /// Given a `Store` and a `StatusTracker`, return `ValidationStatus` items for each /// item in the tracker which reflect errors in the active manifest or which would not /// be reported as a validation error for any ingredient. +#[cfg(feature = "v1_api")] pub fn status_for_store( store: &Store, validation_log: &impl StatusTracker, ) -> Vec { - let validation_results = validation_results_for_store(store, validation_log); + let validation_results = + crate::validation_results::ValidationResults::from_store(store, validation_log); validation_results.validation_errors().unwrap_or_default() } diff --git a/sdk/tests/common/compare_readers.rs b/sdk/tests/common/compare_readers.rs index 880ed0dcd..5095483a0 100644 --- a/sdk/tests/common/compare_readers.rs +++ b/sdk/tests/common/compare_readers.rs @@ -124,14 +124,14 @@ pub fn compare_readers(reader1: &Reader, reader2: &Reader) -> Result } // creates list of manifests in the order they are first seen from the active manifest -fn gather_manifests(manifest_store: &Reader, manifest_label: &str, labels: &mut Vec) { +fn gather_manifests(reader: &Reader, manifest_label: &str, labels: &mut Vec) { if !labels.contains(&manifest_label.to_string()) { labels.push(manifest_label.to_string()); } - if let Some(manifest) = manifest_store.get_manifest(manifest_label) { + if let Some(manifest) = reader.get_manifest(manifest_label) { for ingredient in manifest.ingredients() { if let Some(label) = ingredient.active_manifest() { - gather_manifests(manifest_store, label, labels); + gather_manifests(reader, label, labels); } } }