diff --git a/CHANGELOG.md b/CHANGELOG.md index a9890218..8f270208 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## Unreleased + +- New `ReadableOCEL` and `AppendableOCEL` traits; OCEL exporters (CSV, XML, SQL, JSON) and the JSON importer are generic over them +- `SlimLinkedOCEL` implements `AppendableOCEL`, with auto-declare types, auto-grow attributes, and value coercion via new `OCELAttributeValue::try_coerce_to`; misordered streams are buffered and resolved on `finalize` +- `import_ocel_json_into` and `import_ocel_xml_into` stream JSON / XML directly into any `AppendableOCEL` (e.g., `SlimLinkedOCEL`) without materializing an `OCEL` first; `SlimLinkedOCEL::import_from_*` uses the streaming paths automatically +- `ReadableOCEL::iter_events_of_type` / `iter_objects_of_type` (default filters; `SlimLinkedOCEL` overrides via per-type indices, avoiding a full scan per type in SQL export) +- New `OCELAttributeType::as_type_str` returns `&'static str` (cheaper than `to_type_string`) +- `EventIndex` / `ObjectIndex` are now `u32`-backed; `into_inner` returns `u32` (**Breaking**) +- `SlimLinkedOCEL`, `SlimOCELEvent`, `SlimOCELObject` no longer derive `Deserialize` (still serialize); construct via `Importable` or `AppendableOCEL` (**Breaking**) +- `SlimOCELEvent::relationships` and `SlimOCELObject::relationships` are now `Vec<(QualifierIdx, ObjectIndex)>` (was `Vec<(String, ObjectIndex)>`); resolve qualifier strings via `SlimLinkedOCEL::qualifier_str` (**Breaking**) +- CSV exporter streams rows instead of buffering all rows; tracks `(time, object_id)` pairs in a `HashSet` to skip redundant object-attribute rows +- `SlimLinkedOCEL::from_ocel` now goes through `AppendableOCEL`, so it shares attribute coercion, schema-grow, and duplicate-id detection with the streaming import paths. Behavior changes: events/objects with attribute names not in the declared type schema grow the schema (before they were silently dropped); duplicate event/object ids are skipped with a warning (before they were kept but unreachable); references to undeclared types auto-create the type (before they were dropped with warning) +- `try_json_to_ocel` returning `Result` added alongside the existing panicking `json_to_ocel` +- `SlimLinkedOCEL::get_o2o_rev_obs_of_obtype` / `get_e2o_rev_evs_of_evtype`: qualifier-optional reverse-relation getters filtered by type +- New public object-centric analysis functions (also exposed as bindings): per-event sojourn and synchronization times with optional `top_k` (`analysis::object_centric::oc_performance`), E2O `(event_type, object_type)` counts and `source -> target` conversion rate (`analysis::object_centric::oc_statistics`), and per-object-type directly-follows graph and activity-trace variants (`discovery::object_centric::dfg` / `variants`) +- Fix SQL export/import of floats and timestamps: floats are written as `DOUBLE PRECISION` (full f64 precision) and timestamps as naive UTC (avoids a double-applied timezone offset); import maps `DOUBLE` / `DOUBLE PRECISION` columns back to float, so round-trips no longer drop float attributes +- New direct dependency on `hashbrown` for the slim per-id hash tables + ## 0.5.6 - Translate a `ProcessTree` into a `PetriNet`: diff --git a/Cargo.lock b/Cargo.lock index 425731ba..cde4efbd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3117,6 +3117,7 @@ dependencies = [ "duckdb", "flate2", "graphviz-rust", + "hashbrown 0.15.5", "inventory", "itertools 0.14.0", "kuzu", diff --git a/macros_process_mining/src/lib.rs b/macros_process_mining/src/lib.rs index 79e33b59..4b8d0403 100644 --- a/macros_process_mining/src/lib.rs +++ b/macros_process_mining/src/lib.rs @@ -325,16 +325,16 @@ pub fn register_binding(args: TokenStream, item: TokenStream) -> TokenStream { let serialization_logic = if attrs.debug_output { quote! { let final_result = format!("{:?}", result); - serde_json::to_value(final_result).map_err(|e| e.to_string()) + serde_json::to_vec(&final_result).map_err(|e| e.to_string()) } } else if attrs.stringify_error { quote! { let ok_result = result.map_err(|e| e.to_string())?; - serde_json::to_value(ok_result).map_err(|e| e.to_string()) + serde_json::to_vec(&ok_result).map_err(|e| e.to_string()) } } else { quote! { - serde_json::to_value(result).map_err(|e| e.to_string()) + serde_json::to_vec(&result).map_err(|e| e.to_string()) } }; @@ -398,7 +398,7 @@ pub fn register_binding(args: TokenStream, item: TokenStream) -> TokenStream { quote! { let id = format!("res_{}", uuid::Uuid::new_v4()); __state_guard.insert(id.clone(), crate::bindings::RegistryItem::#variant_ident(result)); - serde_json::to_value(id).map_err(|e| e.to_string()) + serde_json::to_vec(&id).map_err(|e| e.to_string()) } } else { serialization_logic.clone() @@ -421,7 +421,7 @@ pub fn register_binding(args: TokenStream, item: TokenStream) -> TokenStream { }; let id = format!("res_{}", uuid::Uuid::new_v4()); state_lock.add(&id, crate::bindings::RegistryItem::#variant_ident(result)); - serde_json::to_value(id).map_err(|e| e.to_string()) + serde_json::to_vec(&id).map_err(|e| e.to_string()) } } else { quote! { @@ -468,7 +468,7 @@ pub fn register_binding(args: TokenStream, item: TokenStream) -> TokenStream { use serde_json::Value; use std::sync::RwLock; - fn #wrapper_name(args: &Value, state_lock: &AppState) -> Result { + fn #wrapper_name(args: &Value, state_lock: &AppState) -> Result, String> { let arg_map = args.as_object().ok_or("Args must be JSON object")?; #execution_block } diff --git a/process_mining/Cargo.toml b/process_mining/Cargo.toml index 6c5ecf90..bd13e8b6 100644 --- a/process_mining/Cargo.toml +++ b/process_mining/Cargo.toml @@ -19,6 +19,7 @@ chrono = { version = "0.4.40", features = ["serde"] } duckdb = { version = "1.2.1", optional = true, features = ["chrono"]} flate2 = "1.1.1" graphviz-rust = { version = "0.9.3", optional = true } +hashbrown = "0.15" itertools = { version = "0.14.0" } kuzu = {version = "=0.11.2", optional = true} nalgebra = { version = "0.33.2", optional = true } diff --git a/process_mining/examples/ocel_stats.rs b/process_mining/examples/ocel_stats.rs index 1e147448..6ec77d38 100644 --- a/process_mining/examples/ocel_stats.rs +++ b/process_mining/examples/ocel_stats.rs @@ -1,7 +1,11 @@ -use process_mining::{Importable, OCEL}; +use process_mining::core::event_data::object_centric::linked_ocel::{ + LinkedOCELAccess, SlimLinkedOCEL, +}; +use process_mining::Importable; use std::env; use std::error::Error; use std::path::PathBuf; +use std::time::Instant; fn main() -> Result<(), Box> { let args: Vec = env::args().collect(); @@ -12,25 +16,32 @@ fn main() -> Result<(), Box> { let path = PathBuf::from(&args[1]); println!("Importing OCEL from {:?}", path); + let now = Instant::now(); + let ocel = SlimLinkedOCEL::import_from_path(&path)?; + println!("Successfully imported OCEL in {:?}.", now.elapsed()); + println!("Number of events: {}", ocel.get_all_evs().count()); + println!("Number of objects: {}", ocel.get_all_obs().count()); - let ocel = OCEL::import_from_path(&path)?; - println!("Successfully imported OCEL."); - println!("Number of events: {}", ocel.events.len()); - println!("Number of objects: {}", ocel.objects.len()); - - println!( - "Event Types: {:?}", - ocel.event_types - .iter() - .map(|et| &et.name) - .collect::>() - ); + println!("Event Types: {:?}", ocel.get_ev_types().collect::>()); println!( "Object Types: {:?}", - ocel.object_types - .iter() - .map(|ot| &ot.name) - .collect::>() + ocel.get_ob_types().collect::>() ); + + let preview_n = 10; + println!("First {} events:", preview_n); + for ev in ocel.get_all_evs().take(preview_n) { + let ev_type = ocel.get_ev_type_of(ev); + let timestamp = ocel.get_ev_time(ev); + println!( + "Event {:?}: Type: {}, Timestamp: {}", + ev, ev_type, timestamp + ); + let attrs = ocel.get_ev_attrs(ev); + for attr in attrs { + let val = ocel.get_ev_attr_val(ev, attr); + println!(" Attribute: {} = {:?}", attr, val); + } + } Ok(()) } diff --git a/process_mining/src/analysis/object_centric/mod.rs b/process_mining/src/analysis/object_centric/mod.rs index 2d635921..d850e927 100644 --- a/process_mining/src/analysis/object_centric/mod.rs +++ b/process_mining/src/analysis/object_centric/mod.rs @@ -1,3 +1,5 @@ //! Object-centric Process Analysis pub mod object_attribute_changes; +pub mod oc_performance; +pub mod oc_statistics; diff --git a/process_mining/src/analysis/object_centric/oc_performance.rs b/process_mining/src/analysis/object_centric/oc_performance.rs new file mode 100644 index 00000000..b658851a --- /dev/null +++ b/process_mining/src/analysis/object_centric/oc_performance.rs @@ -0,0 +1,152 @@ +//! Object-centric performance analysis over [`SlimLinkedOCEL`]: per-event sojourn and +//! synchronization times. + +use macros_process_mining::register_binding; +use rayon::prelude::*; + +use crate::core::event_data::object_centric::linked_ocel::{ + slim_linked_ocel::{EventIndex, ObjectIndex}, + LinkedOCELAccess, SlimLinkedOCEL, +}; + +/// Each object's reverse-E2O events in `(time, id)` order. +/// +/// Events are sorted per call, so this makes no assumption about global event ordering +fn sorted_events_per_object(ocel: &SlimLinkedOCEL) -> Vec> { + (0..ocel.get_num_obs() as u32) + .into_par_iter() + .map(|i| { + let mut evs: Vec = + ObjectIndex::from(i).get_e2o_rev(ocel).copied().collect(); + evs.sort_by(|a, b| { + a.get_time(ocel) + .cmp(b.get_time(ocel)) + .then_with(|| ocel.get_ev_id(a).cmp(ocel.get_ev_id(b))) + }); + evs + }) + .collect() +} + +/// The `(time, id)`-immediate predecessor of `e` on object `o`, using the +/// per-object sorted lists from [`sorted_events_per_object`]. +/// `None` if `e` is the first event on `o`. +#[inline] +fn df_predecessor( + sorted: &[Vec], + ocel: &SlimLinkedOCEL, + e: EventIndex, + o: ObjectIndex, +) -> Option { + let evs = &sorted[o.into_inner() as usize]; + let key = (e.get_time(ocel), ocel.get_ev_id(&e)); + let pos = evs + .binary_search_by(|x| (x.get_time(ocel), ocel.get_ev_id(x)).cmp(&key)) + .ok()?; + pos.checked_sub(1).map(|p| evs[p]) +} + +/// Per-event synchronization time and the delaying object. +/// +/// For each event with at least one directly-follows predecessor, the synchronization time is +/// `max_predecessor_time - min_predecessor_time` in integer microseconds (the span between its +/// earliest and latest directly-preceding event). The delaying object is the object linking the +/// latest predecessor (ties broken by ascending object id). +/// Returns one row `(event_id, sync_us, delaying_object_id)` per qualifying event. +/// +/// `top_k`: if `Some(k)`, return only the `k` rows with the largest `sync_us`, ties broken by +/// ascending event id, sorted descending. `None` returns every qualifying event. +#[register_binding] +pub fn locel_oc_perf_sync_per_event( + ocel: &SlimLinkedOCEL, + #[bind(default)] top_k: Option, +) -> Vec<(String, i64, String)> { + let sorted = sorted_events_per_object(ocel); + let mut rows: Vec<(EventIndex, i64, ObjectIndex)> = (0..ocel.get_num_evs() as u32) + .into_par_iter() + .filter_map(|i| { + let e = EventIndex::from(i); + let mut min_us = i64::MAX; + // (latest predecessor time, its object) = the delaying edge. + let mut delaying: Option<(i64, ObjectIndex)> = None; + for &o in e.get_e2o(ocel) { + if let Some(p) = df_predecessor(&sorted, ocel, e, o) { + let t = p.get_time(ocel).timestamp_micros(); + min_us = min_us.min(t); + let keep = match delaying { + Some((bt, bo)) => { + bt > t || (bt == t && ocel.get_ob_id(&bo) <= ocel.get_ob_id(&o)) + } + None => false, + }; + if !keep { + delaying = Some((t, o)); + } + } + } + delaying.map(|(max_us, o)| (e, max_us - min_us, o)) + }) + .collect(); + if let Some(k) = top_k { + let cmp = |a: &(EventIndex, i64, ObjectIndex), b: &(EventIndex, i64, ObjectIndex)| { + b.1.cmp(&a.1) + .then_with(|| ocel.get_ev_id(&a.0).cmp(ocel.get_ev_id(&b.0))) + }; + if k < rows.len() { + rows.select_nth_unstable_by(k, cmp); + rows.truncate(k); + } + rows.sort_unstable_by(cmp); + } + rows.into_iter() + .map(|(e, max_minus_min, o)| { + ( + ocel.get_ev_id(&e).to_string(), + max_minus_min, + ocel.get_ob_id(&o).to_string(), + ) + }) + .collect() +} + +/// Per-event sojourn time. +/// +/// For each event with at least one directly-follows predecessor, the sojourn time is +/// `event_time - latest_predecessor_time` in integer microseconds. Returns one row +/// `(event_id, sojourn_us)` per qualifying event. +/// +/// `top_k`: if `Some(k)`, return only the `k` rows with the largest `sojourn_us`, ties broken by +/// ascending event id, sorted descending. `None` returns every qualifying event. +#[register_binding] +pub fn locel_oc_perf_sojourn_per_event( + ocel: &SlimLinkedOCEL, + #[bind(default)] top_k: Option, +) -> Vec<(String, i64)> { + let sorted = sorted_events_per_object(ocel); + let mut rows: Vec<(EventIndex, i64)> = (0..ocel.get_num_evs() as u32) + .into_par_iter() + .filter_map(|i| { + let e = EventIndex::from(i); + let latest = e + .get_e2o(ocel) + .filter_map(|&o| df_predecessor(&sorted, ocel, e, o)) + .map(|p| p.get_time(ocel).timestamp_micros()) + .max()?; + Some((e, e.get_time(ocel).timestamp_micros() - latest)) + }) + .collect(); + if let Some(k) = top_k { + let cmp = |a: &(EventIndex, i64), b: &(EventIndex, i64)| { + b.1.cmp(&a.1) + .then_with(|| ocel.get_ev_id(&a.0).cmp(ocel.get_ev_id(&b.0))) + }; + if k < rows.len() { + rows.select_nth_unstable_by(k, cmp); + rows.truncate(k); + } + rows.sort_unstable_by(cmp); + } + rows.into_iter() + .map(|(e, sojourn_us)| (ocel.get_ev_id(&e).to_string(), sojourn_us)) + .collect() +} diff --git a/process_mining/src/analysis/object_centric/oc_statistics.rs b/process_mining/src/analysis/object_centric/oc_statistics.rs new file mode 100644 index 00000000..0de8af63 --- /dev/null +++ b/process_mining/src/analysis/object_centric/oc_statistics.rs @@ -0,0 +1,79 @@ +//! Descriptive statistics over object-centric event data (E2O type counts, conversion rates). + +use std::collections::HashMap; + +use macros_process_mining::register_binding; +use rayon::prelude::*; + +use crate::core::event_data::object_centric::linked_ocel::{ + slim_linked_ocel::{EventIndex, ObjectIndex}, + LinkedOCELAccess, SlimLinkedOCEL, +}; + +/// Count E2O relationships per `(event_type, object_type)` pair. +/// +/// Each entry is `(event_type, object_type, count)`. A single `(event, object)` pair connected +/// by multiple qualifiers contributes once per qualifier. Pairs with zero relations are omitted. +/// Result row order is unspecified. +#[register_binding] +pub fn locel_event_object_type_counts(ocel: &SlimLinkedOCEL) -> Vec<(String, String, i64)> { + let num_events = ocel.get_num_evs() as u32; + let counts: HashMap<(usize, usize), i64> = (0..num_events) + .into_par_iter() + .fold(HashMap::new, |mut acc, i| { + let ev = EventIndex::from(i).get_ev(ocel); + for (_q, ob) in &ev.relationships { + let ot = ob.get_ob(ocel).object_type; + *acc.entry((ev.event_type, ot)).or_insert(0) += 1; + } + acc + }) + .reduce(HashMap::new, merge_sum_maps); + let ev_types: Vec<&str> = ::get_ev_types(ocel).collect(); + let ob_types: Vec<&str> = ::get_ob_types(ocel).collect(); + counts + .into_iter() + .map(|((e, o), c)| (ev_types[e].to_string(), ob_types[o].to_string(), c)) + .collect() +} + +/// Conversion rate from `source_type` to `target_type` via O2O, restricted to targets touched by `activity`. +/// +/// Returns the fraction of `source_type` objects that have at least one outgoing O2O edge to a +/// `target_type` object related (via E2O) to some event of the given event type. Returns `0.0` +/// if no `source_type` objects exist. +#[register_binding] +pub fn locel_conversion_rate( + ocel: &SlimLinkedOCEL, + activity: String, + source_type: String, + target_type: String, +) -> f64 { + let sources: Vec = ocel.get_obs_of_type(&source_type).copied().collect(); + let total = sources.len(); + if total == 0 { + return 0.0; + } + let reached = sources + .par_iter() + .filter(|&&s| { + s.get_o2o(ocel).any(|&t| { + t.get_ob_type(ocel) == &target_type + && t.get_e2o_rev(ocel) + .any(|&e| e.get_ev_type(ocel) == &activity) + }) + }) + .count(); + reached as f64 / total as f64 +} + +/// Merge `b` into `a` by summing `i64` counts for matching keys. Used as the rayon reduce step. +fn merge_sum_maps( + mut a: HashMap, + b: HashMap, +) -> HashMap { + for (k, v) in b { + *a.entry(k).or_insert(0) += v; + } + a +} diff --git a/process_mining/src/bindings/mod.rs b/process_mining/src/bindings/mod.rs index 144a6fdb..01ec3956 100644 --- a/process_mining/src/bindings/mod.rs +++ b/process_mining/src/bindings/mod.rs @@ -177,7 +177,8 @@ impl RegistryItem { serde_json::to_value(locel).map_err(|e| e.to_string()) } RegistryItem::SlimLinkedOCEL(locel) => { - serde_json::to_value(locel).map_err(|e| e.to_string()) + let ocel = locel.construct_ocel(); + serde_json::to_value(ocel).map_err(|e| e.to_string()) } RegistryItem::EventLogActivityProjection(proj) => { serde_json::to_value(proj).map_err(|e| e.to_string()) @@ -197,9 +198,7 @@ impl RegistryItem { OCEL::import_from_path(path).map_err(|e| e.to_string())?, )), RegistryItemKind::SlimLinkedOCEL => Ok(RegistryItem::SlimLinkedOCEL({ - OCEL::import_from_path(path) - .map(SlimLinkedOCEL::from_ocel) - .map_err(|e| e.to_string())? + SlimLinkedOCEL::import_from_path(path).map_err(|e| e.to_string())? })), RegistryItemKind::IndexLinkedOCEL => Ok(RegistryItem::IndexLinkedOCEL( IndexLinkedOCEL::import_from_path(path).map_err(|e| e.to_string())?, @@ -346,8 +345,9 @@ pub struct Binding { pub id: &'static str, /// Name of the function pub name: &'static str, - /// Function handler (executing the function with (de-)serializing inputs/outputs) - pub handler: fn(&Value, &AppState) -> Result, + /// Function handler (executing the function with (de-)serializing inputs/outputs). + /// Returns the result pre-serialized as UTF-8 JSON bytes. + pub handler: fn(&Value, &AppState) -> Result, String>, /// Documentation of function pub docs: fn() -> Vec, /// Module path of declared function @@ -549,8 +549,9 @@ pub fn resolve_argument( Ok(value) } -/// Call the specified function with the passed arguments -pub fn call(binding: &Binding, args: &Value, state: &AppState) -> Result { +/// Call the specified function with the passed arguments. +/// Returns the result pre-serialized as UTF-8 JSON bytes. +pub fn call(binding: &Binding, args: &Value, state: &AppState) -> Result, String> { (binding.handler)(args, state) } diff --git a/process_mining/src/bindings/slim_ocel_bindings.rs b/process_mining/src/bindings/slim_ocel_bindings.rs index 27957d6d..a1fb03aa 100644 --- a/process_mining/src/bindings/slim_ocel_bindings.rs +++ b/process_mining/src/bindings/slim_ocel_bindings.rs @@ -236,6 +236,14 @@ fn locel_get_e2o_rev(ocel: &SlimLinkedOCEL, ob: ObjectIndex) -> Vec<(String, Eve .collect() } +/// Get the activity trace of an object (i.e., the sequence of event types connected to the object, ordered by event timestamp) +#[register_binding] +fn get_obj_activity_trace(ocel: &SlimLinkedOCEL, ob: ObjectIndex) -> Vec { + ob.get_obj_activity_trace(ocel) + .map(|act| act.to_string()) + .collect() +} + /// Get the outgoing O2O relationships of an object as `(qualifier, object_index)` pairs. #[register_binding] fn locel_get_o2o(ocel: &SlimLinkedOCEL, ob: ObjectIndex) -> Vec<(String, ObjectIndex)> { @@ -301,3 +309,62 @@ fn locel_get_ob_attr_vals( fn locel_construct_ocel(ocel: &SlimLinkedOCEL) -> OCEL { ocel.construct_ocel() } + +#[register_binding] +fn get_object_ids_of_type(ocel: &SlimLinkedOCEL, ob_type: String) -> Vec { + ocel.get_obs_of_type(&ob_type) + .map(|ob| ocel.get_ob_id(ob).to_string()) + .collect() +} + +#[register_binding] +fn get_event_ids_of_type(ocel: &SlimLinkedOCEL, ev_type: String) -> Vec { + ocel.get_evs_of_type(&ev_type) + .map(|ev| ocel.get_ev_id(ev).to_string()) + .collect() +} + +#[register_binding] +fn get_object_type_of_id(ocel: &SlimLinkedOCEL, ob_id: &String) -> Option { + ocel.get_ob_by_id(ob_id) + .map(|ob| ob.get_ob_type(ocel).to_string()) +} + +#[register_binding] +fn get_e2o_rev_ids(ocel: &SlimLinkedOCEL, ob_id: &String) -> Option> { + ocel.get_ob_by_id(ob_id).map(|ob| { + ob.get_e2o_rev(ocel) + .map(|ev| ocel.get_ev_id(ev).to_string()) + .collect() + }) +} + +#[register_binding] +fn get_e2o_ids(ocel: &SlimLinkedOCEL, ev_id: &String) -> Option> { + ocel.get_ev_by_id(ev_id).map(|ev| { + ev.get_e2o(ocel) + .map(|ob| ocel.get_ob_id(ob).to_string()) + .collect() + }) +} + +#[register_binding] +fn get_o2o_ids(ocel: &SlimLinkedOCEL, ob_id: &String) -> Option> { + ocel.get_ob_by_id(ob_id).map(|ob| { + ob.get_o2o(ocel) + .map(|target_ob| ocel.get_ob_id(target_ob).to_string()) + .collect() + }) +} + +#[register_binding] +fn get_event_type_of_id(ocel: &SlimLinkedOCEL, ev_id: &String) -> Option { + ocel.get_ev_by_id(ev_id) + .map(|ev| ev.get_ev_type(ocel).to_string()) +} + +#[register_binding] +fn get_event_timestamp_of_id(ocel: &SlimLinkedOCEL, ev_id: &String) -> Option { + ocel.get_ev_by_id(ev_id) + .map(|ev| ev.get_time(ocel).to_string()) +} diff --git a/process_mining/src/core/event_data/case_centric/dataframe/mod.rs b/process_mining/src/core/event_data/case_centric/dataframe/mod.rs index 898d994e..11645718 100644 --- a/process_mining/src/core/event_data/case_centric/dataframe/mod.rs +++ b/process_mining/src/core/event_data/case_centric/dataframe/mod.rs @@ -53,8 +53,11 @@ fn attribute_value_to_any_value<'a>(from: &AttributeValue) -> AnyValue<'a> { // Fallback for testing: // return AnyValue::StringOwned(v.to_string().into()); AnyValue::Datetime( - v.timestamp_nanos_opt().unwrap(), - polars::prelude::TimeUnit::Nanoseconds, + v.timestamp_micros(), + polars::prelude::TimeUnit::Microseconds, + // Unfortunately, we cannot pass fixed timezone offsets here. + // Polars would also not support mixed timezones in a single column. + // Thus, timezone information is lost, but the timestamps are still correct (i.e., the right moment in time) None, ) } @@ -215,12 +218,21 @@ fn any_value_to_attribute_value(from: &AnyValue<'_>) -> AttributeValue { AnyValue::Int64(v) => AttributeValue::Int(*v), AnyValue::Float32(v) => AttributeValue::Float((*v).into()), AnyValue::Float64(v) => AttributeValue::Float(*v), - AnyValue::Datetime(ns, _, _) => { - // Convert nanos to micros; tz is not used! - let d: DateTime<_> = DateTime::from_timestamp_micros(ns / 1000) - .unwrap() - .fixed_offset(); - AttributeValue::Date(d) + AnyValue::Datetime(v, tu, tz) => { + let d = match tu { + TimeUnit::Nanoseconds => Some(DateTime::from_timestamp_nanos(*v)), + TimeUnit::Microseconds => DateTime::from_timestamp_micros(*v), + TimeUnit::Milliseconds => DateTime::from_timestamp_millis(*v), + }; + if let Some(d) = d { + if let Some(tz) = tz.and_then(|tz| tz.to_chrono().ok()) { + let d_tz = d.with_timezone(&tz); + return AttributeValue::Date(d_tz.fixed_offset()); + } else { + return AttributeValue::Date(d.fixed_offset()); + } + } + AttributeValue::None() } x => AttributeValue::String(format!("{x:?}")), } diff --git a/process_mining/src/core/event_data/case_centric/utils/activity_projection.rs b/process_mining/src/core/event_data/case_centric/utils/activity_projection.rs index e8f8fe38..051b6aa5 100644 --- a/process_mining/src/core/event_data/case_centric/utils/activity_projection.rs +++ b/process_mining/src/core/event_data/case_centric/utils/activity_projection.rs @@ -45,6 +45,7 @@ pub struct EventLogActivityProjection { /// Traces in the event log projection /// /// Each pair represents one activity projection and the number of occurences in the log + /// Sorted by frequency (descending) pub traces: Vec<(Vec, u64)>, } @@ -80,10 +81,13 @@ impl<'a> From<&mut XESParsingTraceStream<'a>> for EventLogActivityProjection { *traces.entry(trace_acts).or_insert(0) += 1; } + let mut traces: Vec<_> = traces.into_iter().collect(); + // Sort by frequency (descending) once + traces.sort_by_key(|(_, freq)| std::cmp::Reverse(*freq)); Self { activities, act_to_index, - traces: traces.into_iter().collect(), + traces, } } } @@ -142,17 +146,21 @@ impl From<&EventLog> for EventLogActivityProjection { *traces_set.entry(trace).or_insert(0) += 1; }); + let mut traces: Vec<_> = traces_set.into_iter().collect(); + traces.sort_by_key(|(_, freq)| std::cmp::Reverse(*freq)); EventLogActivityProjection { activities, act_to_index, - traces: traces_set.into_iter().collect(), + traces, } } } impl EventLogActivityProjection { - /// Convenience function to get sorted activity name lists back from a list of `acts` - pub fn acts_to_names(&self, acts: &[usize]) -> Vec { + /// Reconstructs sorted activity name from a list of indices + /// + /// Uses the internal index -> activity mapping. + pub fn acts_to_names_sorted(&self, acts: &[usize]) -> Vec { let mut ret: Vec = acts .iter() .map(|act| self.activities[*act].clone()) @@ -160,6 +168,95 @@ impl EventLogActivityProjection { ret.sort(); ret } + + /// Convert activity indices to names, preserving the original order + /// + /// Uses the internal index -> activity mapping + pub fn reconstruct_activities(&self, acts: &[usize]) -> Vec { + acts.iter() + .map(|act| self.activities[*act].clone()) + .collect() + } +} + +/// A process variant (activity sequence with its frequency) +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)] +pub struct ProcessVariant { + /// The activity sequence of the variant as activity names + pub activities: Vec, + /// Number of cases corresponding to this variant + pub count: u64, + /// Percentage of total cases corresponding to this variant + pub percentage: f64, +} + +#[register_binding] +/// Get the number of distinct trace variants in the projection +pub fn get_num_variants(projection: &EventLogActivityProjection) -> usize { + projection.traces.len() +} + +#[register_binding] +/// Get the total number of cases in the projection +pub fn get_num_cases(projection: &EventLogActivityProjection) -> u64 { + projection.traces.iter().map(|(_, freq)| freq).sum() +} + +#[register_binding] +/// Get the list of all activity names in the projection +pub fn get_projection_activities(projection: &EventLogActivityProjection) -> &[String] { + &projection.activities +} + +/// Get the most frequent process variants as an iterator, sorted by frequency (descending) +/// +/// Each variant includes the activity sequence as names, its count, and percentage of total cases. +/// +/// Assumes `projection.traces` is sorted by descending frequency, as maintained by the provided +/// constructors. +pub fn get_variants_iter( + projection: &EventLogActivityProjection, +) -> impl Iterator + use<'_> { + let total_cases = get_num_cases(projection); + projection + .traces + .iter() + .map(move |(acts, freq)| ProcessVariant { + activities: projection.reconstruct_activities(acts), + count: *freq, + percentage: if total_cases > 0 { + (*freq as f64 / total_cases as f64) * 100.0 + } else { + 0.0 + }, + }) +} + +#[register_binding] +/// Get all process variants, sorted by frequency (descending) +/// +/// Each variant includes the activity sequence as names, its count, and percentage of total cases. +/// +/// Assumes `projection.traces` is sorted by descending frequency, as maintained by the provided +/// constructors. +pub fn get_variants(projection: &EventLogActivityProjection) -> Vec { + get_variants_iter(projection).collect() +} + +#[register_binding] +/// Get the `n` most frequent process variants, sorted by frequency (descending) +/// +/// Each variant includes the activity sequence as names, its count, and percentage of total cases. +/// To get all variants, pass `n` = [`get_num_variants`] or use the [`get_variants`] function. +/// To get only the most frequent variant, pass `n` = 1. +/// +/// Assumes `projection.traces` is sorted by descending frequency, as maintained by the provided +/// constructors. +pub fn get_top_n_variants( + projection: &EventLogActivityProjection, + n: usize, +) -> Vec { + get_variants_iter(projection).take(n).collect() } /// @@ -372,3 +469,43 @@ impl Exportable for EventLogActivityProjection { vec![ExtensionWithMime::new("json", "application/json")] } } + +#[cfg(test)] +mod test { + use super::*; + use crate::{test_utils::get_test_data_path, EventLog, Importable}; + + #[test] + fn test_variants_rtfm() { + let path = get_test_data_path() + .join("xes") + .join("Road_Traffic_Fine_Management_Process.xes.gz"); + let log = EventLog::import_from_path(path).unwrap(); + let num_cases = log.traces.len(); + let projection = log_to_activity_projection(&log); + assert_eq!(get_num_cases(&projection), num_cases as u64); + let variants = get_variants(&projection); + assert_eq!(variants.len(), 231); + assert_eq!( + &get_top_n_variants(&projection, 2), + &[ + ProcessVariant { + activities: vec![ + "Create Fine".to_string(), + "Send Fine".to_string(), + "Insert Fine Notification".to_string(), + "Add penalty".to_string(), + "Send for Credit Collection".to_string(), + ], + count: 56_482, + percentage: 56_482.0 / num_cases as f64 * 100.0, + }, + ProcessVariant { + activities: vec!["Create Fine".to_string(), "Payment".to_string(),], + count: 46_371, + percentage: 46_371.0 / num_cases as f64 * 100.0, + } + ] + ) + } +} diff --git a/process_mining/src/core/event_data/object_centric/appendable.rs b/process_mining/src/core/event_data/object_centric/appendable.rs new file mode 100644 index 00000000..8c54e11e --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/appendable.rs @@ -0,0 +1,114 @@ +//! Appendable OCEL trait +use std::convert::Infallible; + +use chrono::{DateTime, FixedOffset}; + +use crate::core::event_data::object_centric::ocel_struct::{ + OCELEvent, OCELEventAttribute, OCELObject, OCELObjectAttribute, OCELRelationship, OCELType, + OCEL, +}; + +/// Appendable trait for OCEL data. +/// +/// Handling of misordered input (appends before declarations, late declarations of an +/// already-seen type, forward-referenced relationships) is implementation-defined; see +/// each impl's docs. +pub trait AppendableOCEL { + /// Type of error returned by the `declare_*` / `append_*` methods and `finalize`. + type Error; + + /// Declare an event type. Behavior on re-declaration is implementation-defined. + fn declare_event_type(&mut self, event_type: OCELType) -> Result<(), Self::Error>; + /// Declare an object type. Behavior on re-declaration is implementation-defined. + fn declare_object_type(&mut self, object_type: OCELType) -> Result<(), Self::Error>; + + /// Append an event. + fn append_event( + &mut self, + id: String, + event_type: &str, + time: DateTime, + attributes: Vec, + relationships: Vec, + ) -> Result<(), Self::Error>; + + /// Append an object. + fn append_object( + &mut self, + id: String, + object_type: &str, + attributes: Vec, + relationships: Vec, + ) -> Result<(), Self::Error>; + + /// Resolve any pending forward references. Default impl is a no-op. + fn finalize(&mut self) -> Result<(), Self::Error> { + Ok(()) + } +} + +impl AppendableOCEL for OCEL { + type Error = Infallible; + + fn declare_event_type(&mut self, event_type: OCELType) -> Result<(), Self::Error> { + // Overwrite type if it already exists + if let Some(et) = self + .event_types + .iter_mut() + .find(|et| et.name == event_type.name) + { + *et = event_type; + } else { + self.event_types.push(event_type); + } + Ok(()) + } + + fn declare_object_type(&mut self, object_type: OCELType) -> Result<(), Self::Error> { + // Overwrite type if it already exists + if let Some(ot) = self + .object_types + .iter_mut() + .find(|ot| ot.name == object_type.name) + { + *ot = object_type; + } else { + self.object_types.push(object_type); + } + Ok(()) + } + + fn append_event( + &mut self, + id: String, + event_type: &str, + time: DateTime, + attributes: Vec, + relationships: Vec, + ) -> Result<(), Self::Error> { + self.events.push(OCELEvent { + id, + event_type: event_type.to_string(), + time, + attributes, + relationships, + }); + Ok(()) + } + + fn append_object( + &mut self, + id: String, + object_type: &str, + attributes: Vec, + relationships: Vec, + ) -> Result<(), Self::Error> { + self.objects.push(OCELObject { + id, + object_type: object_type.to_string(), + attributes, + relationships, + }); + Ok(()) + } +} diff --git a/process_mining/src/core/event_data/object_centric/dataframe/mod.rs b/process_mining/src/core/event_data/object_centric/dataframe/mod.rs index c51f327c..ddce13f5 100644 --- a/process_mining/src/core/event_data/object_centric/dataframe/mod.rs +++ b/process_mining/src/core/event_data/object_centric/dataframe/mod.rs @@ -270,8 +270,11 @@ fn ocel_attribute_val_to_any_value(val: &OCELAttributeValue) -> AnyValue<'_> { match val { OCELAttributeValue::String(s) => AnyValue::StringOwned(s.into()), OCELAttributeValue::Time(t) => AnyValue::Datetime( - t.timestamp_nanos_opt().unwrap(), - TimeUnit::Nanoseconds, + t.timestamp_micros(), + TimeUnit::Microseconds, + // Unfortunately, we cannot pass fixed timezone offsets here. + // Polars would also not support mixed timezones in a single column. + // Thus, timezone information is lost, but the timestamps are still correct (i.e., the right moment in time) None, ), OCELAttributeValue::Integer(i) => AnyValue::Int64(*i), @@ -414,11 +417,7 @@ pub fn ocel_to_dataframes(ocel: &OCEL) -> OCELDataFrames { &all_evs_with_rels .iter() .map(|(e, _r)| { - AnyValue::Datetime( - e.time.timestamp_nanos_opt().unwrap(), - TimeUnit::Nanoseconds, - None, - ) + AnyValue::Datetime(e.time.timestamp_micros(), TimeUnit::Microseconds, None) }) .collect::>(), false, @@ -576,8 +575,8 @@ pub fn ocel_to_dataframes(ocel: &OCEL) -> OCELDataFrames { }) .map(|date| { AnyValue::Datetime( - date.timestamp_nanos_opt().unwrap(), - TimeUnit::Nanoseconds, + date.timestamp_micros(), + TimeUnit::Microseconds, None, ) }) @@ -643,8 +642,8 @@ pub fn ocel_to_dataframes(ocel: &OCEL) -> OCELDataFrames { .iter() .map(|o| { AnyValue::Datetime( - o.time.timestamp_nanos_opt().unwrap(), - TimeUnit::Nanoseconds, + o.time.timestamp_micros(), + TimeUnit::Microseconds, None, ) }) @@ -695,10 +694,15 @@ pub fn event_type_to_df<'a, I: LinkedOCELAccess<'a>>( let id_series = Series::from_iter(evs.iter().map(|ev| ev.id.as_str())) .into_column() .with_name("id".into()); + // The cast below reinterprets the i64 value rather than scaling it, + // so the integer must already be in the column's declared TimeUnit + // (microseconds). Using `timestamp_millis()` here previously yielded + // values 1000× too small — every event landed in 1970 because Polars + // read 1.45e12 ms as 1.45e12 µs. let timestamp_series = - Series::from_iter(evs.iter().map(|ev| ev.time.to_utc().timestamp_millis())) + Series::from_iter(evs.iter().map(|ev| ev.time.to_utc().timestamp_micros())) .cast(&polars::prelude::DataType::Datetime( - TimeUnit::Milliseconds, + TimeUnit::Microseconds, Some(TimeZone::UTC), ))? .into_column() @@ -951,9 +955,11 @@ pub fn object_attribute_changes_to_df<'a, I: LinkedOCELAccess<'a>>( ATTRIBUTE_CHANGE_DF_FROM_TIME.into(), &changes .iter() - .map(|c| c.2.map(|t| t.timestamp_millis()).into()) + // µs to match the declared Datetime(Microseconds) column; + // see comment in `event_type_to_df` for context. + .map(|c| c.2.map(|t| t.timestamp_micros()).into()) .collect::>(), - &polars::prelude::DataType::Datetime(TimeUnit::Milliseconds, Some(TimeZone::UTC)), + &polars::prelude::DataType::Datetime(TimeUnit::Microseconds, Some(TimeZone::UTC)), false, )? .into_column(); @@ -961,9 +967,9 @@ pub fn object_attribute_changes_to_df<'a, I: LinkedOCELAccess<'a>>( ATTRIBUTE_CHANGE_DF_TO_TIME.into(), &changes .iter() - .map(|c| c.3.map(|t| t.timestamp_millis()).into()) + .map(|c| c.3.map(|t| t.timestamp_micros()).into()) .collect::>(), - &polars::prelude::DataType::Datetime(TimeUnit::Milliseconds, Some(TimeZone::UTC)), + &polars::prelude::DataType::Datetime(TimeUnit::Microseconds, Some(TimeZone::UTC)), false, )? .into_column(); diff --git a/process_mining/src/core/event_data/object_centric/graph_db/ocel_kuzudb.rs b/process_mining/src/core/event_data/object_centric/graph_db/ocel_kuzudb.rs index 03027c08..912979c2 100644 --- a/process_mining/src/core/event_data/object_centric/graph_db/ocel_kuzudb.rs +++ b/process_mining/src/core/event_data/object_centric/graph_db/ocel_kuzudb.rs @@ -1,3 +1,5 @@ +#[cfg(feature = "dataframes")] +use macros_process_mining::register_binding; use polars::{error::PolarsResult, frame::DataFrame, io::SerWriter, prelude::CsvWriter}; use std::{fs::File, path::Path}; @@ -63,7 +65,6 @@ impl From for KuzuDBExportError { } } -#[cfg(feature = "dataframes")] /// Export an [`OCEL`] as a [kuzu](https://github.com/kuzudb/kuzu) database /// /// This export function does not create different node types for different event/object types @@ -73,8 +74,10 @@ impl From for KuzuDBExportError { /// For E2O relationships, the `E2O` relation is used, pointing from events to objects, with an additional relationship qualifier. /// /// **Limitations**: This function is work-in-progress, currently some aspects (O2O relationships, object attribute changes) are not recorded. -pub fn export_ocel_to_kuzudb_generic>( - db_path: P, +#[cfg(feature = "dataframes")] +#[register_binding(stringify_error)] +pub fn export_ocel_to_kuzudb_generic( + db_path: impl AsRef, ocel: &OCEL, ) -> Result<(), KuzuDBExportError> { use kuzu::{Connection, Database, SystemConfig}; @@ -136,7 +139,15 @@ pub fn export_ocel_to_kuzudb_generic>( fn export_df_to_csv>(df: &mut DataFrame, export_path: P) -> PolarsResult<()> { let f = File::create(export_path)?; - let mut csvw = CsvWriter::new(f); + // Force microsecond fractional seconds + Z suffix so the timestamp + // column round-trips through CSV without losing the µs precision + // that downstream consumers (Kuzu's TIMESTAMP column type, OCEL + // canonicalization invariants) rely on. Polars' default datetime + // format is "%Y-%m-%dT%H:%M:%S" — no fractional seconds — which + // silently rounded everything to whole seconds in the typed Kuzu + // export. + let mut csvw = + CsvWriter::new(f).with_datetime_format(Some("%Y-%m-%dT%H:%M:%S%.6fZ".to_string())); csvw.finish(df)?; Ok(()) } @@ -159,8 +170,9 @@ fn ocel_attribute_type_to_kuzu_dtype(attr_type: &str) -> &'static str { } /// Export an OCEL to a (strictly typed) kuzu database #[cfg(feature = "dataframes")] -pub fn export_ocel_to_kuzudb_typed<'a, P: AsRef>( - db_path: P, +#[register_binding(stringify_error)] +pub fn export_ocel_to_kuzudb_typed<'a>( + db_path: impl AsRef, locel: &'a impl LinkedOCELAccess<'a>, ) -> Result<(), KuzuDBExportError> { use std::fs::remove_file; diff --git a/process_mining/src/core/event_data/object_centric/io.rs b/process_mining/src/core/event_data/object_centric/io.rs index 174570a0..d445853d 100644 --- a/process_mining/src/core/event_data/object_centric/io.rs +++ b/process_mining/src/core/event_data/object_centric/io.rs @@ -107,6 +107,18 @@ impl From for OCELIOError { } } +#[cfg(any(feature = "ocel-duckdb", feature = "ocel-sqlite"))] +impl From for OCELIOError { + fn from(e: DatabaseError) -> Self { + match e { + #[cfg(feature = "ocel-sqlite")] + DatabaseError::SQLITE(e) => OCELIOError::Sqlite(e), + #[cfg(feature = "ocel-duckdb")] + DatabaseError::DUCKDB(e) => OCELIOError::DuckDB(e), + } + } +} + impl Importable for OCEL { type Error = OCELIOError; type ImportOptions = (); @@ -137,7 +149,10 @@ impl Importable for OCEL { options: Self::ImportOptions, ) -> Result { if let Some(inner) = format.strip_suffix(".gz") { - let gz: Box = Box::new(flate2::read::GzDecoder::new(reader)); + // Buffer the compressed bytes; `GzDecoder` reads from its inner in chunks. + let gz: Box = Box::new(flate2::read::GzDecoder::new( + std::io::BufReader::new(reader), + )); return Self::import_from_reader_with_options(gz, inner, options); } if format.ends_with("json") || format.ends_with("jsonocel") { @@ -272,12 +287,7 @@ impl Exportable for OCEL { return crate::core::event_data::object_centric::ocel_sql::export_ocel_sqlite_to_path( self, path, ) - .map_err(|e| match e { - #[cfg(feature = "ocel-sqlite")] - DatabaseError::SQLITE(e) => OCELIOError::Sqlite(e), - #[cfg(feature = "ocel-duckdb")] - DatabaseError::DUCKDB(e) => OCELIOError::DuckDB(e), - }); + .map_err(OCELIOError::from); #[cfg(not(feature = "ocel-sqlite"))] return Err(OCELIOError::UnsupportedFormat( "SQLite support not enabled".to_string(), @@ -288,12 +298,7 @@ impl Exportable for OCEL { crate::core::event_data::object_centric::ocel_sql::export_ocel_duckdb_to_path( self, path, ) - .map_err(|e| match e { - #[cfg(feature = "ocel-sqlite")] - DatabaseError::SQLITE(e) => OCELIOError::Sqlite(e), - #[cfg(feature = "ocel-duckdb")] - DatabaseError::DUCKDB(e) => OCELIOError::DuckDB(e), - }) + .map_err(OCELIOError::from) } #[cfg(not(feature = "ocel-duckdb"))] return Err(OCELIOError::UnsupportedFormat( @@ -338,12 +343,7 @@ impl Exportable for OCEL { { #[cfg(feature = "ocel-sqlite")] { - let b = export_ocel_sqlite_to_vec(self).map_err(|e| match e { - #[cfg(feature = "ocel-sqlite")] - DatabaseError::SQLITE(e) => OCELIOError::Sqlite(e), - #[cfg(feature = "ocel-duckdb")] - DatabaseError::DUCKDB(e) => OCELIOError::DuckDB(e), - })?; + let b = export_ocel_sqlite_to_vec(self).map_err(OCELIOError::from)?; writer.write_all(&b)?; Ok(()) } diff --git a/process_mining/src/core/event_data/object_centric/linked_ocel/slim_linked_ocel.rs b/process_mining/src/core/event_data/object_centric/linked_ocel/slim_linked_ocel.rs index 76bf8572..87e3dd88 100644 --- a/process_mining/src/core/event_data/object_centric/linked_ocel/slim_linked_ocel.rs +++ b/process_mining/src/core/event_data/object_centric/linked_ocel/slim_linked_ocel.rs @@ -4,12 +4,13 @@ use std::{ borrow::{Borrow, Cow}, collections::HashMap, + hash::BuildHasher, io::{Read, Write}, path::Path, }; use chrono::{DateTime, FixedOffset}; -use itertools::Itertools; +use hashbrown::{DefaultHashBuilder, HashTable}; use macros_process_mining::RegistryEntity; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -18,9 +19,14 @@ use uuid::Uuid; use crate::{ core::{ event_data::object_centric::{ - io::OCELIOError, linked_ocel::LinkedOCELAccess, OCELAttributeValue, OCELEvent, - OCELEventAttribute, OCELObject, OCELObjectAttribute, OCELRelationship, OCELType, - OCELTypeAttribute, + appendable::AppendableOCEL, + io::OCELIOError, + linked_ocel::LinkedOCELAccess, + ocel_json::import_ocel_json_into, + ocel_xml::xml_ocel_import::{import_ocel_xml_into, OCELImportOptions}, + readable::{OCELLookup, ReadableOCEL}, + OCELAttributeType, OCELAttributeValue, OCELEvent, OCELEventAttribute, OCELObject, + OCELObjectAttribute, OCELRelationship, OCELType, OCELTypeAttribute, }, io::ExtensionWithMime, OCEL, @@ -28,45 +34,184 @@ use crate::{ Exportable, Importable, }; +/// Interned qualifier identifier. Indexes into [`SlimLinkedOCEL::qualifiers`]. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)] +#[serde(transparent)] +pub struct QualifierIdx(u32); + +impl QualifierIdx { + /// Return the raw `u32` index into the qualifier table. + #[inline] + pub fn into_inner(self) -> u32 { + self.0 + } +} + +/// Insert `item` into `v` keeping `v` sorted ascending; if `item` is already present, +/// leaves `v` unchanged. Used for reverse-relationship lists where multi-qualifier +/// edges between the same pair must contribute a single entry. +fn sorted_insert_unique(v: &mut Vec, item: T) { + if let Err(pos) = v.binary_search(&item) { + v.insert(pos, item); + } +} + +/// Intern `s` into `qualifiers`, returning its index. +fn intern_qualifier( + qualifiers: &mut Vec, + qualifier_index: &mut HashTable, + hasher: &DefaultHashBuilder, + s: String, +) -> QualifierIdx { + let h = hasher.hash_one(&s); + if let Some(&i) = qualifier_index.find(h, |&j| qualifiers[j as usize] == s) { + return QualifierIdx(i); + } + let idx = qualifiers.len() as u32; + qualifiers.push(s); + qualifier_index.insert_unique(h, idx, |&j| hasher.hash_one(&qualifiers[j as usize])); + QualifierIdx(idx) +} + +/// Insert `new_type` or merge into an existing entry. New attributes are appended; existing +/// attributes keep their slot but adopt the new `value_type`. Slots are kept stable because +/// already-appended events/objects index attributes positionally. +/// +/// Values already stored under an attribute whose declared `value_type` is overridden here +/// are left as-is, so the schema's `value_type` and the stored value variant may diverge for +/// items appended before the late declaration. +fn declare_or_merge_type( + types: &mut Vec, + index: &mut HashMap, + per_type: &mut Vec>, + new_type: OCELType, +) { + if let Some(&idx) = index.get(&new_type.name) { + let dst = &mut types[idx].attributes; + for a in new_type.attributes { + match dst.iter_mut().find(|d| d.name == a.name) { + Some(existing) => existing.value_type = a.value_type, + None => dst.push(a), + } + } + return; + } + let idx = types.len(); + index.insert(new_type.name.clone(), idx); + per_type.push(Vec::new()); + types.push(new_type); +} + +/// Returns the index of `name` in `index`, or registers a fresh empty type if unknown. +fn ensure_type_idx( + types: &mut Vec, + index: &mut HashMap, + per_type: &mut Vec>, + name: &str, +) -> usize { + if let Some(&i) = index.get(name) { + return i; + } + let i = types.len(); + index.insert(name.to_string(), i); + per_type.push(Vec::new()); + types.push(OCELType { + name: name.to_string(), + attributes: Vec::new(), + }); + i +} + +/// Reconcile a value already known to the type (i.e., its `name` is declared). +/// +/// If the value's variant agrees with the declared `value_type` (or the value is `Null`, +/// the missing sentinel), the value is returned as-is. Otherwise the function tries +/// [`OCELAttributeValue::try_coerce_to`]; on success the coerced value is returned silently, +/// on failure the original value is returned and a warning is emitted to stderr. +fn reconcile_known_value( + value: OCELAttributeValue, + declared: OCELAttributeType, + owner_kind: &str, + owner_id: &str, + attr_name: &str, +) -> OCELAttributeValue { + if matches!(value, OCELAttributeValue::Null) { + return value; + } + // `OCELAttributeType::Null` here means the declared type string was unrecognized + // (see `OCELAttributeType::from_type_str`); pass the value through unchanged. + if declared == OCELAttributeType::Null { + return value; + } + let observed = value.get_type(); + if declared == observed { + return value; + } + match value.try_coerce_to(declared) { + Some(coerced) => coerced, + None => { + eprintln!( + "[rust4pm] warning: {} {:?} attribute {:?}: value variant {:?} differs from declared type {:?} and cannot be coerced; storing as-is", + owner_kind, + owner_id, + attr_name, + observed.as_type_str(), + declared.as_type_str(), + ); + value + } + } +} + +/// Inner index type for events and objects. +/// +/// The public `EventIndex` and `ObjectIndex` types are thin wrappers around this, providing type safety and OCEL-specific accessors. +pub type InnerIndex = u32; + /// An Event Index /// /// Points to an event in the context of a given OCEL #[derive( PartialEq, Eq, Hash, Clone, Copy, Debug, PartialOrd, Ord, Serialize, Deserialize, JsonSchema, )] -pub struct EventIndex(usize); +pub struct EventIndex(InnerIndex); impl From<&EventIndex> for EventIndex { fn from(value: &EventIndex) -> Self { *value } } -impl From for EventIndex { - fn from(value: usize) -> Self { +impl From for EventIndex { + fn from(value: u32) -> Self { Self(value) } } impl EventIndex { + /// Inner index as `usize` for slice/Vec indexing. + #[inline] + fn ix(self) -> usize { + self.0 as usize + } /// Get the (slim) event referenced by this index in the locel /// /// Note: If there is no event at the specified index, this will access an array out of bounds! /// Use the [`EventIndex::get_ev_opt`] version if you want to handle this explicitly. pub fn get_ev<'a>(&self, locel: &'a SlimLinkedOCEL) -> &'a SlimOCELEvent { - &locel.events[self.0] + &locel.events[self.ix()] } /// Get the (slim) event referenced by this index in the locel /// /// This version explicitly handles scenarios where the event might not exist. /// In case you are sure that the object exists, use the [`EventIndex::get_ev`] function instead. pub fn get_ev_opt<'a>(&self, locel: &'a SlimLinkedOCEL) -> Option<&'a SlimOCELEvent> { - locel.events.get(self.0) + locel.events.get(self.ix()) } /// Get the event type of the event referenced through this event index pub fn get_ev_type<'a>(&self, locel: &'a SlimLinkedOCEL) -> &'a String { - &locel.event_types[locel.events[self.0].event_type].name + &locel.event_types[locel.events[self.ix()].event_type].name } /// Get the timestamp of this event pub fn get_time<'a>(&self, locel: &'a SlimLinkedOCEL) -> &'a DateTime { - &locel.events[self.0].time + &locel.events[self.ix()].time } /// Get E2O relationships of this event pub fn get_e2o<'a>( @@ -75,10 +220,9 @@ impl EventIndex { ) -> impl Iterator + use<'a> { locel .events - .get(self.0) + .get(self.ix()) .into_iter() .flat_map(|e| e.relationships.iter().map(|(_q, o)| o)) - // .copied() } /// Get an attribute value of this event, specified by the attribute name /// @@ -105,7 +249,7 @@ impl EventIndex { attr_name: &str, locel: &'a mut SlimLinkedOCEL, ) -> Option<&'a mut OCELAttributeValue> { - let ev = &mut locel.events[self.0]; + let ev = &mut locel.events[self.ix()]; let (index, _attr) = locel.event_types[ev.event_type] .attributes .iter() @@ -139,18 +283,17 @@ impl EventIndex { .relationships .iter() .map(|(q, o)| OCELRelationship { - object_id: locel.objects[o.into_inner()].id.clone(), - qualifier: q.to_string(), + object_id: locel.objects[o.into_inner() as usize].id.clone(), + qualifier: locel.qualifier_str(*q).to_string(), }) .collect(), } } -} -impl EventIndex { + /// Retrieve inner index value /// /// Warning: Only use carefully, as wrong usage can lead to invalid `EventIndex` references, even when using only a single OCEL - pub fn into_inner(self) -> usize { + pub fn into_inner(self) -> InnerIndex { self.0 } } @@ -161,36 +304,41 @@ impl EventIndex { /// An Object Index /// /// Points to an object in the context of a given OCEL -pub struct ObjectIndex(usize); +pub struct ObjectIndex(InnerIndex); impl From<&ObjectIndex> for ObjectIndex { fn from(value: &ObjectIndex) -> Self { *value } } -impl From for ObjectIndex { - fn from(value: usize) -> Self { +impl From for ObjectIndex { + fn from(value: InnerIndex) -> Self { Self(value) } } impl ObjectIndex { + /// Inner index as `usize` for slice/Vec indexing. + #[inline] + fn ix(self) -> usize { + self.0 as usize + } /// Get the (slim) object referred to by this index in the locel /// /// Note: If there is no object at the specified index, this will access an array out of bounds! /// Use the [`ObjectIndex::get_ob_opt`] version if you want to handle this explicitly. pub fn get_ob<'a>(&self, locel: &'a SlimLinkedOCEL) -> &'a SlimOCELObject { - &locel.objects[self.0] + &locel.objects[self.ix()] } /// Get the (slim) object referred to by this index in the locel /// /// This version explicitly handles scenarios where the object might not exist. /// In case you are sure that the object exists, use the [`ObjectIndex::get_ob`] function instead. pub fn get_ob_opt<'a>(&self, locel: &'a SlimLinkedOCEL) -> Option<&'a SlimOCELObject> { - locel.objects.get(self.0) + locel.objects.get(self.ix()) } /// Get the object type of the object referenced through this object index pub fn get_ob_type<'a>(&self, locel: &'a SlimLinkedOCEL) -> &'a String { - &locel.object_types[locel.objects[self.0].object_type].name + &locel.object_types[locel.objects[self.ix()].object_type].name } /// Get O2O relationships pub fn get_o2o<'a>( @@ -199,11 +347,10 @@ impl ObjectIndex { ) -> impl Iterator + use<'a> { locel .objects - .get(self.0) + .get(self.ix()) .into_iter() .flat_map(|o| &o.relationships) .map(|(_q, o)| o) - // .copied() } /// Get reverse O2O relationships pub fn get_o2o_rev<'a>( @@ -211,12 +358,10 @@ impl ObjectIndex { locel: &'a SlimLinkedOCEL, ) -> impl Iterator + use<'a> { locel - .o2o_rel_rev - .get(self.0) + .objects + .get(self.ix()) .into_iter() - .flatten() - .flatten() - // .copied() + .flat_map(|o| o.o2o_rev.iter()) } /// Get reverse E2O relationships pub fn get_e2o_rev<'a>( @@ -224,12 +369,10 @@ impl ObjectIndex { locel: &'a SlimLinkedOCEL, ) -> impl Iterator + use<'a> { locel - .e2o_rel_rev - .get(self.0) + .objects + .get(self.ix()) .into_iter() - .flatten() - .flatten() - // .copied() + .flat_map(|o| o.e2o_rev.iter()) } /// Get reverse E2O relationships of all events with the specified event type /// @@ -240,16 +383,95 @@ impl ObjectIndex { evtype: &'a str, ) -> impl Iterator + use<'a> { let evtype_index = locel.evtype_to_index.get(evtype).copied(); - let ob_idx = self.0; + let this = *self; evtype_index.into_iter().flat_map(move |ei| { locel - .e2o_rel_rev - .get(ob_idx) + .objects + .get(this.ix()) .into_iter() - .flat_map(move |x| x.get(ei)) - .flatten() + .flat_map(|o| o.e2o_rev.iter()) + .filter(move |ev| locel.events[ev.ix()].event_type == ei) }) } + /// Get reverse O2O source objects of the specified type, optionally filtered by qualifier. + /// + /// When `qualifier` is `None`, iterates `o2o_rev` directly (no relationship re-scan). + /// When `qualifier` is `Some(q)`, only yields sources that have a relationship to this + /// object with that qualifier. + pub fn get_o2o_rev_obs_of_obtype<'a>( + &self, + locel: &'a SlimLinkedOCEL, + obtype: &'a str, + qualifier: Option<&'a str>, + ) -> impl Iterator + use<'a> { + let obtype_index = locel.obtype_to_index.get(obtype).copied(); + let this = *self; + obtype_index.into_iter().flat_map(move |oi| { + locel + .objects + .get(this.ix()) + .into_iter() + .flat_map(|o| o.o2o_rev.iter()) + .filter(move |o| locel.objects[o.ix()].object_type == oi) + .filter(move |o| match qualifier { + None => true, + Some(q) => locel.objects[o.ix()] + .relationships + .iter() + .any(|(rq, rt)| *rt == this && locel.qualifier_str(*rq) == q), + }) + }) + } + /// Get reverse E2O source events of the specified type, optionally filtered by qualifier. + /// + /// When `qualifier` is `None`, iterates `e2o_rev` directly (no relationship re-scan). + /// When `qualifier` is `Some(q)`, only yields events that reference this object with that qualifier. + pub fn get_e2o_rev_evs_of_evtype<'a>( + &self, + locel: &'a SlimLinkedOCEL, + evtype: &'a str, + qualifier: Option<&'a str>, + ) -> impl Iterator + use<'a> { + let evtype_index = locel.evtype_to_index.get(evtype).copied(); + let this = *self; + evtype_index.into_iter().flat_map(move |ei| { + locel + .objects + .get(this.ix()) + .into_iter() + .flat_map(|o| o.e2o_rev.iter()) + .filter(move |e| locel.events[e.ix()].event_type == ei) + .filter(move |e| match qualifier { + None => true, + Some(q) => locel.events[e.ix()] + .relationships + .iter() + .any(|(rq, o)| *o == this && locel.qualifier_str(*rq) == q), + }) + }) + } + /// Get the activity trace of this object as event-type indices, ordered by event timestamp + /// + /// Each yielded `usize` is the internal event-type index of an event connected to this object. + /// This is the cheap form of the trace — useful when you intend to group, count, or otherwise compare + /// traces without allocating string copies. Use [`ObjectIndex::get_obj_activity_trace`] if you want + /// the event-type names directly. + pub fn get_obj_activity_trace_evtype_indices<'a>( + &self, + locel: &'a SlimLinkedOCEL, + ) -> impl Iterator + use<'a> { + let mut events: Vec = self.get_e2o_rev(locel).copied().collect(); + events.sort_by_key(|e| *e.get_time(locel)); + events.into_iter().map(move |e| e.get_ev(locel).event_type) + } + /// Get the activity trace of this object (i.e., the sequence of event types connected to the object, ordered by event timestamp) + pub fn get_obj_activity_trace<'a>( + &self, + locel: &'a SlimLinkedOCEL, + ) -> impl Iterator + use<'a> { + self.get_obj_activity_trace_evtype_indices(locel) + .map(move |i| &locel.event_types[i].name) + } /// Get attribute values of this object, specified by the attribute name /// /// Returns [`None`] if there is no such attribute. @@ -275,7 +497,7 @@ impl ObjectIndex { attr_name: &str, locel: &'a mut SlimLinkedOCEL, ) -> Option<&'a mut Vec<(DateTime, OCELAttributeValue)>> { - let ob = &mut locel.objects[self.0]; + let ob = &mut locel.objects[self.ix()]; let (index, _attr) = locel.object_types[ob.object_type] .attributes .iter() @@ -311,25 +533,23 @@ impl ObjectIndex { .relationships .iter() .map(|(q, o)| OCELRelationship { - object_id: locel.objects[o.into_inner()].id.clone(), - qualifier: q.to_string(), + object_id: locel.objects[o.into_inner() as usize].id.clone(), + qualifier: locel.qualifier_str(*q).to_string(), }) .collect(), } } -} -impl ObjectIndex { /// Retrieve inner index value /// /// Warning: Only use carefully, as wrong usage can lead to invalid `ObjectIndex` references, even when using only a single OCEL - pub fn into_inner(self) -> usize { + pub fn into_inner(self) -> InnerIndex { self.0 } } /// Either an event or an object index -#[derive(PartialEq, Eq, Hash, Clone, Copy, Debug, Serialize, Deserialize, JsonSchema)] +#[derive(PartialEq, Eq, Hash, Clone, Copy, Debug, PartialOrd, Ord, Serialize, Deserialize)] pub enum EventOrObjectIndex { /// An event index Event(EventIndex), @@ -347,40 +567,35 @@ impl From for EventOrObjectIndex { } } -fn sorted_insert(vec: &mut Vec, to_add: T, mut f: impl FnMut(&T) -> B) { - if let Err(index) = vec.binary_search_by_key(&f(&to_add), f) { - vec.insert(index, to_add); - } -} -#[derive(Debug, Clone, Serialize, Deserialize, RegistryEntity, Default)] +#[derive(Debug, Clone, RegistryEntity, Default)] /// An object-centric event log where events and objects are referenced by integer indices /// ([`EventIndex`] / [`ObjectIndex`]) returned from the `add_*` methods, and each indexed /// event/object is an instance of an event/object type (activity / object class) declared /// beforehand with an ordered list of attributes. pub struct SlimLinkedOCEL { - /// Events events: Vec, - /// Objects objects: Vec, - /// Event types (Activities) event_types: Vec, - /// Object types object_types: Vec, - event_ids_to_index: HashMap, - object_ids_to_index: HashMap, - /// Events per Event Type + /// Event-ID -> [`EventIndex`] lookup. Stores only `u32` indices; the id string + /// lives on [`SlimOCELEvent::id`] and is reached through `events`. + event_ids_to_index: HashTable, + /// Object-ID -> [`ObjectIndex`] lookup. See [`Self::event_ids_to_index`]. + object_ids_to_index: HashTable, + /// Hasher used for [`Self::event_ids_to_index`] and [`Self::object_ids_to_index`]. + hasher: DefaultHashBuilder, events_per_type: Vec>, - /// List of object indices per object type objects_per_type: Vec>, - /// Reverse E2O relationships - /// Split by event type (i.e., first level: object index -> event type index -> List of events) - /// The final list of events should be sorted! - e2o_rel_rev: Vec>>, - /// Reverse O2O Relationships (i.e., first level object index -> object type index -> List of objects) - /// The final list of objects should be sorted! - o2o_rel_rev: Vec>>, evtype_to_index: HashMap, obtype_to_index: HashMap, + /// Distinct relationship qualifiers; relationships carry [`QualifierIdx`] indices into this. + qualifiers: Vec, + /// Qualifier-string -> [`QualifierIdx`] lookup. Indexes into [`Self::qualifiers`]. + qualifier_index: HashTable, + /// Forward E2O references whose target object id was unknown at insert time. + pending_e2o: Vec<(EventIndex, OCELRelationship)>, + /// Forward O2O references whose target object id was unknown at insert time. + pending_o2o: Vec<(ObjectIndex, OCELRelationship)>, } impl SlimLinkedOCEL { /// Create a new empty `SlimLinkedOCEL` @@ -389,136 +604,67 @@ impl SlimLinkedOCEL { pub fn new() -> Self { Self::default() } - /// Convert an unlinked [`OCEL`] to a [`SlimLinkedOCEL`] + /// Convert an unlinked [`OCEL`] to a [`SlimLinkedOCEL`]. /// + /// Events are sorted by time before insertion so that `events_per_type` lists are + /// time-ordered. Duplicate event/object ids are skipped with a warning. Unknown + /// types referenced by an event/object are auto-declared on first use, and + /// attributes not listed in the declared schema cause the schema to grow. pub fn from_ocel(mut ocel: OCEL) -> Self { - let evtype_to_index: HashMap<_, _> = ocel - .event_types - .iter() - .enumerate() - .map(|(i, t)| (t.name.clone(), i)) - .collect(); - let obtype_to_index: HashMap<_, _> = ocel - .object_types - .iter() - .enumerate() - .map(|(i, t)| (t.name.clone(), i)) - .collect(); ocel.events.sort_by_key(|e| e.time); - let event_ids_to_index: HashMap<_, _> = ocel - .events - .iter() - .enumerate() - .map(|(ev_index, e)| (e.id.clone(), EventIndex(ev_index))) - .collect(); - let object_ids_to_index: HashMap<_, _> = ocel - .objects - .iter() - .enumerate() - .map(|(ob_index, o)| (o.id.clone(), ObjectIndex(ob_index))) - .collect(); - - let mut events_per_type: Vec> = vec![Vec::new(); ocel.event_types.len()]; - let mut objects_per_type: Vec> = vec![Vec::new(); ocel.object_types.len()]; - let mut e2o_rel_rev: Vec>> = - vec![vec![Vec::new(); ocel.event_types.len()]; ocel.objects.len()]; - let mut o2o_rel_rev: Vec>> = - vec![vec![Vec::new(); ocel.object_types.len()]; ocel.objects.len()]; - let events: Vec = ocel - .events - .into_iter() - .enumerate() - .map(|(e_i, e)| { - let evtype_index = *evtype_to_index.get(&e.event_type).unwrap(); - let ev_index = EventIndex(e_i); - events_per_type[evtype_index].push(ev_index); - let ev_type = &ocel.event_types[evtype_index]; - SlimOCELEvent { - id: e.id, - event_type: evtype_index, - time: e.time, - attributes: ev_type - .attributes - .iter() - .map(|a| { - e.attributes - .iter() - .find(|ea| ea.name == a.name) - .map(|ea| ea.value.clone()) - .unwrap_or(OCELAttributeValue::Null) - }) - .collect(), - relationships: e - .relationships - .into_iter() - .flat_map(|rel| { - // Side effect: We also insert the reverse relation here! - // The filter_map and ? here prevent invalid O2O/E2O references :( - let rel_obj_id = object_ids_to_index.get(&rel.object_id)?; - e2o_rel_rev[rel_obj_id.into_inner()][evtype_index].push(ev_index); - Some((rel.qualifier, *rel_obj_id)) - }) - // These are sorted! - // In particular, this allows more efficient binary search for checking if an element is related - .sorted_unstable_by_key(|(_q, o)| *o) - .collect(), - } - }) - .collect(); - let objects: Vec = ocel - .objects - .into_iter() - .enumerate() - .map(|(o_i, o)| { - let obtype_index = *obtype_to_index.get(&o.object_type).unwrap(); - let ob_index = ObjectIndex(o_i); - objects_per_type[obtype_index].push(ob_index); - let ob_type = &ocel.object_types[obtype_index]; - SlimOCELObject { - id: o.id, - object_type: obtype_index, - attributes: ob_type - .attributes - .iter() - .map(|a| { - o.attributes - .iter() - .filter(|ea| ea.name == a.name) - .map(|ea| (ea.time, ea.value.clone())) - .collect() - }) - .collect(), - relationships: o - .relationships - .into_iter() - .filter_map(|rel| { - // Side effect: We also insert the reverse relation here! - // The filter_map and ? here prevent invalid O2O/E2O references :( - let rel_obj_id = object_ids_to_index.get(&rel.object_id)?; - o2o_rel_rev[rel_obj_id.into_inner()][obtype_index].push(ob_index); - Some((rel.qualifier, *rel_obj_id)) - }) - // These are sorted! - // In particular, this allows more efficient binary search for checking if an element is related - .sorted_unstable_by_key(|(_q, e)| *e) - .collect(), - } - }) - .collect(); - Self { - events, - objects, - event_types: ocel.event_types, - object_types: ocel.object_types, - object_ids_to_index, - event_ids_to_index, - events_per_type, - objects_per_type, - e2o_rel_rev, - o2o_rel_rev, - evtype_to_index, - obtype_to_index, + let mut linked = SlimLinkedOCEL::new(); + for et in ocel.event_types { + let _ = linked.declare_event_type(et); } + for ot in ocel.object_types { + let _ = linked.declare_object_type(ot); + } + for o in ocel.objects { + let OCELObject { + id, + object_type, + attributes, + relationships, + } = o; + if let Err(e) = linked.append_object(id, &object_type, attributes, relationships) { + eprintln!("[rust4pm] warning: skipping object: {e}"); + } + } + for ev in ocel.events { + let OCELEvent { + id, + event_type, + time, + attributes, + relationships, + } = ev; + if let Err(e) = linked.append_event(id, &event_type, time, attributes, relationships) { + eprintln!("[rust4pm] warning: skipping event: {e}"); + } + } + let _ = linked.finalize(); + linked + } + + /// Resolve a qualifier index to its string form. Panics if `idx` is out of range, so + /// only safe for indices read from this OCEL's own relationship lists. Use + /// [`Self::try_qualifier_str`] for indices that may have come from another OCEL or + /// from a deserialized struct. + #[inline] + pub fn qualifier_str(&self, idx: QualifierIdx) -> &str { + &self.qualifiers[idx.0 as usize] + } + + /// Resolve a qualifier index to its string form, or `None` if `idx` is invalid + #[inline] + pub fn try_qualifier_str(&self, idx: QualifierIdx) -> Option<&str> { + self.qualifiers.get(idx.0 as usize).map(String::as_str) + } + + /// All distinct relationship qualifier strings, indexed by [`QualifierIdx`]. + #[inline] + pub fn qualifiers(&self) -> &[String] { + &self.qualifiers } /// Get all events of the specified event type @@ -547,33 +693,17 @@ impl SlimLinkedOCEL { /// Add a new event type to the OCEL, with the specified attributes pub fn add_event_type(&mut self, event_type: &str, attributes: Vec) { - if self.evtype_to_index.contains_key(event_type) { - return; - } - let new_index = self.event_types.len(); - self.evtype_to_index - .insert(event_type.to_string(), new_index); - self.events_per_type.push(Vec::new()); - self.event_types.push(OCELType { + let _ = self.declare_event_type(OCELType { name: event_type.to_string(), attributes, }); - self.e2o_rel_rev.iter_mut().for_each(|x| x.push(Vec::new())); } /// Add a new object type to the OCEL, with the specified attributes pub fn add_object_type(&mut self, object_type: &str, attributes: Vec) { - if self.obtype_to_index.contains_key(object_type) { - return; - } - let new_index = self.object_types.len(); - self.obtype_to_index - .insert(object_type.to_string(), new_index); - self.objects_per_type.push(Vec::new()); - self.object_types.push(OCELType { + let _ = self.declare_object_type(OCELType { name: object_type.to_string(), attributes, }); - self.o2o_rel_rev.iter_mut().for_each(|x| x.push(Vec::new())); } /// Add a new event to the OCEL @@ -595,22 +725,38 @@ impl SlimLinkedOCEL { time: DateTime, id: Option, mut attributes: Vec, - mut relationships: Vec<(String, ObjectIndex)>, + relationships: Vec<(String, ObjectIndex)>, ) -> Option { let etype = self.evtype_to_index.get(event_type)?; let id = id.unwrap_or_else(|| Uuid::new_v4().to_string()); - if self.event_ids_to_index.contains_key(&id) { + let h = self.hasher.hash_one(&id); + if self + .event_ids_to_index + .find(h, |&j| self.events[j as usize].id == id) + .is_some() + { return None; } - let new_ev_index = EventIndex(self.events.len()); - self.event_ids_to_index.insert(id.clone(), new_ev_index); - self.events_per_type.get_mut(*etype)?.push(new_ev_index); - // Relationships should be sorted - relationships.sort_by_key(|(_q, o)| *o); - for (_q, o) in &relationships { - // Special case: As the event is newly appended and thus currently has the highest index, we know that when added to the end, the E2O-rev list is still sorted - self.e2o_rel_rev[o.0][*etype].push(new_ev_index); + let new_ev_index = EventIndex(self.events.len() as u32); + { + let events = &self.events; + let hasher = &self.hasher; + self.event_ids_to_index + .insert_unique(h, new_ev_index.0, |&j| { + hasher.hash_one(&events[j as usize].id) + }); } + self.events_per_type.get_mut(*etype)?.push(new_ev_index); + let mut interned: Vec<(QualifierIdx, ObjectIndex)> = { + let qualifiers = &mut self.qualifiers; + let qualifier_index = &mut self.qualifier_index; + let hasher = &self.hasher; + relationships + .into_iter() + .map(|(q, o)| (intern_qualifier(qualifiers, qualifier_index, hasher, q), o)) + .collect() + }; + interned.sort_by_key(|(_q, o)| *o); // Pad (or truncate) attributes to expected length; warn on mismatch let expected_attr_len = self.event_types[*etype].attributes.len(); if attributes.len() != expected_attr_len { @@ -624,12 +770,15 @@ impl SlimLinkedOCEL { } attributes.resize_with(expected_attr_len, || OCELAttributeValue::Null); + for (_q, obj) in &interned { + sorted_insert_unique(&mut self.objects[obj.0 as usize].e2o_rev, new_ev_index); + } self.events.push(SlimOCELEvent { id, event_type: *etype, time, attributes, - relationships, + relationships: interned, }); Some(new_ev_index) } @@ -650,26 +799,38 @@ impl SlimLinkedOCEL { object_type: &str, id: Option, mut attributes: Vec, OCELAttributeValue)>>, - mut relationships: Vec<(String, ObjectIndex)>, + relationships: Vec<(String, ObjectIndex)>, ) -> Option { let otype = self.obtype_to_index.get(object_type)?; let id = id.unwrap_or_else(|| Uuid::new_v4().to_string()); - if self.object_ids_to_index.contains_key(&id) { + let h = self.hasher.hash_one(&id); + if self + .object_ids_to_index + .find(h, |&j| self.objects[j as usize].id == id) + .is_some() + { return None; } - let new_ob_index = ObjectIndex(self.objects.len()); - self.e2o_rel_rev - .push(vec![Vec::new(); self.events_per_type.len()]); - self.o2o_rel_rev - .push(vec![Vec::new(); self.objects_per_type.len()]); - self.object_ids_to_index.insert(id.clone(), new_ob_index); - self.objects_per_type.get_mut(*otype)?.push(new_ob_index); - // Relationships should be sorted - relationships.sort_by_key(|(_q, o)| *o); - for (_q, o) in &relationships { - // Special case: As the object is newly appended and thus currently has the highest index, we know that when added to the end, the O2O-rev list is still sorted - self.o2o_rel_rev[o.0][*otype].push(new_ob_index); + let new_ob_index = ObjectIndex(self.objects.len() as u32); + { + let objects = &self.objects; + let hasher = &self.hasher; + self.object_ids_to_index + .insert_unique(h, new_ob_index.0, |&j| { + hasher.hash_one(&objects[j as usize].id) + }); } + self.objects_per_type.get_mut(*otype)?.push(new_ob_index); + let mut interned: Vec<(QualifierIdx, ObjectIndex)> = { + let qualifiers = &mut self.qualifiers; + let qualifier_index = &mut self.qualifier_index; + let hasher = &self.hasher; + relationships + .into_iter() + .map(|(q, o)| (intern_qualifier(qualifiers, qualifier_index, hasher, q), o)) + .collect() + }; + interned.sort_by_key(|(_q, o)| *o); // Pad (or truncate) attributes to expected length; warn on mismatch let expected_attr_len = self.object_types[*otype].attributes.len(); if attributes.len() != expected_attr_len { @@ -683,14 +844,20 @@ impl SlimLinkedOCEL { } attributes.resize_with(expected_attr_len, Vec::new); + for (_q, target) in &interned { + sorted_insert_unique(&mut self.objects[target.0 as usize].o2o_rev, new_ob_index); + } self.objects.push(SlimOCELObject { id, object_type: *otype, attributes, - relationships, + relationships: interned, + e2o_rev: Vec::new(), + o2o_rev: Vec::new(), }); Some(new_ob_index) } + /// Add an E2O relationship between the passed event and object, with the specified qualifier. /// /// Multiple relationships between the same `(event, object)` pair are allowed as long as their @@ -698,19 +865,25 @@ impl SlimLinkedOCEL { /// /// Returns `true` on success, `false` if either index is out of bounds (with a stderr warning). pub fn add_e2o(&mut self, event: EventIndex, object: ObjectIndex, qualifier: String) -> bool { - if event.0 >= self.events.len() || object.0 >= self.objects.len() { + if (event.0 as usize) >= self.events.len() || (object.0 as usize) >= self.objects.len() { eprintln!( "[rust4pm] warning: add_e2o called with invalid index(es) (event={}, object={}); ignored", event.0, object.0 ); return false; } - let evtype_index = self.events[event.0].event_type; - sorted_insert(&mut self.e2o_rel_rev[object.0][evtype_index], event, |x| *x); - let rels = &mut self.events[event.0].relationships; - if !rels.iter().any(|(q, o)| o == &object && q == &qualifier) { + let q_idx = intern_qualifier( + &mut self.qualifiers, + &mut self.qualifier_index, + &self.hasher, + qualifier, + ); + let rels = &mut self.events[event.0 as usize].relationships; + let changed = !rels.iter().any(|(q, o)| o == &object && *q == q_idx); + if changed { let insert_pos = rels.partition_point(|(_q, o)| o < &object); - rels.insert(insert_pos, (qualifier, object)); + rels.insert(insert_pos, (q_idx, object)); + sorted_insert_unique(&mut self.objects[object.0 as usize].e2o_rev, event); } true } @@ -726,23 +899,26 @@ impl SlimLinkedOCEL { to_obj: ObjectIndex, qualifier: String, ) -> bool { - if from_obj.0 >= self.objects.len() || to_obj.0 >= self.objects.len() { + if (from_obj.0 as usize) >= self.objects.len() || (to_obj.0 as usize) >= self.objects.len() + { eprintln!( "[rust4pm] warning: add_o2o called with invalid index(es) (from_obj={}, to_obj={}); ignored", from_obj.0, to_obj.0 ); return false; } - let from_obj_type_index = self.objects[from_obj.0].object_type; - sorted_insert( - &mut self.o2o_rel_rev[to_obj.0][from_obj_type_index], - from_obj, - |x| *x, + let q_idx = intern_qualifier( + &mut self.qualifiers, + &mut self.qualifier_index, + &self.hasher, + qualifier, ); - let rels = &mut self.objects[from_obj.0].relationships; - if !rels.iter().any(|(q, o)| o == &to_obj && q == &qualifier) { + let rels = &mut self.objects[from_obj.0 as usize].relationships; + let changed = !rels.iter().any(|(q, o)| o == &to_obj && *q == q_idx); + if changed { let insert_pos = rels.partition_point(|(_q, o)| o < &to_obj); - rels.insert(insert_pos, (qualifier, to_obj)); + rels.insert(insert_pos, (q_idx, to_obj)); + sorted_insert_unique(&mut self.objects[to_obj.0 as usize].o2o_rev, from_obj); } true } @@ -750,36 +926,43 @@ impl SlimLinkedOCEL { /// /// Returns `true` on success, `false` if either index is out of bounds (with a stderr warning). pub fn delete_e2o(&mut self, event: &EventIndex, object: &ObjectIndex) -> bool { - if event.0 >= self.events.len() || object.0 >= self.objects.len() { + if (event.0 as usize) >= self.events.len() || (object.0 as usize) >= self.objects.len() { eprintln!( "[rust4pm] warning: delete_e2o called with invalid index(es) (event={}, object={}); ignored", event.0, object.0 ); return false; } - let evtype_index = self.events[event.0].event_type; - self.e2o_rel_rev[object.0][evtype_index].retain(|e| e != event); - self.events[event.0] - .relationships - .retain(|(_q, o)| o != object); + let rels = &mut self.events[event.0 as usize].relationships; + let before = rels.len(); + rels.retain(|(_q, o)| o != object); + if rels.len() != before { + self.objects[object.0 as usize] + .e2o_rev + .retain(|e| e != event); + } true } /// Remove all O2O relationships from `from_obj` to `to_obj` (across every qualifier). /// /// Returns `true` on success, `false` if either index is out of bounds (with a stderr warning). pub fn delete_o2o(&mut self, from_obj: &ObjectIndex, to_obj: &ObjectIndex) -> bool { - if from_obj.0 >= self.objects.len() || to_obj.0 >= self.objects.len() { + if (from_obj.0 as usize) >= self.objects.len() || (to_obj.0 as usize) >= self.objects.len() + { eprintln!( "[rust4pm] warning: delete_o2o called with invalid index(es) (from_obj={}, to_obj={}); ignored", from_obj.0, to_obj.0 ); return false; } - let from_obj_type = self.objects[from_obj.0].object_type; - self.o2o_rel_rev[to_obj.0][from_obj_type].retain(|e| e != from_obj); - self.objects[from_obj.0] - .relationships - .retain(|(_q, o)| o != to_obj); + let rels = &mut self.objects[from_obj.0 as usize].relationships; + let before = rels.len(); + rels.retain(|(_q, o)| o != to_obj); + if rels.len() != before { + self.objects[to_obj.0 as usize] + .o2o_rev + .retain(|o| o != from_obj); + } true } } @@ -789,10 +972,11 @@ impl From for SlimLinkedOCEL { } } -/// A slim version of an OCEL Event +/// A slim version of an OCEL Event. /// -/// Some fields (i.e., `event_type` and relationships) are modified for easier and memory-efficient usage -#[derive(Debug, Clone, Serialize, Deserialize)] +/// Qualifier strings in relationships are interned via [`SlimLinkedOCEL::qualifiers`] +/// and referenced here by [`QualifierIdx`] instead of owned `String`s. +#[derive(Debug, Clone, Serialize)] pub struct SlimOCELEvent { /// Event ID pub id: String, @@ -802,16 +986,16 @@ pub struct SlimOCELEvent { /// `DateTime` when event occured pub time: DateTime, /// Event attributes - #[serde(default)] pub attributes: Vec, - /// E2O (Event-to-Object) relationships - #[serde(default)] - pub relationships: Vec<(String, ObjectIndex)>, + /// E2O relationships as `(qualifier_idx, object)` pairs, sorted ascending by + /// [`ObjectIndex`]. Resolve qualifier strings via [`SlimLinkedOCEL::qualifier_str`]. + pub relationships: Vec<(QualifierIdx, ObjectIndex)>, } -/// A slim version of an OCEL Object +/// A slim version of an OCEL Object. /// -/// Some fields (i.e., `object_type` and relationships) are modified for easier and memory-efficient usage -#[derive(Debug, Clone, Serialize, Deserialize)] +/// Qualifier strings in relationships are interned via [`SlimLinkedOCEL::qualifiers`] +/// and referenced here by [`QualifierIdx`] instead of owned `String`s. +#[derive(Debug, Clone, Serialize)] pub struct SlimOCELObject { /// Object ID pub id: String, @@ -819,11 +1003,486 @@ pub struct SlimOCELObject { #[serde(rename = "type")] pub object_type: usize, /// Object attributes (each inner [`Vec`] holds the time-indexed values for one declared attribute) - #[serde(default)] pub attributes: Vec, OCELAttributeValue)>>, - /// O2O (Object-to-Object) relationships - #[serde(default)] - pub relationships: Vec<(String, ObjectIndex)>, + /// O2O relationships as `(qualifier_idx, target_object)` pairs, sorted ascending by + /// [`ObjectIndex`]. Resolve qualifier strings via [`SlimLinkedOCEL::qualifier_str`]. + pub relationships: Vec<(QualifierIdx, ObjectIndex)>, + /// Reverse E2O: events whose forward relationships reference this object. + #[serde(skip)] + pub e2o_rev: Vec, + /// Reverse O2O: source objects whose forward relationships reference this object. + #[serde(skip)] + pub o2o_rev: Vec, +} + +/// Errors returned by [`AppendableOCEL`] operations on [`SlimLinkedOCEL`] +#[derive(Debug)] +pub enum SlimAppendError { + /// Event id already used + DuplicateEventId(String), + /// Object id already used + DuplicateObjectId(String), +} + +impl std::fmt::Display for SlimAppendError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::DuplicateEventId(id) => write!(f, "Duplicate event id: {id}"), + Self::DuplicateObjectId(id) => write!(f, "Duplicate object id: {id}"), + } + } +} + +impl std::error::Error for SlimAppendError {} + +impl From for OCELIOError { + fn from(e: SlimAppendError) -> Self { + OCELIOError::Other(e.to_string()) + } +} + +impl AppendableOCEL for SlimLinkedOCEL { + type Error = SlimAppendError; + + fn declare_event_type(&mut self, event_type: OCELType) -> Result<(), Self::Error> { + declare_or_merge_type( + &mut self.event_types, + &mut self.evtype_to_index, + &mut self.events_per_type, + event_type, + ); + Ok(()) + } + + fn declare_object_type(&mut self, object_type: OCELType) -> Result<(), Self::Error> { + declare_or_merge_type( + &mut self.object_types, + &mut self.obtype_to_index, + &mut self.objects_per_type, + object_type, + ); + Ok(()) + } + + fn append_event( + &mut self, + id: String, + event_type: &str, + time: DateTime, + attributes: Vec, + relationships: Vec, + ) -> Result<(), Self::Error> { + let h_id = self.hasher.hash_one(&id); + if self + .event_ids_to_index + .find(h_id, |&j| self.events[j as usize].id == id) + .is_some() + { + return Err(SlimAppendError::DuplicateEventId(id)); + } + let etype = ensure_type_idx( + &mut self.event_types, + &mut self.evtype_to_index, + &mut self.events_per_type, + event_type, + ); + let new_idx = EventIndex(self.events.len() as u32); + // Auto-declare unknown attribute names. Warn only on schema drift (type already + // had entries); silent during bootstrap of an empty auto-declared type. + let attrs = &mut self.event_types[etype].attributes; + for a in &attributes { + if !attrs.iter().any(|d| d.name == a.name) { + if !attrs.is_empty() { + eprintln!( + "[rust4pm] warning: event {:?} of type {:?} has attribute {:?} not in the existing type schema; auto-growing the type", + id, event_type, a.name + ); + } + attrs.push(OCELTypeAttribute { + name: a.name.clone(), + value_type: a.value.get_type().as_type_str().to_string(), + }); + } + } + let positional: Vec = attrs + .iter() + .map(|d| { + let declared = OCELAttributeType::from_type_str(&d.value_type); + match attributes.iter().find(|a| a.name == d.name) { + Some(a) => { + reconcile_known_value(a.value.clone(), declared, "event", &id, &a.name) + } + None => OCELAttributeValue::Null, + } + }) + .collect(); + let mut resolved: Vec<(QualifierIdx, ObjectIndex)> = + Vec::with_capacity(relationships.len()); + for r in relationships { + let h = self.hasher.hash_one(&r.object_id); + let lookup = self + .object_ids_to_index + .find(h, |&j| self.objects[j as usize].id == r.object_id) + .copied(); + match lookup { + Some(o_idx) => { + let q = intern_qualifier( + &mut self.qualifiers, + &mut self.qualifier_index, + &self.hasher, + r.qualifier, + ); + resolved.push((q, ObjectIndex(o_idx))); + } + None => self.pending_e2o.push((new_idx, r)), + } + } + resolved.sort_by_key(|(_q, o)| *o); + { + let events = &self.events; + let hasher = &self.hasher; + self.event_ids_to_index + .insert_unique(h_id, new_idx.0, |&j| { + hasher.hash_one(&events[j as usize].id) + }); + } + self.events_per_type[etype].push(new_idx); + for (_q, obj) in &resolved { + sorted_insert_unique(&mut self.objects[obj.0 as usize].e2o_rev, new_idx); + } + self.events.push(SlimOCELEvent { + id, + event_type: etype, + time, + attributes: positional, + relationships: resolved, + }); + Ok(()) + } + + fn append_object( + &mut self, + id: String, + object_type: &str, + attributes: Vec, + relationships: Vec, + ) -> Result<(), Self::Error> { + let h_id = self.hasher.hash_one(&id); + if self + .object_ids_to_index + .find(h_id, |&j| self.objects[j as usize].id == id) + .is_some() + { + return Err(SlimAppendError::DuplicateObjectId(id)); + } + let otype = ensure_type_idx( + &mut self.object_types, + &mut self.obtype_to_index, + &mut self.objects_per_type, + object_type, + ); + let new_idx = ObjectIndex(self.objects.len() as u32); + let attrs = &mut self.object_types[otype].attributes; + for a in &attributes { + if !attrs.iter().any(|d| d.name == a.name) { + if !attrs.is_empty() { + eprintln!( + "[rust4pm] warning: object {:?} of type {:?} has attribute {:?} not in the existing type schema; auto-growing the type", + id, object_type, a.name + ); + } + attrs.push(OCELTypeAttribute { + name: a.name.clone(), + value_type: a.value.get_type().as_type_str().to_string(), + }); + } + } + let positional: Vec, OCELAttributeValue)>> = attrs + .iter() + .map(|d| { + let declared = OCELAttributeType::from_type_str(&d.value_type); + attributes + .iter() + .filter(|a| a.name == d.name) + .map(|a| { + let v = reconcile_known_value( + a.value.clone(), + declared, + "object", + &id, + &a.name, + ); + (a.time, v) + }) + .collect() + }) + .collect(); + let mut resolved: Vec<(QualifierIdx, ObjectIndex)> = + Vec::with_capacity(relationships.len()); + for r in relationships { + let h = self.hasher.hash_one(&r.object_id); + let lookup = self + .object_ids_to_index + .find(h, |&j| self.objects[j as usize].id == r.object_id) + .copied(); + match lookup { + Some(o_idx) => { + let q = intern_qualifier( + &mut self.qualifiers, + &mut self.qualifier_index, + &self.hasher, + r.qualifier, + ); + resolved.push((q, ObjectIndex(o_idx))); + } + None => self.pending_o2o.push((new_idx, r)), + } + } + resolved.sort_by_key(|(_q, o)| *o); + { + let objects = &self.objects; + let hasher = &self.hasher; + self.object_ids_to_index + .insert_unique(h_id, new_idx.0, |&j| { + hasher.hash_one(&objects[j as usize].id) + }); + } + self.objects_per_type[otype].push(new_idx); + for (_q, target) in &resolved { + sorted_insert_unique(&mut self.objects[target.0 as usize].o2o_rev, new_idx); + } + self.objects.push(SlimOCELObject { + id, + object_type: otype, + attributes: positional, + relationships: resolved, + e2o_rev: Vec::new(), + o2o_rev: Vec::new(), + }); + Ok(()) + } + + fn finalize(&mut self) -> Result<(), Self::Error> { + // Resolve pending E2O / O2O forward refs and re-sort touched relationship lists. + // The OCEL spec disallows duplicate (source, target, qualifier) triples; not + // deduped here, invalid input flows through as-is. + let mut ev_dirty: Vec = Vec::new(); + for (ev_idx, rel) in std::mem::take(&mut self.pending_e2o) { + let h = self.hasher.hash_one(&rel.object_id); + let target = self + .object_ids_to_index + .find(h, |&j| self.objects[j as usize].id == rel.object_id) + .copied(); + match target { + Some(raw_idx) => { + let ob_idx = ObjectIndex(raw_idx); + let q_idx = intern_qualifier( + &mut self.qualifiers, + &mut self.qualifier_index, + &self.hasher, + rel.qualifier, + ); + self.events[ev_idx.0 as usize] + .relationships + .push((q_idx, ob_idx)); + sorted_insert_unique(&mut self.objects[ob_idx.0 as usize].e2o_rev, ev_idx); + ev_dirty.push(ev_idx); + } + None => { + eprintln!( + "[rust4pm] warning: dropping E2O reference to unknown object id {:?}", + rel.object_id + ); + } + } + } + ev_dirty.sort_unstable(); + ev_dirty.dedup(); + for ev_idx in ev_dirty { + self.events[ev_idx.0 as usize] + .relationships + .sort_by_key(|(_q, o)| *o); + } + + let mut ob_dirty: Vec = Vec::new(); + for (from_idx, rel) in std::mem::take(&mut self.pending_o2o) { + let h = self.hasher.hash_one(&rel.object_id); + let target = self + .object_ids_to_index + .find(h, |&j| self.objects[j as usize].id == rel.object_id) + .copied(); + match target { + Some(raw_idx) => { + let to_idx = ObjectIndex(raw_idx); + let q_idx = intern_qualifier( + &mut self.qualifiers, + &mut self.qualifier_index, + &self.hasher, + rel.qualifier, + ); + self.objects[from_idx.0 as usize] + .relationships + .push((q_idx, to_idx)); + sorted_insert_unique(&mut self.objects[to_idx.0 as usize].o2o_rev, from_idx); + ob_dirty.push(from_idx); + } + None => { + eprintln!( + "[rust4pm] warning: dropping O2O reference to unknown object id {:?}", + rel.object_id + ); + } + } + } + ob_dirty.sort_unstable(); + ob_dirty.dedup(); + for ob_idx in ob_dirty { + self.objects[ob_idx.0 as usize] + .relationships + .sort_by_key(|(_q, o)| *o); + } + + // Streaming append preserves input order; from_ocel pre-sorts events by time. + // Sort here so `events_per_type` is time-ordered regardless of import path. + let events = &self.events; + for per_type in &mut self.events_per_type { + per_type.sort_by_key(|ei| events[ei.0 as usize].time); + } + + Ok(()) + } +} + +impl ReadableOCEL for SlimLinkedOCEL { + type Lookup<'a> = SlimOCELLookup<'a>; + fn event_types(&self) -> &[OCELType] { + &self.event_types + } + fn object_types(&self) -> &[OCELType] { + &self.object_types + } + fn iter_events(&self) -> Box> + '_> { + Box::new((0..self.events.len()).map(|i| Cow::Owned(EventIndex(i as u32).fat_ev(self)))) + } + fn iter_events_sorted_by_time(&self) -> Box> + '_> { + // Streaming append paths may leave events out of order; sort indices, not events. + let mut indices: Vec = (0..self.events.len() as u32).collect(); + indices.sort_by_key(|&i| self.events[i as usize].time); + Box::new( + indices + .into_iter() + .map(move |i| Cow::Owned(EventIndex(i).fat_ev(self))), + ) + } + fn iter_objects(&self) -> Box> + '_> { + Box::new((0..self.objects.len()).map(|i| Cow::Owned(ObjectIndex(i as u32).fat_ob(self)))) + } + fn iter_events_of_type<'a>( + &'a self, + type_name: &'a str, + ) -> Box> + 'a> { + let Some(&idx) = self.evtype_to_index.get(type_name) else { + return Box::new(std::iter::empty()); + }; + Box::new( + self.events_per_type[idx] + .iter() + .map(move |ei| Cow::Owned(ei.fat_ev(self))), + ) + } + fn iter_objects_of_type<'a>( + &'a self, + type_name: &'a str, + ) -> Box> + 'a> { + let Some(&idx) = self.obtype_to_index.get(type_name) else { + return Box::new(std::iter::empty()); + }; + Box::new( + self.objects_per_type[idx] + .iter() + .map(move |oi| Cow::Owned(oi.fat_ob(self))), + ) + } + fn lookup(&self) -> SlimOCELLookup<'_> { + SlimOCELLookup { locel: self } + } +} + +/// Thin wrapper around [`SlimLinkedOCEL`] that satisfies [`OCELLookup`] without +/// materializing any objects. Lookups go straight through the existing index +/// tables and slim object representation. +#[derive(Debug)] +pub struct SlimOCELLookup<'a> { + locel: &'a SlimLinkedOCEL, +} + +impl<'a> SlimOCELLookup<'a> { + fn find_idx(&self, id: &str) -> Option { + let h = self.locel.hasher.hash_one(id); + self.locel + .object_ids_to_index + .find(h, |&j| self.locel.objects[j as usize].id == id) + .map(|&i| i as usize) + } +} + +impl<'a> OCELLookup for SlimOCELLookup<'a> { + fn iter_object_ids<'b>(&'b self) -> Box + 'b> { + Box::new(self.locel.objects.iter().map(|o| o.id.as_str())) + } + fn get_id_borrow(&self, id: &str) -> Option<&str> { + let i = self.find_idx(id)?; + Some(self.locel.objects[i].id.as_str()) + } + fn object_type_of(&self, id: &str) -> Option<&str> { + let i = self.find_idx(id)?; + let slim = &self.locel.objects[i]; + Some(self.locel.object_types[slim.object_type].name.as_str()) + } + fn object_attributes<'b>( + &'b self, + id: &str, + ) -> Box)> + 'b> + { + match self.find_idx(id) { + Some(i) => { + let slim = &self.locel.objects[i]; + let ob_type = &self.locel.object_types[slim.object_type]; + Box::new( + ob_type + .attributes + .iter() + .enumerate() + .flat_map(move |(idx, at)| { + slim.attributes + .get(idx) + .into_iter() + .flatten() + .map(move |(t, v)| (at.name.as_str(), v, *t)) + }), + ) + } + None => Box::new(std::iter::empty()), + } + } + fn object_relationships<'b>( + &'b self, + id: &str, + ) -> Box + 'b> { + match self.find_idx(id) { + Some(i) => { + let slim = &self.locel.objects[i]; + let locel = self.locel; + Box::new(slim.relationships.iter().map(move |(q, target)| { + ( + locel.objects[target.0 as usize].id.as_str(), + locel.qualifier_str(*q), + ) + })) + } + None => Box::new(std::iter::empty()), + } + } } impl<'a> LinkedOCELAccess<'a> for SlimLinkedOCEL { @@ -848,11 +1507,11 @@ impl<'a> LinkedOCELAccess<'a> for SlimLinkedOCEL { } fn get_all_evs(&'a self) -> impl Iterator { - (0..self.events.len()).map(EventIndex) + (0..self.events.len()).map(|i| EventIndex(i as u32)) } fn get_all_obs(&'a self) -> impl Iterator { - (0..self.objects.len()).map(ObjectIndex) + (0..self.objects.len()).map(|i| ObjectIndex(i as u32)) } fn get_full_ev(&'a self, index: impl Borrow) -> Cow<'a, OCELEvent> { @@ -867,24 +1526,23 @@ impl<'a> LinkedOCELAccess<'a> for SlimLinkedOCEL { &'a self, index: impl Borrow, ) -> impl Iterator { - self.events[index.borrow().0] + self.events[index.borrow().0 as usize] .relationships .iter() - .map(|(q, o_idx)| (q.as_str(), o_idx)) + .map(|(q, o_idx)| (self.qualifier_str(*q), o_idx)) } fn get_e2o_rev( &'a self, index: impl Borrow, ) -> impl Iterator { - // `relationships` is sorted by object index; could use partition_point if lists grow large. let target = *index.borrow(); index.borrow().get_e2o_rev(self).flat_map(move |e| { - self.events[e.0] + self.events[e.0 as usize] .relationships .iter() .filter(move |(_q, o)| *o == target) - .map(move |(q, _)| (q.as_str(), e)) + .map(move |(q, _)| (self.qualifier_str(*q), e)) }) } @@ -892,10 +1550,10 @@ impl<'a> LinkedOCELAccess<'a> for SlimLinkedOCEL { &'a self, index: impl Borrow, ) -> impl Iterator { - self.objects[index.borrow().0] + self.objects[index.borrow().0 as usize] .relationships .iter() - .map(|(q, o_idx)| (q.as_str(), o_idx)) + .map(|(q, o_idx)| (self.qualifier_str(*q), o_idx)) } fn get_o2o_rev( @@ -904,11 +1562,11 @@ impl<'a> LinkedOCELAccess<'a> for SlimLinkedOCEL { ) -> impl Iterator { let target = *index.borrow(); index.borrow().get_o2o_rev(self).flat_map(move |o1| { - self.objects[o1.0] + self.objects[o1.0 as usize] .relationships .iter() .filter(move |(_q, o2)| *o2 == target) - .map(move |(q, _)| (q.as_str(), o1)) + .map(move |(q, _)| (self.qualifier_str(*q), o1)) }) } @@ -934,7 +1592,7 @@ impl<'a> LinkedOCELAccess<'a> for SlimLinkedOCEL { fn get_ev_attrs(&'a self, ev: impl Borrow) -> impl Iterator { self.events - .get(ev.borrow().0) + .get(ev.borrow().0 as usize) .and_then(|e| self.event_types.get(e.event_type)) .iter() .flat_map(|et| &et.attributes) @@ -953,7 +1611,7 @@ impl<'a> LinkedOCELAccess<'a> for SlimLinkedOCEL { fn get_ob_attrs(&'a self, ob: impl Borrow) -> impl Iterator { self.objects - .get(ob.borrow().0) + .get(ob.borrow().0 as usize) .and_then(|o| self.object_types.get(o.object_type)) .iter() .flat_map(|et| &et.attributes) @@ -974,19 +1632,27 @@ impl<'a> LinkedOCELAccess<'a> for SlimLinkedOCEL { } fn get_ob_id(&'a self, ob: impl Borrow) -> &'a str { - self.objects[ob.borrow().0].id.as_str() + self.objects[ob.borrow().0 as usize].id.as_str() } fn get_ev_id(&'a self, ev: impl Borrow) -> &'a str { - self.events[ev.borrow().0].id.as_str() + self.events[ev.borrow().0 as usize].id.as_str() } fn get_ev_by_id(&'a self, ev_id: impl AsRef) -> Option { - self.event_ids_to_index.get(ev_id.as_ref()).copied() + let id = ev_id.as_ref(); + let h = self.hasher.hash_one(id); + self.event_ids_to_index + .find(h, |&j| self.events[j as usize].id == id) + .map(|&i| EventIndex(i)) } fn get_ob_by_id(&'a self, ob_id: impl AsRef) -> Option { - self.object_ids_to_index.get(ob_id.as_ref()).copied() + let id = ob_id.as_ref(); + let h = self.hasher.hash_one(id); + self.object_ids_to_index + .find(h, |&j| self.objects[j as usize].id == id) + .map(|&i| ObjectIndex(i)) } fn get_ev_time(&'a self, ev: impl Borrow) -> &'a DateTime { @@ -1000,11 +1666,11 @@ impl<'a> LinkedOCELAccess<'a> for SlimLinkedOCEL { ) -> impl Iterator { let ob_type_index = self.obtype_to_index.get(ob_type.as_ref()); ob_type_index.into_iter().flat_map(move |ot_index| { - self.events[index.borrow().0] + self.events[index.borrow().0 as usize] .relationships .iter() .filter(move |(_q, o)| &o.get_ob(self).object_type == ot_index) - .map(|(q, o)| (q.as_str(), o)) + .map(|(q, o)| (self.qualifier_str(*q), o)) }) } fn get_o2o_of_type( @@ -1014,11 +1680,11 @@ impl<'a> LinkedOCELAccess<'a> for SlimLinkedOCEL { ) -> impl Iterator { let ob_type_index = self.obtype_to_index.get(ob_type.as_ref()); ob_type_index.into_iter().flat_map(move |ot_index| { - self.objects[index.borrow().0] + self.objects[index.borrow().0 as usize] .relationships .iter() .filter(move |(_q, o)| &o.get_ob(self).object_type == ot_index) - .map(|(q, o)| (q.as_str(), o)) + .map(|(q, o)| (self.qualifier_str(*q), o)) }) } @@ -1028,20 +1694,18 @@ impl<'a> LinkedOCELAccess<'a> for SlimLinkedOCEL { ev_type: impl AsRef, ) -> impl Iterator { let evtype_index = self.evtype_to_index.get(ev_type.as_ref()).copied(); - let ob_idx = index.borrow().0; let target = *index.borrow(); evtype_index.into_iter().flat_map(move |ei| { - self.e2o_rel_rev - .get(ob_idx) - .into_iter() - .flat_map(move |x| x.get(ei)) - .flatten() + self.objects[target.0 as usize] + .e2o_rev + .iter() + .filter(move |e| self.events[e.0 as usize].event_type == ei) .flat_map(move |e| { e.get_ev(self) .relationships .iter() .filter(move |(_q, o)| *o == target) - .map(move |(q, _)| (q.as_str(), e)) + .map(move |(q, _)| (self.qualifier_str(*q), e)) }) }) } @@ -1051,20 +1715,18 @@ impl<'a> LinkedOCELAccess<'a> for SlimLinkedOCEL { from_ob_type: impl AsRef, ) -> impl Iterator { let obtype_index = self.obtype_to_index.get(from_ob_type.as_ref()).copied(); - let ob_idx = to_obj.borrow().0; let target = *to_obj.borrow(); obtype_index.into_iter().flat_map(move |oi| { - self.o2o_rel_rev - .get(ob_idx) - .into_iter() - .flat_map(move |x| x.get(oi)) - .flatten() + self.objects[target.0 as usize] + .o2o_rev + .iter() + .filter(move |o| self.objects[o.0 as usize].object_type == oi) .flat_map(move |o| { o.get_ob(self) .relationships .iter() .filter(move |(_q, e)| *e == target) - .map(move |(q, _)| (q.as_str(), o)) + .map(move |(q, _)| (self.qualifier_str(*q), o)) }) }) } @@ -1079,8 +1741,28 @@ impl Importable for SlimLinkedOCEL { format: &str, _: Self::ImportOptions, ) -> Result { - let ocel = OCEL::import_from_reader(reader, format)?; - Ok(SlimLinkedOCEL::from_ocel(ocel)) + if let Some(inner) = format.strip_suffix(".gz") { + // Buffer the compressed bytes; `GzDecoder` reads from its inner in chunks. + let gz: Box = Box::new(flate2::read::GzDecoder::new( + std::io::BufReader::new(reader), + )); + return Self::import_from_reader_with_options(gz, inner, ()); + } + if format.ends_with("xml") || format.ends_with("xmlocel") { + let mut xml_reader = quick_xml::Reader::from_reader(std::io::BufReader::new(reader)); + let mut slim = SlimLinkedOCEL::new(); + import_ocel_xml_into(&mut xml_reader, &mut slim, OCELImportOptions::default())?; + slim.finalize()?; + Ok(slim) + } else if format.ends_with("json") || format.ends_with("jsonocel") { + let mut slim = SlimLinkedOCEL::new(); + import_ocel_json_into(std::io::BufReader::new(reader), &mut slim)?; + slim.finalize()?; + Ok(slim) + } else { + let ocel = OCEL::import_from_reader(reader, format)?; + Ok(SlimLinkedOCEL::from_ocel(ocel)) + } } fn infer_format(path: &Path) -> Option { @@ -1096,16 +1778,545 @@ impl Exportable for SlimLinkedOCEL { type Error = OCELIOError; type ExportOptions = (); + fn infer_format(path: &Path) -> Option { + ::infer_format(path) + } + + fn export_to_path_with_options>( + &self, + path: P, + _: Self::ExportOptions, + ) -> Result<(), Self::Error> { + let path = path.as_ref(); + let format = ::infer_format(path).ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "Could not infer format from path", + ) + })?; + if format.ends_with("sqlite") || (format.ends_with("db") && !format.ends_with("duckdb")) { + #[cfg(feature = "ocel-sqlite")] + return crate::core::event_data::object_centric::ocel_sql::export_ocel_sqlite_to_path( + self, path, + ) + .map_err(OCELIOError::from); + #[cfg(not(feature = "ocel-sqlite"))] + return Err(OCELIOError::UnsupportedFormat( + "SQLite support not enabled".to_string(), + )); + } + if format.ends_with("duckdb") { + #[cfg(feature = "ocel-duckdb")] + return crate::core::event_data::object_centric::ocel_sql::export_ocel_duckdb_to_path( + self, path, + ) + .map_err(OCELIOError::from); + #[cfg(not(feature = "ocel-duckdb"))] + return Err(OCELIOError::UnsupportedFormat( + "DuckDB support not enabled".to_string(), + )); + } + let file = std::fs::File::create(path)?; + let writer = std::io::BufWriter::new(file); + Self::export_to_writer(self, writer, &format) + } + fn export_to_writer_with_options( &self, - writer: W, + #[cfg(feature = "ocel-sqlite")] mut writer: W, + #[cfg(not(feature = "ocel-sqlite"))] writer: W, format: &str, _: Self::ExportOptions, ) -> Result<(), Self::Error> { - self.construct_ocel().export_to_writer(writer, format) + if let Some(inner) = format.strip_suffix(".gz") { + let mut encoder = flate2::write::GzEncoder::new( + Box::new(writer) as Box, + flate2::Compression::default(), + ); + self.export_to_writer_with_options(&mut encoder, inner, ())?; + encoder.finish()?; + return Ok(()); + } + if format.ends_with("json") || format.ends_with("jsonocel") { + crate::core::event_data::object_centric::ocel_json::export_ocel_json_to_writer( + self, writer, + ) + .map_err(OCELIOError::Io) + } else if format.ends_with("xml") || format.ends_with("xmlocel") { + crate::core::event_data::object_centric::ocel_xml::xml_ocel_export::export_ocel_xml( + writer, self, + ) + .map_err(OCELIOError::Xml) + } else if format.ends_with("ocel.csv") { + crate::core::event_data::object_centric::ocel_csv::export_ocel_csv(writer, self) + .map_err(|e| OCELIOError::Other(e.to_string())) + } else if format.ends_with("sqlite") + || (format.ends_with("db") && !format.ends_with("duckdb")) + { + #[cfg(feature = "ocel-sqlite")] + { + let bytes = + crate::core::event_data::object_centric::ocel_sql::export_ocel_sqlite_to_vec( + self, + ) + .map_err(OCELIOError::from)?; + writer.write_all(&bytes)?; + Ok(()) + } + #[cfg(not(feature = "ocel-sqlite"))] + return Err(OCELIOError::UnsupportedFormat( + "SQLite support not enabled".to_string(), + )); + } else if format.ends_with("duckdb") { + Err(OCELIOError::UnsupportedFormat( + "DuckDB export to writer not supported".to_string(), + )) + } else { + Err(OCELIOError::UnsupportedFormat(format.to_string())) + } } fn known_export_formats() -> Vec { ::known_export_formats() } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::event_data::object_centric::OCELAttributeType; + + fn empty_type(name: &str) -> OCELType { + OCELType { + name: name.into(), + attributes: Vec::new(), + } + } + fn ts(s: &str) -> DateTime { + DateTime::parse_from_rfc3339(s).unwrap() + } + + #[test] + fn append_resolves_forward_e2o_on_finalize() { + let mut s: SlimLinkedOCEL = SlimLinkedOCEL::new(); + s.declare_event_type(empty_type("place")).unwrap(); + s.declare_object_type(empty_type("order")).unwrap(); + s.append_event( + "e1".into(), + "place", + ts("2024-01-01T00:00:00Z"), + Vec::new(), + vec![OCELRelationship { + object_id: "o1".into(), + qualifier: "for".into(), + }], + ) + .unwrap(); + s.append_object("o1".into(), "order", Vec::new(), Vec::new()) + .unwrap(); + s.finalize().unwrap(); + + let ev = LinkedOCELAccess::get_ev_by_id(&s, "e1").unwrap(); + let ob = LinkedOCELAccess::get_ob_by_id(&s, "o1").unwrap(); + assert_eq!(s.events[ev.into_inner() as usize].relationships.len(), 1); + assert_eq!(s.events[ev.into_inner() as usize].relationships[0].1, ob); + } + + #[test] + fn multi_qualifier_e2o_dedups_reverse_index() { + let mut s: SlimLinkedOCEL = SlimLinkedOCEL::new(); + s.declare_event_type(empty_type("place")).unwrap(); + s.declare_object_type(empty_type("order")).unwrap(); + s.append_object("o1".into(), "order", Vec::new(), Vec::new()) + .unwrap(); + s.append_event( + "e1".into(), + "place", + ts("2024-01-01T00:00:00Z"), + Vec::new(), + vec![ + OCELRelationship { + object_id: "o1".into(), + qualifier: "for".into(), + }, + OCELRelationship { + object_id: "o1".into(), + qualifier: "via".into(), + }, + ], + ) + .unwrap(); + + let e1 = LinkedOCELAccess::get_ev_by_id(&s, "e1").unwrap(); + let o1 = LinkedOCELAccess::get_ob_by_id(&s, "o1").unwrap(); + assert_eq!(s.objects[o1.into_inner() as usize].e2o_rev, vec![e1]); + + // add_e2o with another qualifier on the same pair stays a single entry. + s.add_e2o(e1, o1, "alt".into()); + assert_eq!(s.objects[o1.into_inner() as usize].e2o_rev, vec![e1]); + } + + #[test] + fn multi_qualifier_o2o_dedups_reverse_index() { + let mut s: SlimLinkedOCEL = SlimLinkedOCEL::new(); + s.declare_object_type(empty_type("a")).unwrap(); + s.append_object("o2".into(), "a", Vec::new(), Vec::new()) + .unwrap(); + s.append_object( + "o1".into(), + "a", + Vec::new(), + vec![ + OCELRelationship { + object_id: "o2".into(), + qualifier: "first".into(), + }, + OCELRelationship { + object_id: "o2".into(), + qualifier: "second".into(), + }, + ], + ) + .unwrap(); + + let o1 = LinkedOCELAccess::get_ob_by_id(&s, "o1").unwrap(); + let o2 = LinkedOCELAccess::get_ob_by_id(&s, "o2").unwrap(); + assert_eq!(s.objects[o2.into_inner() as usize].o2o_rev, vec![o1]); + + s.add_o2o(o1, o2, "third".into()); + assert_eq!(s.objects[o2.into_inner() as usize].o2o_rev, vec![o1]); + } + + #[test] + fn reverse_index_one_entry_per_distinct_event() { + let mut s: SlimLinkedOCEL = SlimLinkedOCEL::new(); + s.declare_event_type(empty_type("e")).unwrap(); + s.declare_object_type(empty_type("o")).unwrap(); + s.append_object("o1".into(), "o", Vec::new(), Vec::new()) + .unwrap(); + for id in ["e3", "e2", "e1"] { + s.append_event( + id.into(), + "e", + ts("2024-01-01T00:00:00Z"), + Vec::new(), + vec![OCELRelationship { + object_id: "o1".into(), + qualifier: "q".into(), + }], + ) + .unwrap(); + } + let o1 = LinkedOCELAccess::get_ob_by_id(&s, "o1").unwrap(); + let rev = &s.objects[o1.into_inner() as usize].e2o_rev; + assert!( + rev.windows(2).all(|w| w[0] < w[1]), + "e2o_rev must be sorted ascending: {:?}", + rev + ); + assert_eq!(rev.len(), 3); + } + + #[test] + fn append_resolves_forward_o2o_on_finalize() { + let mut s: SlimLinkedOCEL = SlimLinkedOCEL::new(); + s.declare_object_type(empty_type("a")).unwrap(); + s.append_object( + "o1".into(), + "a", + Vec::new(), + vec![OCELRelationship { + object_id: "o2".into(), + qualifier: "next".into(), + }], + ) + .unwrap(); + s.append_object("o2".into(), "a", Vec::new(), Vec::new()) + .unwrap(); + s.finalize().unwrap(); + + let o1 = LinkedOCELAccess::get_ob_by_id(&s, "o1").unwrap(); + let o2 = LinkedOCELAccess::get_ob_by_id(&s, "o2").unwrap(); + let next_q = intern_qualifier( + &mut s.qualifiers, + &mut s.qualifier_index, + &s.hasher, + "next".to_string(), + ); + assert_eq!( + s.objects[o1.into_inner() as usize].relationships, + vec![(next_q, o2)] + ); + } + + #[test] + fn append_unknown_type_auto_declares() { + let mut s: SlimLinkedOCEL = SlimLinkedOCEL::new(); + s.append_event( + "e1".into(), + "missing", + ts("2024-01-01T00:00:00Z"), + vec![OCELEventAttribute { + name: "x".into(), + value: OCELAttributeValue::Integer(7), + }], + Vec::new(), + ) + .unwrap(); + assert_eq!(s.event_types().len(), 1); + assert_eq!(s.event_types()[0].name, "missing"); + assert_eq!(s.event_types()[0].attributes.len(), 1); + assert_eq!(s.event_types()[0].attributes[0].name, "x"); + assert_eq!(s.event_types()[0].attributes[0].value_type, "integer"); + } + + #[test] + fn append_value_type_mismatch_coerces_when_possible() { + // Declared `price: float`, `count: integer`, `label: string`. + // We pass mismatched variants that are all coercible. + let mut s: SlimLinkedOCEL = SlimLinkedOCEL::new(); + s.declare_event_type(OCELType { + name: "act".into(), + attributes: vec![ + OCELTypeAttribute::new("price", &OCELAttributeType::Float), + OCELTypeAttribute::new("count", &OCELAttributeType::Integer), + OCELTypeAttribute::new("label", &OCELAttributeType::String), + ], + }) + .unwrap(); + s.append_event( + "e1".into(), + "act", + ts("2024-01-01T00:00:00Z"), + vec![ + OCELEventAttribute { + name: "price".into(), + value: OCELAttributeValue::Integer(42), + }, + OCELEventAttribute { + name: "count".into(), + value: OCELAttributeValue::String("7".into()), + }, + OCELEventAttribute { + name: "label".into(), + value: OCELAttributeValue::Integer(99), + }, + ], + Vec::new(), + ) + .unwrap(); + let ev = EventIndex(0); + assert!(matches!( + ev.get_attribute_value("price", &s), + Some(OCELAttributeValue::Float(f)) if (*f - 42.0).abs() < 1e-9 + )); + assert!(matches!( + ev.get_attribute_value("count", &s), + Some(OCELAttributeValue::Integer(7)) + )); + match ev.get_attribute_value("label", &s) { + Some(OCELAttributeValue::String(v)) => assert_eq!(v, "99"), + other => panic!("expected String '99', got {:?}", other), + } + } + + #[test] + fn append_value_type_mismatch_uncoercible_falls_back() { + // Declared `x: integer`. We pass a String that does not parse to i64. + // Coercion fails; the value is stored as-is (with a warning to stderr). + let mut s: SlimLinkedOCEL = SlimLinkedOCEL::new(); + s.declare_event_type(OCELType { + name: "act".into(), + attributes: vec![OCELTypeAttribute::new("x", &OCELAttributeType::Integer)], + }) + .unwrap(); + s.append_event( + "e1".into(), + "act", + ts("2024-01-01T00:00:00Z"), + vec![OCELEventAttribute { + name: "x".into(), + value: OCELAttributeValue::String("not-an-int".into()), + }], + Vec::new(), + ) + .unwrap(); + let ev = EventIndex(0); + match ev.get_attribute_value("x", &s) { + Some(OCELAttributeValue::String(v)) => assert_eq!(v, "not-an-int"), + other => panic!("expected String fallback, got {:?}", other), + } + } + + #[test] + fn append_missing_declared_attr_reads_as_null() { + // Declared `x` and `y`; event only has `x`. Reading `y` returns Null. + let mut s: SlimLinkedOCEL = SlimLinkedOCEL::new(); + s.declare_event_type(OCELType { + name: "act".into(), + attributes: vec![ + OCELTypeAttribute::new("x", &OCELAttributeType::Integer), + OCELTypeAttribute::new("y", &OCELAttributeType::String), + ], + }) + .unwrap(); + s.append_event( + "e1".into(), + "act", + ts("2024-01-01T00:00:00Z"), + vec![OCELEventAttribute { + name: "x".into(), + value: OCELAttributeValue::Integer(7), + }], + Vec::new(), + ) + .unwrap(); + let ev = EventIndex(0); + let fat = ev.fat_ev(&s); + assert_eq!(fat.attributes.len(), 2); + assert_eq!(fat.attributes[0].name, "x"); + assert!(matches!( + fat.attributes[0].value, + OCELAttributeValue::Integer(7) + )); + assert_eq!(fat.attributes[1].name, "y"); + assert!(matches!(fat.attributes[1].value, OCELAttributeValue::Null)); + } + + #[test] + fn declare_after_append_appends_missing_attrs_only() { + let mut s: SlimLinkedOCEL = SlimLinkedOCEL::new(); + // Auto-grow infers `first: integer` from the value variant. + s.append_event( + "e1".into(), + "act", + ts("2024-01-01T00:00:00Z"), + vec![OCELEventAttribute { + name: "first".into(), + value: OCELAttributeValue::Integer(1), + }], + Vec::new(), + ) + .unwrap(); + // Real declaration: declares `first` as `float` (overriding the inferred `integer`) + // and adds `second`. Declared name order is reversed relative to what we observed. + s.declare_event_type(OCELType { + name: "act".into(), + attributes: vec![ + OCELTypeAttribute::new("second", &OCELAttributeType::String), + OCELTypeAttribute::new("first", &OCELAttributeType::Float), + ], + }) + .unwrap(); + let attrs = &s.event_types()[0].attributes; + assert_eq!(attrs.len(), 2); + // Existing position preserved: `first` stays at slot 0 (where e1 stored its value). + assert_eq!(attrs[0].name, "first"); + // Declaration overrides the inferred metadata. + assert_eq!(attrs[0].value_type, "float"); + // New attribute appended at the end. + assert_eq!(attrs[1].name, "second"); + assert_eq!(attrs[1].value_type, "string"); + // Stored value is not re-coerced to match the new declared type: the variant stays + // `Integer` even though the schema now says `float`. Documented on `declare_or_merge_type`. + let ev0 = &s.events[0]; + assert!(matches!(ev0.attributes[0], OCELAttributeValue::Integer(1))); + } + + #[test] + fn iter_events_of_type_matches_filter_for_slim_and_ocel() { + let mut s: SlimLinkedOCEL = SlimLinkedOCEL::new(); + s.declare_event_type(empty_type("a")).unwrap(); + s.declare_event_type(empty_type("b")).unwrap(); + s.declare_object_type(empty_type("o1")).unwrap(); + s.declare_object_type(empty_type("o2")).unwrap(); + for (i, t) in [("a", "e1"), ("b", "e2"), ("a", "e3"), ("b", "e4")] + .iter() + .enumerate() + { + s.append_event( + t.1.into(), + t.0, + ts(&format!("2024-01-0{}T00:00:00Z", i + 1)), + Vec::new(), + Vec::new(), + ) + .unwrap(); + } + for (i, t) in [("o1", "x"), ("o2", "y"), ("o1", "z")].iter().enumerate() { + s.append_object( + t.1.into(), + t.0, + vec![OCELObjectAttribute { + name: "k".into(), + value: OCELAttributeValue::Integer(i as i64), + time: ts("2024-01-01T00:00:00Z"), + }], + Vec::new(), + ) + .unwrap(); + } + s.finalize().unwrap(); + + let slim_a: Vec = s.iter_events_of_type("a").map(|e| e.id.clone()).collect(); + assert_eq!(slim_a, vec!["e1".to_string(), "e3".to_string()]); + let slim_o1: Vec = s.iter_objects_of_type("o1").map(|o| o.id.clone()).collect(); + assert_eq!(slim_o1, vec!["x".to_string(), "z".to_string()]); + assert_eq!(s.iter_events_of_type("missing").count(), 0); + + // The default impl on `OCEL` filters; the `SlimLinkedOCEL` override walks per-type + // indices. Both must yield the same set. + let ocel = s.construct_ocel(); + let ocel_a: Vec = ocel + .iter_events_of_type("a") + .map(|e| e.id.clone()) + .collect(); + let ocel_o1: Vec = ocel + .iter_objects_of_type("o1") + .map(|o| o.id.clone()) + .collect(); + let mut slim_a_sorted = slim_a; + let mut ocel_a_sorted = ocel_a; + slim_a_sorted.sort(); + ocel_a_sorted.sort(); + assert_eq!(slim_a_sorted, ocel_a_sorted); + let mut slim_o1_sorted = slim_o1; + let mut ocel_o1_sorted = ocel_o1; + slim_o1_sorted.sort(); + ocel_o1_sorted.sort(); + assert_eq!(slim_o1_sorted, ocel_o1_sorted); + } + + #[test] + fn append_attributes_become_positional() { + let mut s: SlimLinkedOCEL = SlimLinkedOCEL::new(); + s.declare_event_type(OCELType { + name: "act".into(), + attributes: vec![ + OCELTypeAttribute::new("first", &OCELAttributeType::String), + OCELTypeAttribute::new("second", &OCELAttributeType::Integer), + ], + }) + .unwrap(); + s.append_event( + "e1".into(), + "act", + ts("2024-01-01T00:00:00Z"), + vec![ + OCELEventAttribute { + name: "second".into(), + value: OCELAttributeValue::Integer(42), + }, + OCELEventAttribute { + name: "first".into(), + value: OCELAttributeValue::String("hi".into()), + }, + ], + Vec::new(), + ) + .unwrap(); + let ev = &s.events[0]; + assert_eq!(ev.attributes[0], OCELAttributeValue::String("hi".into())); + assert_eq!(ev.attributes[1], OCELAttributeValue::Integer(42)); + } +} diff --git a/process_mining/src/core/event_data/object_centric/mod.rs b/process_mining/src/core/event_data/object_centric/mod.rs index 2f14f819..f83f4da6 100644 --- a/process_mining/src/core/event_data/object_centric/mod.rs +++ b/process_mining/src/core/event_data/object_centric/mod.rs @@ -1,6 +1,7 @@ //! Object-centric Event Data //! +pub mod appendable; /// Convert an OCEL to a Polars `DataFrame` /// /// See the [`dataframe::ocel_to_dataframes`] function. @@ -18,7 +19,8 @@ pub mod ocel_csv; pub mod ocel_json; pub mod ocel_sql; pub(crate) mod ocel_struct; -#[doc(inline)] -pub use ocel_struct::*; pub mod ocel_xml; +pub mod readable; pub mod utils; +#[doc(inline)] +pub use ocel_struct::*; diff --git a/process_mining/src/core/event_data/object_centric/ocel_csv/csv_ocel_export.rs b/process_mining/src/core/event_data/object_centric/ocel_csv/csv_ocel_export.rs index dc4bc873..d2e9d7f7 100644 --- a/process_mining/src/core/event_data/object_centric/ocel_csv/csv_ocel_export.rs +++ b/process_mining/src/core/event_data/object_centric/ocel_csv/csv_ocel_export.rs @@ -1,6 +1,9 @@ //! CSV Export for OCEL 2.0 -use crate::core::event_data::object_centric::ocel_struct::{OCELAttributeValue, OCELObject, OCEL}; +use crate::core::event_data::object_centric::{ + ocel_struct::OCELAttributeValue, + readable::{OCELLookup, ReadableOCEL}, +}; use chrono::{DateTime, FixedOffset}; use std::{ collections::{HashMap, HashSet}, @@ -58,14 +61,6 @@ impl From for OCELCSVExportError { } } -struct ExportRow<'a> { - timestamp: Option<&'a DateTime>, - id: &'a str, - activity: &'a str, - object_refs: HashMap<&'a str, Vec>>, - event_attrs: HashMap<&'a str, String>, -} - struct ObjectRef<'a> { object_id: &'a str, qualifier: &'a str, @@ -73,188 +68,200 @@ struct ObjectRef<'a> { } /// Export OCEL to CSV format -pub fn export_ocel_csv(writer: W, ocel: &OCEL) -> Result<(), OCELCSVExportError> { +pub fn export_ocel_csv(writer: W, ocel: &O) -> Result<(), OCELCSVExportError> +where + W: Write, + O: ReadableOCEL + ?Sized, +{ export_ocel_csv_with_options(writer, ocel, &OCELCSVExportOptions::default()) } /// Export OCEL to CSV format with custom options -pub fn export_ocel_csv_with_options( +pub fn export_ocel_csv_with_options( writer: W, - ocel: &OCEL, + ocel: &O, options: &OCELCSVExportOptions, -) -> Result<(), OCELCSVExportError> { +) -> Result<(), OCELCSVExportError> +where + W: Write, + O: ReadableOCEL + ?Sized, +{ let mut csv_writer = csv::Writer::from_writer(writer); - let objects_by_id: HashMap<&str, &OCELObject> = - ocel.objects.iter().map(|o| (o.id.as_str(), o)).collect(); + let lookup = ocel.lookup(); let mut object_type_names: Vec<&str> = ocel - .object_types + .object_types() .iter() .map(|ot| ot.name.as_str()) .collect(); object_type_names.sort(); let mut event_attr_names: Vec<&str> = ocel - .events + .event_types() .iter() - .flat_map(|e| e.attributes.iter().map(|a| a.name.as_str())) + .flat_map(|et| et.attributes.iter().map(|a| a.name.as_str())) .collect::>() .into_iter() .collect(); event_attr_names.sort(); - // Build header let mut headers: Vec = vec!["id".into(), "activity".into(), "timestamp".into()]; headers.extend(object_type_names.iter().map(|n| format!("ot:{n}"))); headers.extend(event_attr_names.iter().map(|n| format!("ea:{n}"))); csv_writer.write_record(&headers)?; - let mut rows: Vec> = Vec::new(); + // (time, object_id) pairs covered by event rows, so the later object-attribute pass + // can skip rows that would duplicate an event row's value at the same timestamp. + let mut event_obj_times: HashSet<(DateTime, &str)> = HashSet::new(); - // Event rows - for event in &ocel.events { + for event_cow in ocel.iter_events_sorted_by_time() { + let event = event_cow.as_ref(); let mut object_refs: HashMap<&str, Vec>> = HashMap::new(); for rel in &event.relationships { - if let Some(obj) = objects_by_id.get(rel.object_id.as_str()) { - let matching_attrs: Vec<_> = obj - .attributes - .iter() - .filter(|a| a.time == event.time) + let Some(obj_id) = lookup.get_id_borrow(&rel.object_id) else { + continue; + }; + event_obj_times.insert((event.time, obj_id)); + if let Some(obj_type) = lookup.object_type_of(obj_id) { + let obj_attrs: serde_json::Map<_, _> = lookup + .object_attributes(obj_id) + .filter(|(_, _, t)| *t == event.time) + .map(|(name, value, _)| (name.to_string(), ocel_value_to_json(value))) .collect(); - let obj_attrs = if matching_attrs.is_empty() { - None - } else { - Some( - matching_attrs - .iter() - .map(|a| (a.name.clone(), ocel_value_to_json(&a.value))) - .collect::>(), - ) - }; - object_refs - .entry(&obj.object_type) - .or_default() - .push(ObjectRef { - object_id: &rel.object_id, - qualifier: &rel.qualifier, - attributes: obj_attrs, - }); + object_refs.entry(obj_type).or_default().push(ObjectRef { + object_id: obj_id, + qualifier: &rel.qualifier, + attributes: (!obj_attrs.is_empty()).then_some(obj_attrs), + }); } } - rows.push(ExportRow { - timestamp: Some(&event.time), - id: &event.id, - activity: &event.event_type, - object_refs, - event_attrs: event - .attributes - .iter() - .map(|a| (a.name.as_str(), a.value.to_string())) - .collect(), - }); + let event_attrs: HashMap<&str, String> = event + .attributes + .iter() + .map(|a| (a.name.as_str(), a.value.to_string())) + .collect(); + write_record( + &mut csv_writer, + &event.id, + &event.event_type, + Some(event.time), + &object_refs, + &event_attrs, + &object_type_names, + &event_attr_names, + options, + )?; } - // O2O relationship rows if options.include_o2o { - for obj in &ocel.objects { - if obj.relationships.is_empty() { - continue; - } + for obj_id in lookup.iter_object_ids() { let mut object_refs: HashMap<&str, Vec>> = HashMap::new(); - for rel in &obj.relationships { - if let Some(target) = objects_by_id.get(rel.object_id.as_str()) { - object_refs - .entry(target.object_type.as_str()) - .or_default() - .push(ObjectRef { - object_id: &rel.object_id, - qualifier: &rel.qualifier, - attributes: None, - }); + let mut had_relationship = false; + for (target_id, qualifier) in lookup.object_relationships(obj_id) { + if let Some(target_type) = lookup.object_type_of(target_id) { + had_relationship = true; + object_refs.entry(target_type).or_default().push(ObjectRef { + object_id: target_id, + qualifier, + attributes: None, + }); } } - rows.push(ExportRow { - timestamp: None, - id: &obj.id, - activity: "o2o", - object_refs, - event_attrs: HashMap::new(), - }); + if !had_relationship { + continue; + } + write_record( + &mut csv_writer, + obj_id, + "o2o", + None, + &object_refs, + &HashMap::new(), + &object_type_names, + &event_attr_names, + options, + )?; } } - // Object attribute change rows if options.include_object_attribute_changes { - for obj in &ocel.objects { - let mut attrs_by_time: HashMap<&DateTime, Vec<_>> = HashMap::new(); - for attr in &obj.attributes { - attrs_by_time.entry(&attr.time).or_default().push(attr); + for obj_id in lookup.iter_object_ids() { + let Some(obj_type) = lookup.object_type_of(obj_id) else { + continue; + }; + let mut attrs_by_time: HashMap< + DateTime, + Vec<(String, serde_json::Value)>, + > = HashMap::new(); + for (name, value, time) in lookup.object_attributes(obj_id) { + attrs_by_time + .entry(time) + .or_default() + .push((name.to_string(), ocel_value_to_json(value))); } for (time, attrs) in attrs_by_time { - let in_event = ocel.events.iter().any(|e| { - &e.time == time && e.relationships.iter().any(|r| r.object_id == obj.id) - }); - if in_event { + if event_obj_times.contains(&(time, obj_id)) { continue; } - let attr_map: serde_json::Map<_, _> = attrs - .iter() - .map(|a| (a.name.clone(), ocel_value_to_json(&a.value))) - .collect(); + let attr_map: serde_json::Map<_, _> = attrs.into_iter().collect(); let mut object_refs: HashMap<&str, Vec>> = HashMap::new(); - object_refs - .entry(&obj.object_type) - .or_default() - .push(ObjectRef { - object_id: &obj.id, - qualifier: "", - attributes: Some(attr_map), - }); - rows.push(ExportRow { - timestamp: Some(time), - id: "", - activity: "", - object_refs, - event_attrs: HashMap::new(), + object_refs.entry(obj_type).or_default().push(ObjectRef { + object_id: obj_id, + qualifier: "", + attributes: Some(attr_map), }); + write_record( + &mut csv_writer, + "", + "", + Some(time), + &object_refs, + &HashMap::new(), + &object_type_names, + &event_attr_names, + options, + )?; } } } - - // Sort: timestamped rows first (by time), then O2O rows (by id) - rows.sort_by(|a, b| match (a.timestamp, b.timestamp) { - (None, None) => a.id.cmp(b.id), - (None, Some(_)) => std::cmp::Ordering::Greater, - (Some(_), None) => std::cmp::Ordering::Less, - (Some(at), Some(bt)) => at.cmp(bt), - }); - - // Write rows - for row in rows { - let mut record = vec![ - row.id.to_string(), - row.activity.to_string(), - row.timestamp - .map(|ts| format_timestamp(ts, options)) - .unwrap_or_default(), - ]; - record.extend(object_type_names.iter().map(|ot| { - row.object_refs - .get(*ot) - .map(|r| format_object_refs(r)) - .unwrap_or_default() - })); - record.extend( - event_attr_names - .iter() - .map(|ea| row.event_attrs.get(*ea).cloned().unwrap_or_default()), - ); - csv_writer.write_record(&record)?; - } csv_writer.flush()?; Ok(()) } +#[allow(clippy::too_many_arguments)] +fn write_record( + csv_writer: &mut csv::Writer, + id: &str, + activity: &str, + timestamp: Option>, + object_refs: &HashMap<&str, Vec>>, + event_attrs: &HashMap<&str, String>, + object_type_names: &[&str], + event_attr_names: &[&str], + options: &OCELCSVExportOptions, +) -> Result<(), OCELCSVExportError> { + let mut record: Vec = vec![ + id.to_string(), + activity.to_string(), + timestamp + .map(|ts| format_timestamp(&ts, options)) + .unwrap_or_default(), + ]; + record.extend(object_type_names.iter().map(|ot| { + object_refs + .get(*ot) + .map(|r| format_object_refs(r)) + .unwrap_or_default() + })); + record.extend( + event_attr_names + .iter() + .map(|ea| event_attrs.get(*ea).cloned().unwrap_or_default()), + ); + csv_writer.write_record(&record)?; + Ok(()) +} + fn format_timestamp(dt: &DateTime, options: &OCELCSVExportOptions) -> String { options .date_format @@ -273,12 +280,10 @@ fn format_object_refs(refs: &[ObjectRef<'_>]) -> String { } if let Some(attrs) = &r.attributes { if !attrs.is_empty() { - match serde_json::to_string(attrs) { - Ok(json) => s.push_str(&json), - Err(e) => { - eprintln!("Failed to serialize object attributes to JSON: {}", e); - } - } + s.push_str( + &serde_json::to_string(attrs) + .expect("serde_json::Map always serializes to a string"), + ); } } s @@ -301,19 +306,24 @@ fn ocel_value_to_json(value: &OCELAttributeValue) -> serde_json::Value { } /// Export OCEL to a CSV file at the specified path -pub fn export_ocel_csv_to_path>( - ocel: &OCEL, - path: P, -) -> Result<(), OCELCSVExportError> { +pub fn export_ocel_csv_to_path(ocel: &O, path: P) -> Result<(), OCELCSVExportError> +where + P: AsRef, + O: ReadableOCEL + ?Sized, +{ export_ocel_csv(std::io::BufWriter::new(std::fs::File::create(path)?), ocel) } /// Export OCEL to a CSV file at the specified path with options -pub fn export_ocel_csv_to_path_with_options>( - ocel: &OCEL, +pub fn export_ocel_csv_to_path_with_options( + ocel: &O, path: P, options: &OCELCSVExportOptions, -) -> Result<(), OCELCSVExportError> { +) -> Result<(), OCELCSVExportError> +where + P: AsRef, + O: ReadableOCEL + ?Sized, +{ export_ocel_csv_with_options( std::io::BufWriter::new(std::fs::File::create(path)?), ocel, @@ -322,7 +332,10 @@ pub fn export_ocel_csv_to_path_with_options>( } /// Export OCEL to a CSV string -pub fn export_ocel_csv_to_string(ocel: &OCEL) -> Result { +pub fn export_ocel_csv_to_string(ocel: &O) -> Result +where + O: ReadableOCEL + ?Sized, +{ let mut buf = Vec::new(); export_ocel_csv(&mut buf, ocel)?; String::from_utf8(buf).map_err(|e| { @@ -331,10 +344,13 @@ pub fn export_ocel_csv_to_string(ocel: &OCEL) -> Result( + ocel: &O, options: &OCELCSVExportOptions, -) -> Result { +) -> Result +where + O: ReadableOCEL + ?Sized, +{ let mut buf = Vec::new(); export_ocel_csv_with_options(&mut buf, ocel, options)?; String::from_utf8(buf).map_err(|e| { @@ -348,7 +364,8 @@ mod tests { use crate::core::event_data::object_centric::{ ocel_csv::import_ocel_csv, ocel_struct::{ - OCELEvent, OCELObjectAttribute, OCELRelationship, OCELType, OCELTypeAttribute, + OCELEvent, OCELObject, OCELObjectAttribute, OCELRelationship, OCELType, + OCELTypeAttribute, OCEL, }, }; diff --git a/process_mining/src/core/event_data/object_centric/ocel_json/mod.rs b/process_mining/src/core/event_data/object_centric/ocel_json/mod.rs index 1d321c51..41771104 100644 --- a/process_mining/src/core/event_data/object_centric/ocel_json/mod.rs +++ b/process_mining/src/core/event_data/object_centric/ocel_json/mod.rs @@ -5,24 +5,43 @@ use std::{ path::Path, }; -use crate::core::event_data::object_centric::ocel_struct::OCEL; +use serde::{ + de::{DeserializeSeed, IgnoredAny, MapAccess, SeqAccess, Visitor}, + Deserializer, Serialize, Serializer, +}; + +use crate::core::event_data::object_centric::{ + appendable::AppendableOCEL, + io::OCELIOError, + ocel_struct::{OCELEvent, OCELObject, OCELType, OCEL}, + readable::ReadableOCEL, +}; /// -/// Serialize [`OCEL`] as a JSON [`String`] -/// -/// [`serde_json`] can also be used to convert [`OCEL`] to other targets (e.g., `serde_json::to_writer`) +/// Serialize an OCEL backend (e.g., [`OCEL`] or +/// [`super::linked_ocel::SlimLinkedOCEL`]) as a JSON [`String`]. /// -pub fn ocel_to_json(ocel: &OCEL) -> String { - serde_json::to_string(ocel).unwrap() +pub fn ocel_to_json(ocel: &R) -> String { + let bytes = export_ocel_json_to_vec(ocel).expect("writing JSON to a Vec cannot fail"); + String::from_utf8(bytes).expect("serde_json always emits valid UTF-8") } /// -/// Import [`OCEL`] from a JSON [`String`] +/// Import [`OCEL`] from a JSON [`String`], returning a [`Result`]. /// /// [`serde_json`] can also be used to import [`OCEL`] from other targets (e.g., `serde_json::from_reader`) /// +pub fn try_json_to_ocel(ocel_json: &str) -> Result { + serde_json::from_str(ocel_json) +} + +/// +/// Import [`OCEL`] from a JSON [`String`]. +/// +/// Panics on malformed JSON; prefer [`try_json_to_ocel`] for fallible callers. +/// pub fn json_to_ocel(ocel_json: &str) -> OCEL { - serde_json::from_str(ocel_json).unwrap() + try_json_to_ocel(ocel_json).expect("malformed OCEL JSON") } /// @@ -44,24 +63,354 @@ pub fn import_ocel_json_slice(slice: &[u8]) -> Result { Ok(serde_json::from_slice(slice)?) } +/// Export an OCEL backend to a JSON file at the specified path. +pub fn export_ocel_json_to_path(ocel: &R, path: P) -> Result<(), std::io::Error> +where + R: ReadableOCEL + ?Sized, + P: AsRef, +{ + let writer: BufWriter = BufWriter::new(File::create(path)?); + Ok(write_ocel_json(ocel, writer)?) +} + +/// Export an OCEL backend to JSON in a byte array ([`Vec`]). +pub fn export_ocel_json_to_vec( + ocel: &R, +) -> Result, std::io::Error> { + let mut buf = Vec::new(); + write_ocel_json(ocel, &mut buf)?; + Ok(buf) +} + /// -/// Export [`OCEL`] to a JSON file at the specified path -/// -/// To import an OCEL .json file see [`import_ocel_json_path`] instead. +/// Stream an OCEL backend as JSON into the given writer. /// -pub fn export_ocel_json_to_path>( - ocel: &OCEL, - path: P, -) -> Result<(), std::io::Error> { - let writer: BufWriter = BufWriter::new(File::create(path)?); - Ok(serde_json::to_writer(writer, ocel)?) +pub fn export_ocel_json_to_writer(ocel: &R, writer: W) -> Result<(), std::io::Error> +where + R: ReadableOCEL + ?Sized, + W: std::io::Write, +{ + Ok(write_ocel_json(ocel, writer)?) +} + +/// Stream an OCEL to `writer` as JSON. Field order matches `OCEL`'s `Serialize` derive +/// so `&OCEL` output is byte-identical. +fn write_ocel_json(ocel: &R, writer: W) -> Result<(), serde_json::Error> +where + R: ReadableOCEL + ?Sized, + W: std::io::Write, +{ + use serde::ser::SerializeMap; + let mut ser = serde_json::Serializer::new(writer); + let mut m = ser.serialize_map(Some(4))?; + m.serialize_entry("eventTypes", ocel.event_types())?; + m.serialize_entry("objectTypes", ocel.object_types())?; + m.serialize_entry("events", &EventsStream { ocel })?; + m.serialize_entry("objects", &ObjectsStream { ocel })?; + m.end() +} + +struct EventsStream<'a, R: ?Sized> { + ocel: &'a R, +} + +impl<'a, R: ReadableOCEL + ?Sized> Serialize for EventsStream<'a, R> { + fn serialize(&self, s: S) -> Result { + use serde::ser::SerializeSeq; + let mut seq = s.serialize_seq(None)?; + for e in self.ocel.iter_events() { + seq.serialize_element(&*e)?; + } + seq.end() + } +} + +struct ObjectsStream<'a, R: ?Sized> { + ocel: &'a R, +} + +impl<'a, R: ReadableOCEL + ?Sized> Serialize for ObjectsStream<'a, R> { + fn serialize(&self, s: S) -> Result { + use serde::ser::SerializeSeq; + let mut seq = s.serialize_seq(None)?; + for o in self.ocel.iter_objects() { + seq.serialize_element(&*o)?; + } + seq.end() + } } /// -/// Export [`OCEL`] to JSON in a byte array ([`Vec`]) +/// Stream a JSON-serialized OCEL into an [`AppendableOCEL`]. /// -/// To import an OCEL .json file see [`import_ocel_json_path`] instead. +/// The caller is responsible for invoking [`AppendableOCEL::finalize`] afterwards if +/// the implementation requires it. Type declarations and instances may be intermixed; +/// relationships referencing not-yet-seen object IDs are buffered by implementations +/// that support it. /// -pub fn export_ocel_json_to_vec(ocel: &OCEL) -> Result, std::io::Error> { - Ok(serde_json::to_vec(ocel)?) +pub fn import_ocel_json_into(reader: R, ocel: &mut A) -> Result<(), OCELIOError> +where + R: std::io::Read, + A: AppendableOCEL, + A::Error: Into, +{ + let mut append_err: Option = None; + let mut de = serde_json::Deserializer::from_reader(reader); + let result = de.deserialize_map(OcelTopVisitor { + ocel, + append_err: &mut append_err, + }); + if let Some(e) = append_err { + return Err(e.into()); + } + result?; + Ok(()) +} + +/// Visitor for the top-level OCEL JSON object. Type lists are fully deserialized; +/// `events` and `objects` are streamed element-by-element into the sink. +struct OcelTopVisitor<'a, A: AppendableOCEL> { + ocel: &'a mut A, + append_err: &'a mut Option, +} + +impl<'a, 'de, A: AppendableOCEL> Visitor<'de> for OcelTopVisitor<'a, A> { + type Value = (); + + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("an OCEL JSON object") + } + + fn visit_map>(self, mut map: M) -> Result<(), M::Error> { + while let Some(key) = map.next_key::()? { + match key.as_str() { + "eventTypes" => { + for t in map.next_value::>()? { + if let Err(e) = self.ocel.declare_event_type(t) { + return tunnel_append_err::<_, _, M::Error>(self.append_err, e); + } + } + } + "objectTypes" => { + for t in map.next_value::>()? { + if let Err(e) = self.ocel.declare_object_type(t) { + return tunnel_append_err::<_, _, M::Error>(self.append_err, e); + } + } + } + "events" => { + map.next_value_seed(OcelSeqSeed:: { + ocel: self.ocel, + append_err: self.append_err, + expecting: "an array of OCEL events", + append: |a: &mut A, e: OCELEvent| { + a.append_event( + e.id, + &e.event_type, + e.time, + e.attributes, + e.relationships, + ) + }, + _t: std::marker::PhantomData, + })?; + } + "objects" => { + map.next_value_seed(OcelSeqSeed:: { + ocel: self.ocel, + append_err: self.append_err, + expecting: "an array of OCEL objects", + append: |a: &mut A, o: OCELObject| { + a.append_object(o.id, &o.object_type, o.attributes, o.relationships) + }, + _t: std::marker::PhantomData, + })?; + } + _ => { + let _: IgnoredAny = map.next_value()?; + } + } + } + Ok(()) + } +} + +/// Capture an [`AppendableOCEL`] error in `slot` and return a placeholder serde error. +/// The outer [`import_ocel_json_into`] checks `slot` first and returns the typed error, +/// preserving fidelity that `serde::de::Error::custom` would otherwise lose. +fn tunnel_append_err(slot: &mut Option, err: E) -> Result { + *slot = Some(err); + Err(DE::custom("append error")) +} + +/// Stream a JSON sequence (`events` or `objects`) element-by-element into the OCEL. +struct OcelSeqSeed<'a, A: AppendableOCEL, T, F> { + ocel: &'a mut A, + append_err: &'a mut Option, + expecting: &'static str, + append: F, + _t: std::marker::PhantomData, +} + +impl<'a, 'de, A, T, F> DeserializeSeed<'de> for OcelSeqSeed<'a, A, T, F> +where + A: AppendableOCEL, + T: serde::Deserialize<'de>, + F: FnMut(&mut A, T) -> Result<(), A::Error>, +{ + type Value = (); + fn deserialize>(self, de: D) -> Result<(), D::Error> { + de.deserialize_seq(self) + } +} + +impl<'a, 'de, A, T, F> Visitor<'de> for OcelSeqSeed<'a, A, T, F> +where + A: AppendableOCEL, + T: serde::Deserialize<'de>, + F: FnMut(&mut A, T) -> Result<(), A::Error>, +{ + type Value = (); + + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.expecting) + } + + fn visit_seq>(mut self, mut seq: S) -> Result<(), S::Error> { + while let Some(item) = seq.next_element::()? { + if let Err(err) = (self.append)(self.ocel, item) { + return tunnel_append_err::<_, _, S::Error>(self.append_err, err); + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::event_data::object_centric::linked_ocel::{LinkedOCELAccess, SlimLinkedOCEL}; + use crate::test_utils::{get_test_data_path, sort_ocel_for_equality_compare}; + use std::collections::HashMap; + use std::fs; + + fn fixture_bytes() -> Vec { + fs::read( + get_test_data_path() + .join("ocel") + .join("pm4py-ocel20_example.jsonocel"), + ) + .unwrap() + } + + /// Regression: `&OCEL` JSON output must remain byte-identical to a derived + /// `serde_json::to_vec(&ocel)` reference. + #[test] + fn export_byte_identical_for_ocel() { + let bytes = fixture_bytes(); + let ocel = import_ocel_json_slice(&bytes).unwrap(); + let exported = export_ocel_json_to_vec(&ocel).unwrap(); + let reference = serde_json::to_vec(&ocel).unwrap(); + assert_eq!(exported, reference); + } + + /// Streaming export from `SlimLinkedOCEL` reimports as the original `OCEL`. + #[test] + fn export_slim_roundtrip() { + let bytes = fixture_bytes(); + let mut ocel = import_ocel_json_slice(&bytes).unwrap(); + let slim = SlimLinkedOCEL::from_ocel(ocel.clone()); + let bytes_slim = export_ocel_json_to_vec(&slim).unwrap(); + let mut back = import_ocel_json_slice(&bytes_slim).unwrap(); + sort_ocel_for_equality_compare(&mut ocel); + sort_ocel_for_equality_compare(&mut back); + assert_eq!(back.event_types, ocel.event_types); + assert_eq!(back.object_types, ocel.object_types); + let evs_ref: HashMap<&str, _> = ocel.events.iter().map(|e| (e.id.as_str(), e)).collect(); + let evs_back: HashMap<&str, _> = back.events.iter().map(|e| (e.id.as_str(), e)).collect(); + assert_eq!(evs_ref, evs_back); + let obs_ref: HashMap<&str, _> = ocel.objects.iter().map(|o| (o.id.as_str(), o)).collect(); + let obs_back: HashMap<&str, _> = back.objects.iter().map(|o| (o.id.as_str(), o)).collect(); + assert_eq!(obs_ref, obs_back); + } + + /// Streaming import tolerates a misordered JSON: events/objects appearing before their + /// type lists are auto-declared by the `AppendableOCEL`, and the later type declarations + /// merge missing attributes without reordering. + #[test] + fn import_into_slim_streaming_misordered() { + let bytes = fixture_bytes(); + // Reorder top-level keys so events/objects precede their type lists. + let v: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + let m = v.as_object().unwrap(); + let mut reordered = serde_json::Map::new(); + reordered.insert("events".into(), m["events"].clone()); + reordered.insert("objects".into(), m["objects"].clone()); + reordered.insert("eventTypes".into(), m["eventTypes"].clone()); + reordered.insert("objectTypes".into(), m["objectTypes"].clone()); + let bytes_misordered = serde_json::to_vec(&serde_json::Value::Object(reordered)).unwrap(); + + let mut slim = SlimLinkedOCEL::new(); + import_ocel_json_into(bytes_misordered.as_slice(), &mut slim).unwrap(); + slim.finalize().unwrap(); + let via_ocel = SlimLinkedOCEL::from_ocel(import_ocel_json_slice(&bytes).unwrap()); + + let mut a = slim.construct_ocel(); + let mut b = via_ocel.construct_ocel(); + sort_ocel_for_equality_compare(&mut a); + sort_ocel_for_equality_compare(&mut b); + let a_evs: HashMap<&str, _> = a.events.iter().map(|e| (e.id.as_str(), e)).collect(); + let b_evs: HashMap<&str, _> = b.events.iter().map(|e| (e.id.as_str(), e)).collect(); + assert_eq!(a_evs, b_evs); + let a_obs: HashMap<&str, _> = a.objects.iter().map(|o| (o.id.as_str(), o)).collect(); + let b_obs: HashMap<&str, _> = b.objects.iter().map(|o| (o.id.as_str(), o)).collect(); + assert_eq!(a_obs, b_obs); + } + + /// Sink errors (e.g., a `DuplicateEventId` from `SlimLinkedOCEL`) surface from + /// `import_ocel_json_into` as the typed sink error, not as a generic serde error. + #[test] + fn import_into_slim_surfaces_sink_error() { + let json = br#"{ + "eventTypes": [{"name": "x", "attributes": []}], + "objectTypes": [], + "events": [ + {"id": "dup", "type": "x", "time": "2024-01-01T00:00:00Z", "attributes": [], "relationships": []}, + {"id": "dup", "type": "x", "time": "2024-01-02T00:00:00Z", "attributes": [], "relationships": []} + ], + "objects": [] + }"#; + let mut slim = SlimLinkedOCEL::new(); + let err = import_ocel_json_into(json.as_slice(), &mut slim).unwrap_err(); + match err { + OCELIOError::Other(s) => assert!( + s.contains("Duplicate event id: dup"), + "expected DuplicateEventId, got {s:?}" + ), + other => panic!("expected typed sink error, got {other:?}"), + } + } + + /// Streaming import directly into `SlimLinkedOCEL` matches the via-`from_ocel` baseline. + #[test] + fn import_into_slim_streaming() { + let bytes = fixture_bytes(); + let mut slim = SlimLinkedOCEL::new(); + import_ocel_json_into(bytes.as_slice(), &mut slim).unwrap(); + slim.finalize().unwrap(); + let via_ocel = SlimLinkedOCEL::from_ocel(import_ocel_json_slice(&bytes).unwrap()); + + let mut a = slim.construct_ocel(); + let mut b = via_ocel.construct_ocel(); + sort_ocel_for_equality_compare(&mut a); + sort_ocel_for_equality_compare(&mut b); + assert_eq!(a.event_types, b.event_types); + assert_eq!(a.object_types, b.object_types); + let a_evs: HashMap<&str, _> = a.events.iter().map(|e| (e.id.as_str(), e)).collect(); + let b_evs: HashMap<&str, _> = b.events.iter().map(|e| (e.id.as_str(), e)).collect(); + assert_eq!(a_evs, b_evs); + let a_obs: HashMap<&str, _> = a.objects.iter().map(|o| (o.id.as_str(), o)).collect(); + let b_obs: HashMap<&str, _> = b.objects.iter().map(|o| (o.id.as_str(), o)).collect(); + assert_eq!(a_obs, b_obs); + } } diff --git a/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/duckdb_ocel_export.rs b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/duckdb_ocel_export.rs index 64c6a4f3..5fbe302c 100644 --- a/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/duckdb_ocel_export.rs +++ b/process_mining/src/core/event_data/object_centric/ocel_sql/duckdb/duckdb_ocel_export.rs @@ -1,21 +1,31 @@ -use crate::core::event_data::object_centric::ocel_struct::OCEL; +use crate::core::event_data::object_centric::{ocel_struct::OCEL, readable::ReadableOCEL}; use super::super::export::export_ocel_to_sql_con; use super::super::*; use ::duckdb::Connection; +use macros_process_mining::register_binding; /// -/// Export an [`OCEL`] to an `DuckDB` file at the specified path +/// Export an OCEL to a `DuckDB` file at the specified path /// /// Note: This function is only available if the `ocel-duckdb` feature is enabled. /// -pub fn export_ocel_duckdb_to_path>( - ocel: &OCEL, - path: P, -) -> Result<(), DatabaseError> { +pub fn export_ocel_duckdb_to_path(ocel: &O, path: P) -> Result<(), DatabaseError> +where + P: AsRef, + O: ReadableOCEL + ?Sized, +{ if path.as_ref().exists() { let _ = std::fs::remove_file(&path); } let con = Connection::open(path)?; export_ocel_to_sql_con(&con, ocel) } + +#[register_binding(name = "export_ocel_duckdb_to_path", stringify_error)] +fn export_ocel_duckdb_to_path_binding( + ocel: &OCEL, + path: impl AsRef, +) -> Result<(), DatabaseError> { + export_ocel_duckdb_to_path(ocel, path) +} diff --git a/process_mining/src/core/event_data/object_centric/ocel_sql/export.rs b/process_mining/src/core/event_data/object_centric/ocel_sql/export.rs index ad83b21f..0af7d3eb 100644 --- a/process_mining/src/core/event_data/object_centric/ocel_sql/export.rs +++ b/process_mining/src/core/event_data/object_centric/ocel_sql/export.rs @@ -1,17 +1,16 @@ -use std::collections::HashMap; - -use crate::core::event_data::object_centric::ocel_struct::{OCELTypeAttribute, OCEL}; +use crate::core::event_data::object_centric::readable::ReadableOCEL; use super::*; +/// Export an OCEL log to an SQL Database connection. /// -/// Export an [`OCEL`] log to an SQL Database connection -/// -/// Note: This function is only available if the `ocel-sqlite` or the `ocel-duckdb` feature is enabled. +/// Accepts any [`ReadableOCEL`] backend. /// -pub fn export_ocel_to_sql_con<'a, DC: Into>>( - con: DC, - ocel: &OCEL, -) -> Result<(), DatabaseError> { +/// Available with the `ocel-sqlite` or `ocel-duckdb` feature. +pub fn export_ocel_to_sql_con<'a, DC, O>(con: DC, ocel: &O) -> Result<(), DatabaseError> +where + DC: Into>, + O: ReadableOCEL + ?Sized, +{ let con = con.into(); con.execute_no_params("BEGIN TRANSACTION")?; // event map type @@ -47,9 +46,8 @@ pub fn export_ocel_to_sql_con<'a, DC: Into>>( con.execute_no_params("COMMIT")?; con.execute_no_params("BEGIN TRANSACTION")?; - let mut et_attr_map: HashMap<&String, &Vec> = HashMap::new(); // Tables for event types - for et in &ocel.event_types { + for et in ocel.event_types() { let mut attr_cols = et .attributes .iter() @@ -65,7 +63,6 @@ pub fn export_ocel_to_sql_con<'a, DC: Into>>( if !attr_cols.is_empty() { attr_cols.push(','); } - et_attr_map.insert(&et.name, &et.attributes); con.execute_no_params(&format!(r#"CREATE TABLE IF NOT EXISTS "event_{}" ("{OCEL_ID_COLUMN}" TEXT, "{OCEL_TIME_COLUMN}" TIMESTAMP,{attr_cols} PRIMARY KEY("{OCEL_ID_COLUMN}"))"#,clean_sql_name(&et.name)))?; con.execute( @@ -74,10 +71,8 @@ pub fn export_ocel_to_sql_con<'a, DC: Into>>( )?; } - let mut ot_attr_map: HashMap<&String, &Vec> = HashMap::new(); - // Tables for object types - for ot in &ocel.object_types { + for ot in ocel.object_types() { let mut attr_cols = ot .attributes .iter() @@ -92,9 +87,7 @@ pub fn export_ocel_to_sql_con<'a, DC: Into>>( .join(", "); if !attr_cols.is_empty() { attr_cols.insert_str(0, ", "); - // attr_cols.push(','); } - ot_attr_map.insert(&ot.name, &ot.attributes); con.execute_no_params(&format!(r#"CREATE TABLE IF NOT EXISTS "object_{}" ("{OCEL_ID_COLUMN}" TEXT, "{OCEL_TIME_COLUMN}" TIMESTAMP, {OCEL_CHANGED_FIELD} TEXT{attr_cols})"#,clean_sql_name(&ot.name)))?; con.execute( @@ -103,42 +96,46 @@ pub fn export_ocel_to_sql_con<'a, DC: Into>>( )?; } - con.add_objects("object", ocel.objects.iter())?; - - for ot in &ocel.object_types { - let obs = ocel.objects.iter().filter(|ob| ob.object_type == ot.name); + con.add_objects("object", ocel.iter_objects())?; - con.add_object_changes_for_type(&clean_sql_name(&format!("object_{}", ot.name)), ot, obs)?; + for ot in ocel.object_types() { + con.add_object_changes_for_type( + &clean_sql_name(&format!("object_{}", ot.name)), + ot, + ocel.iter_objects_of_type(&ot.name), + )?; } con.execute_no_params("COMMIT")?; con.execute_no_params("BEGIN TRANSACTION")?; - con.add_o2o_relationships("object_object", ocel.objects.iter())?; + con.add_o2o_relationships("object_object", ocel.iter_objects())?; con.execute_no_params("COMMIT")?; con.execute_no_params("BEGIN TRANSACTION")?; - con.add_events("event", ocel.events.iter())?; - - for et in &ocel.event_types { - let evs = ocel.events.iter().filter(|ob| ob.event_type == et.name); + con.add_events("event", ocel.iter_events())?; - con.add_event_attributes_for_type(&clean_sql_name(&format!("event_{}", et.name)), et, evs)?; + for et in ocel.event_types() { + con.add_event_attributes_for_type( + &clean_sql_name(&format!("event_{}", et.name)), + et, + ocel.iter_events_of_type(&et.name), + )?; } con.execute_no_params("COMMIT")?; con.execute_no_params("BEGIN TRANSACTION")?; - con.add_e2o_relationships("event_object", ocel.events.iter())?; + con.add_e2o_relationships("event_object", ocel.iter_events())?; con.execute_no_params("COMMIT")?; con.execute_no_params("BEGIN TRANSACTION")?; - for ot in &ocel.object_types { + for ot in ocel.object_types() { con.execute_no_params(&format!( r#"CREATE INDEX IF NOT EXISTS "{}_obid" ON "object_{}" ("{OCEL_ID_COLUMN}" ASC)"#, clean_sql_name(&ot.name), clean_sql_name(&ot.name) ))?; } - for et in &ocel.event_types { + for et in ocel.event_types() { con.execute_no_params(&format!( r#"CREATE INDEX IF NOT EXISTS "{}_evid" ON "event_{}" ("{OCEL_ID_COLUMN}" ASC)"#, clean_sql_name(&et.name), diff --git a/process_mining/src/core/event_data/object_centric/ocel_sql/mod.rs b/process_mining/src/core/event_data/object_centric/ocel_sql/mod.rs index 5c8673fd..12525a50 100644 --- a/process_mining/src/core/event_data/object_centric/ocel_sql/mod.rs +++ b/process_mining/src/core/event_data/object_centric/ocel_sql/mod.rs @@ -2,6 +2,8 @@ //! //! 🔐 Requires the `ocel-sqlite` or `ocel-duckdb` feature to be enabled. #![cfg(not(all(not(feature = "ocel-duckdb"), not(feature = "ocel-sqlite"))))] +use std::borrow::Cow; + use chrono::DateTime; pub(crate) const OCEL_ID_COLUMN: &str = "ocel_id"; @@ -51,6 +53,8 @@ pub(crate) fn sql_type_to_ocel(s: &str) -> OCELAttributeType { "REAL" => OCELAttributeType::Float, // Used by duckdb "FLOAT" => OCELAttributeType::Float, + // SQL type written for floats (see `ocel_type_to_sql`); DuckDB PRAGMA reports it as "DOUBLE" + "DOUBLE" | "DOUBLE PRECISION" => OCELAttributeType::Float, "INTEGER" => OCELAttributeType::Integer, "BOOLEAN" => OCELAttributeType::Boolean, "TIMESTAMP" => OCELAttributeType::Time, @@ -61,7 +65,8 @@ pub(crate) fn sql_type_to_ocel(s: &str) -> OCELAttributeType { pub(crate) fn ocel_type_to_sql(attr: &OCELAttributeType) -> &'static str { match attr { OCELAttributeType::String => "TEXT", - OCELAttributeType::Float => "REAL", + // DOUBLE PRECISION instead of REAL for full f64 float support in both DuckDB and SQLite + OCELAttributeType::Float => "DOUBLE PRECISION", OCELAttributeType::Integer => "INTEGER", OCELAttributeType::Boolean => "BOOLEAN", OCELAttributeType::Time => "TIMESTAMP", @@ -184,279 +189,352 @@ impl<'a> DatabaseConnection<'a> { } } - /// Add rows for all OCEL objects to specified database table - pub(crate) fn add_objects(&self, table_name: &str, objects: I) -> Result<(), DatabaseError> + /// Insert one `(id, type)` row per item into `table_name`. Used by both + /// [`add_objects`] (`(id, object_type)`) and [`add_events`] (`(id, event_type)`). + fn add_id_type_rows<'b, T, I, F>( + &self, + table_name: &str, + items: I, + extract: F, + ) -> Result<(), DatabaseError> where - I: IntoIterator, + T: Clone + 'b, + I: IntoIterator>, + F: for<'r> Fn(&'r T) -> [&'r String; 2], { - let object_values = objects.into_iter().map(|o| [&o.id, &o.object_type]); match self { #[cfg(feature = "ocel-sqlite")] DatabaseConnection::SQLITE(connection) => { - for ov in object_values { - connection - .execute(&format!(r#"INSERT INTO "{table_name}" VALUES (?,?)"#), ov)?; + for item in items { + connection.execute( + &format!(r#"INSERT INTO "{table_name}" VALUES (?,?)"#), + extract(&item), + )?; } Ok(()) } #[cfg(feature = "ocel-duckdb")] DatabaseConnection::DUCKDB(connection) => { let mut ap = connection.appender(table_name)?; - Ok(ap.append_rows(object_values)?) + for item in items { + ap.append_row(extract(&item))?; + } + Ok(()) } } } - /// Add rows for all OCEL objects to specified database table - pub(crate) fn add_events(&self, table_name: &str, events: I) -> Result<(), DatabaseError> + + pub(crate) fn add_objects<'b, I>( + &self, + table_name: &str, + objects: I, + ) -> Result<(), DatabaseError> where - I: IntoIterator, + I: IntoIterator>, { - let event_values = events.into_iter().map(|o| [&o.id, &o.event_type]); - match self { - #[cfg(feature = "ocel-sqlite")] - DatabaseConnection::SQLITE(connection) => { - for ov in event_values { - connection - .execute(&format!(r#"INSERT INTO "{table_name}" VALUES (?,?)"#), ov)?; - } - Ok(()) - } - #[cfg(feature = "ocel-duckdb")] - DatabaseConnection::DUCKDB(connection) => { - let mut ap = connection.appender(table_name)?; - Ok(ap.append_rows(event_values)?) - } - } + self.add_id_type_rows(table_name, objects, |o| [&o.id, &o.object_type]) + } + + pub(crate) fn add_events<'b, I>(&self, table_name: &str, events: I) -> Result<(), DatabaseError> + where + I: IntoIterator>, + { + self.add_id_type_rows(table_name, events, |e| [&e.id, &e.event_type]) } /// Add rows for all object changes for _objects of one type_ to the specified database table (e.g., `objects_Orders`) - pub(crate) fn add_object_changes_for_type( + pub(crate) fn add_object_changes_for_type<'b, I>( &self, table_name: &str, object_type: &OCELType, objects: I, ) -> Result<(), DatabaseError> where - I: IntoIterator, + I: IntoIterator>, { - let object_values = objects.into_iter().flat_map(|o| { - let initial_vals: Vec<_> = object_type - .attributes - .iter() - .map(|a| { - let initial_val = o - .attributes - .iter() - .find(|oa| oa.name == a.name && oa.time == DateTime::UNIX_EPOCH); - initial_val.map(|v| v.value.to_string()) - }) - .collect(); - // let v = if initial_vals.is_empty() { - // Vec::default() - // } else { - let v = vec![( - o.id.clone(), - None, - DateTime::UNIX_EPOCH.to_rfc3339(), - initial_vals, - )]; - // }; - v.into_iter().chain( - o.attributes - .iter() - .filter(|a| a.time != DateTime::UNIX_EPOCH) - .map(|a| { - let vals: Vec<_> = object_type - .attributes - .iter() - .map(|ot_attr| { - if a.name == ot_attr.name { - Some(a.value.to_string()) - } else { - None - } - }) - .collect(); - ( - o.id.clone(), - Some(a.name.to_string()), - a.time.to_rfc3339(), - vals, - ) - }), - ) - }); match self { #[cfg(feature = "ocel-sqlite")] DatabaseConnection::SQLITE(connection) => { - for (o_id, changed_field, time, values) in object_values { - let values: Vec<_> = values - .into_iter() - .map(|v| v.map(|v| format!("'{v}'")).unwrap_or("NULL".to_string())) - .collect(); - let mut attr_vals = values.join(", "); - if !attr_vals.is_empty() { - attr_vals.insert_str(0, ", "); - } - connection.execute( - &format!( - r#"INSERT INTO "{table_name}" VALUES (?,?,{}{})"#, - &changed_field - .map(|f| format!("'{f}'")) - .unwrap_or("NULL".to_string()), - attr_vals - ), - [&o_id, &time], - )?; + for o in objects { + write_object_changes_sqlite(connection, table_name, object_type, &o)?; } Ok(()) } #[cfg(feature = "ocel-duckdb")] DatabaseConnection::DUCKDB(connection) => { let mut ap = connection.appender(table_name)?; - let object_values: Vec<_> = object_values.collect(); - let x = object_values.iter().map(|ov| { - let chained: Vec<_> = vec![ - &ov.0 as &dyn ::duckdb::ToSql, - &ov.2 as &dyn ::duckdb::ToSql, - &ov.1 as &dyn ::duckdb::ToSql, - ] - .into_iter() - .chain(ov.3.iter().map(|v| v as &dyn ::duckdb::ToSql)) - .collect(); - - ::duckdb::appender_params_from_iter(chained) - }); - ap.append_rows(x).unwrap(); + for o in objects { + write_object_changes_duckdb(&mut ap, object_type, &o)?; + } Ok(()) } } } - pub(crate) fn add_event_attributes_for_type( + pub(crate) fn add_event_attributes_for_type<'b, I>( &self, table_name: &str, event_type: &OCELType, events: I, ) -> Result<(), DatabaseError> where - I: IntoIterator, + I: IntoIterator>, { - let event_values = events.into_iter().map(|o| { - let values: Vec<_> = event_type - .attributes - .iter() - .map(|a| { - let val = o.attributes.iter().find(|oa| oa.name == a.name); - val.map(|v| v.value.to_string()) - }) - .collect(); - // let v = if initial_vals.is_empty() { - // Vec::default() - // } else { - - (o.id.clone(), o.time.to_rfc3339(), values) - }); match self { #[cfg(feature = "ocel-sqlite")] DatabaseConnection::SQLITE(connection) => { - for (e_id, time, values) in event_values { - let values: Vec<_> = values - .into_iter() - .map(|v| v.map(|v| format!("'{v}'")).unwrap_or("NULL".to_string())) - .collect(); - let mut attr_vals = values.join(", "); - if !attr_vals.is_empty() { - attr_vals.insert_str(0, ", "); - } - connection.execute( - &format!(r#"INSERT INTO "{table_name}" VALUES (?,?{attr_vals})"#), - [&e_id, &time], - )?; + for e in events { + write_event_attrs_sqlite(connection, table_name, event_type, &e)?; } Ok(()) } - #[cfg(feature = "ocel-duckdb")] DatabaseConnection::DUCKDB(connection) => { let mut ap = connection.appender(table_name)?; - let event_values: Vec<_> = event_values.collect(); - let x = event_values.iter().map(|ov| { - let chained: Vec<_> = - vec![&ov.0 as &dyn ::duckdb::ToSql, &ov.1 as &dyn ::duckdb::ToSql] - .into_iter() - .chain(ov.2.iter().map(|v| v as &dyn ::duckdb::ToSql)) - .collect(); - - ::duckdb::appender_params_from_iter(chained) - }); - ap.append_rows(x).unwrap(); + for e in events { + write_event_attrs_duckdb(&mut ap, event_type, &e)?; + } Ok(()) } } } - /// Add rows for all OCEL objects to specified database table - pub(crate) fn add_o2o_relationships( + /// Insert one `(source_id, target_object_id, qualifier)` row per relationship. + /// Used by both [`add_o2o_relationships`] and [`add_e2o_relationships`]. + fn add_relationship_rows<'b, T, I, F>( &self, table_name: &str, - objects: I, + items: I, + extract: F, ) -> Result<(), DatabaseError> where - I: IntoIterator, + T: Clone + 'b, + I: IntoIterator>, + F: for<'r> Fn(&'r T) -> (&'r String, &'r [super::ocel_struct::OCELRelationship]), { - let object_values = objects.into_iter().flat_map(|o| { - o.relationships - .iter() - .map(|r| [&o.id, &r.object_id, &r.qualifier]) - }); match self { #[cfg(feature = "ocel-sqlite")] DatabaseConnection::SQLITE(connection) => { - for ov in object_values { - connection - .execute(&format!(r#"INSERT INTO "{table_name}" VALUES (?,?,?)"#), ov)?; + for item in items { + let (id, rels) = extract(&item); + for r in rels { + connection.execute( + &format!(r#"INSERT INTO "{table_name}" VALUES (?,?,?)"#), + [id, &r.object_id, &r.qualifier], + )?; + } } Ok(()) } #[cfg(feature = "ocel-duckdb")] DatabaseConnection::DUCKDB(connection) => { let mut ap = connection.appender(table_name)?; - Ok(ap.append_rows(object_values)?) + for item in items { + let (id, rels) = extract(&item); + for r in rels { + ap.append_row([id, &r.object_id, &r.qualifier])?; + } + } + Ok(()) } } } - /// Add rows for all OCEL objects to specified database table - pub(crate) fn add_e2o_relationships( + pub(crate) fn add_o2o_relationships<'b, I>( + &self, + table_name: &str, + objects: I, + ) -> Result<(), DatabaseError> + where + I: IntoIterator>, + { + self.add_relationship_rows(table_name, objects, |o| (&o.id, &o.relationships)) + } + + pub(crate) fn add_e2o_relationships<'b, I>( &self, table_name: &str, events: I, ) -> Result<(), DatabaseError> where - I: IntoIterator, + I: IntoIterator>, { - let event_values = events.into_iter().flat_map(|o| { - o.relationships + self.add_relationship_rows(table_name, events, |e| (&e.id, &e.relationships)) + } +} + +#[cfg(feature = "ocel-sqlite")] +fn write_object_changes_sqlite( + connection: &rusqlite::Connection, + table_name: &str, + object_type: &OCELType, + o: &super::ocel_struct::OCELObject, +) -> Result<(), DatabaseError> { + let initial_vals: Vec<_> = object_type + .attributes + .iter() + .map(|a| { + o.attributes .iter() - .map(|r| [&o.id, &r.object_id, &r.qualifier]) - }); - match self { - #[cfg(feature = "ocel-sqlite")] - DatabaseConnection::SQLITE(connection) => { - for ov in event_values { - connection - .execute(&format!(r#"INSERT INTO "{table_name}" VALUES (?,?,?)"#), ov)?; + .find(|oa| oa.name == a.name && oa.time == DateTime::UNIX_EPOCH) + .map(|v| format!("'{}'", v.value)) + .unwrap_or_else(|| "NULL".to_string()) + }) + .collect(); + let mut attr_vals = initial_vals.join(", "); + if !attr_vals.is_empty() { + attr_vals.insert_str(0, ", "); + } + connection.execute( + &format!(r#"INSERT INTO "{table_name}" VALUES (?,?,NULL{attr_vals})"#), + [&o.id, &DateTime::UNIX_EPOCH.to_rfc3339()], + )?; + + for a in o + .attributes + .iter() + .filter(|a| a.time != DateTime::UNIX_EPOCH) + { + let vals: Vec<_> = object_type + .attributes + .iter() + .map(|ot_attr| { + if a.name == ot_attr.name { + format!("'{}'", a.value) + } else { + "NULL".to_string() } - Ok(()) - } - #[cfg(feature = "ocel-duckdb")] - DatabaseConnection::DUCKDB(connection) => { - let mut ap = connection.appender(table_name)?; - Ok(ap.append_rows(event_values)?) - } + }) + .collect(); + let mut attr_vals = vals.join(", "); + if !attr_vals.is_empty() { + attr_vals.insert_str(0, ", "); } + connection.execute( + &format!( + r#"INSERT INTO "{table_name}" VALUES (?,?,'{}'{attr_vals})"#, + a.name + ), + [&o.id, &a.time.to_rfc3339()], + )?; } + Ok(()) +} + +#[cfg(feature = "ocel-duckdb")] +fn write_object_changes_duckdb( + ap: &mut ::duckdb::Appender<'_>, + object_type: &OCELType, + o: &super::ocel_struct::OCELObject, +) -> Result<(), DatabaseError> { + let initial_vals: Vec> = object_type + .attributes + .iter() + .map(|a| { + o.attributes + .iter() + .find(|oa| oa.name == a.name && oa.time == DateTime::UNIX_EPOCH) + .map(|v| v.value.to_string()) + }) + .collect(); + let unix = DateTime::UNIX_EPOCH.naive_utc(); + let no_field: Option = None; + let chained: Vec<&dyn ::duckdb::ToSql> = vec![ + &o.id as &dyn ::duckdb::ToSql, + &unix as &dyn ::duckdb::ToSql, + &no_field as &dyn ::duckdb::ToSql, + ] + .into_iter() + .chain(initial_vals.iter().map(|v| v as &dyn ::duckdb::ToSql)) + .collect(); + ap.append_row(::duckdb::appender_params_from_iter(chained))?; + + for a in o + .attributes + .iter() + .filter(|a| a.time != DateTime::UNIX_EPOCH) + { + let vals: Vec> = object_type + .attributes + .iter() + .map(|ot_attr| { + if a.name == ot_attr.name { + Some(a.value.to_string()) + } else { + None + } + }) + .collect(); + let time_naive = a.time.naive_utc(); + let name_opt: Option = Some(a.name.clone()); + let chained: Vec<&dyn ::duckdb::ToSql> = vec![ + &o.id as &dyn ::duckdb::ToSql, + &time_naive as &dyn ::duckdb::ToSql, + &name_opt as &dyn ::duckdb::ToSql, + ] + .into_iter() + .chain(vals.iter().map(|v| v as &dyn ::duckdb::ToSql)) + .collect(); + ap.append_row(::duckdb::appender_params_from_iter(chained))?; + } + Ok(()) +} + +#[cfg(feature = "ocel-sqlite")] +fn write_event_attrs_sqlite( + connection: &rusqlite::Connection, + table_name: &str, + event_type: &OCELType, + e: &super::ocel_struct::OCELEvent, +) -> Result<(), DatabaseError> { + let vals: Vec<_> = event_type + .attributes + .iter() + .map(|a| { + e.attributes + .iter() + .find(|ea| ea.name == a.name) + .map(|v| format!("'{}'", v.value)) + .unwrap_or_else(|| "NULL".to_string()) + }) + .collect(); + let mut attr_vals = vals.join(", "); + if !attr_vals.is_empty() { + attr_vals.insert_str(0, ", "); + } + connection.execute( + &format!(r#"INSERT INTO "{table_name}" VALUES (?,?{attr_vals})"#), + [&e.id, &e.time.to_rfc3339()], + )?; + Ok(()) +} + +#[cfg(feature = "ocel-duckdb")] +fn write_event_attrs_duckdb( + ap: &mut ::duckdb::Appender<'_>, + event_type: &OCELType, + e: &super::ocel_struct::OCELEvent, +) -> Result<(), DatabaseError> { + let vals: Vec> = event_type + .attributes + .iter() + .map(|a| { + e.attributes + .iter() + .find(|ea| ea.name == a.name) + .map(|v| v.value.to_string()) + }) + .collect(); + let time_naive = e.time.naive_utc(); + let chained: Vec<&dyn ::duckdb::ToSql> = vec![ + &e.id as &dyn ::duckdb::ToSql, + &time_naive as &dyn ::duckdb::ToSql, + ] + .into_iter() + .chain(vals.iter().map(|v| v as &dyn ::duckdb::ToSql)) + .collect(); + ap.append_row(::duckdb::appender_params_from_iter(chained))?; + Ok(()) } #[cfg(test)] diff --git a/process_mining/src/core/event_data/object_centric/ocel_sql/sqlite/sqlite_ocel_export.rs b/process_mining/src/core/event_data/object_centric/ocel_sql/sqlite/sqlite_ocel_export.rs index a417df74..9e9f78df 100644 --- a/process_mining/src/core/event_data/object_centric/ocel_sql/sqlite/sqlite_ocel_export.rs +++ b/process_mining/src/core/event_data/object_centric/ocel_sql/sqlite/sqlite_ocel_export.rs @@ -1,18 +1,19 @@ -use crate::core::event_data::object_centric::ocel_struct::OCEL; +use crate::core::event_data::object_centric::readable::ReadableOCEL; use super::super::export::export_ocel_to_sql_con; use super::super::*; use rusqlite::Connection; /// -/// Export an [`OCEL`] to an `SQLite` file at the specified path +/// Export an OCEL to an `SQLite` file at the specified path /// /// Note: This function is only available if the `ocel-sqlite` feature is enabled. /// -pub fn export_ocel_sqlite_to_path>( - ocel: &OCEL, - path: P, -) -> Result<(), DatabaseError> { +pub fn export_ocel_sqlite_to_path(ocel: &O, path: P) -> Result<(), DatabaseError> +where + P: AsRef, + O: ReadableOCEL + ?Sized, +{ if path.as_ref().exists() { let _ = std::fs::remove_file(&path); } @@ -21,10 +22,13 @@ pub fn export_ocel_sqlite_to_path>( } /// -/// Export an [`OCEL`] to an `SQLite` to a byte array +/// Export an OCEL to a `SQLite` byte array /// /// Note: This function is only available if the `ocel-sqlite` feature is enabled. -pub fn export_ocel_sqlite_to_vec(ocel: &OCEL) -> Result, DatabaseError> { +pub fn export_ocel_sqlite_to_vec(ocel: &O) -> Result, DatabaseError> +where + O: ReadableOCEL + ?Sized, +{ let con = Connection::open_in_memory()?; export_ocel_to_sql_con(&con, ocel)?; let data = con.serialize(rusqlite::MAIN_DB)?; diff --git a/process_mining/src/core/event_data/object_centric/ocel_struct.rs b/process_mining/src/core/event_data/object_centric/ocel_struct.rs index 8a167171..e767b6aa 100644 --- a/process_mining/src/core/event_data/object_centric/ocel_struct.rs +++ b/process_mining/src/core/event_data/object_centric/ocel_struct.rs @@ -289,6 +289,45 @@ impl OCELAttributeValue { OCELAttributeValue::Null => OCELAttributeType::Null, } } + + /// Try to coerce this value so its variant matches `target`. Returns `Some(coerced)` + /// when a lossless or parse-based conversion is defined, `None` otherwise. + /// + /// Defined conversions: + /// anything -> String (via `Display` / RFC3339) + /// Integer -> Float (lossless within `i64` -> `f64`) + /// Float -> Integer (only if `fract() == 0.0` and within `i64` range) + /// String -> Integer (parse via `i64::from_str`) + /// String -> Float (parse via `f64::from_str`) + /// String -> Boolean (case-insensitive `"true"`/`"false"`) + /// String -> Time (parse via RFC3339) + pub fn try_coerce_to(&self, target: OCELAttributeType) -> Option { + use OCELAttributeType as T; + use OCELAttributeValue::*; + match (target, self) { + (T::String, v @ (Integer(_) | Float(_) | Boolean(_) | Time(_))) => { + Some(String(v.to_string())) + } + (T::Float, Integer(i)) => Some(Float(*i as f64)), + (T::Integer, Float(f)) => { + let i = *f as i64; + if (i as f64) == *f { + Some(Integer(i)) + } else { + None + } + } + (T::Integer, String(s)) => s.parse::().ok().map(Integer), + (T::Float, String(s)) => s.parse::().ok().map(Float), + (T::Boolean, String(s)) => match s.to_ascii_lowercase().as_str() { + "true" => Some(Boolean(true)), + "false" => Some(Boolean(false)), + _ => None, + }, + (T::Time, String(s)) => DateTime::parse_from_rfc3339(s).ok().map(Time), + _ => None, + } + } } impl From for OCELAttributeValue { @@ -404,16 +443,21 @@ impl OCELAttributeType { /// See [`OCELAttributeType::from_type_str`] for the reverse functionality. /// pub fn to_type_string(&self) -> String { + self.as_type_str().to_string() + } + + /// Same as [`Self::to_type_string`] but returns a `&'static str`, suitable for + /// hot-path comparisons that would otherwise allocate a `String` per call. + pub fn as_type_str(&self) -> &'static str { match self { OCELAttributeType::String => "string", OCELAttributeType::Float => "float", OCELAttributeType::Boolean => "boolean", OCELAttributeType::Integer => "integer", OCELAttributeType::Time => "time", - // Null is not a real attribute type + // Null is not a real OCEL attribute type; map to "string" for compatibility. OCELAttributeType::Null => "string", } - .to_string() } /// diff --git a/process_mining/src/core/event_data/object_centric/ocel_xml/xml_ocel_export.rs b/process_mining/src/core/event_data/object_centric/ocel_xml/xml_ocel_export.rs index e1a6d401..090bedbe 100644 --- a/process_mining/src/core/event_data/object_centric/ocel_xml/xml_ocel_export.rs +++ b/process_mining/src/core/event_data/object_centric/ocel_xml/xml_ocel_export.rs @@ -8,28 +8,31 @@ use std::{ }; use crate::{ - core::event_data::object_centric::ocel_struct::{OCELRelationship, OCELTypeAttribute, OCEL}, + core::event_data::object_centric::{ + ocel_struct::{OCELRelationship, OCELTypeAttribute}, + readable::ReadableOCEL, + }, XMLWriterWrapper, }; const OK: Result<(), std::io::Error> = Ok(()); /// -/// Export [`OCEL`] to XML Writer +/// Export an OCEL to an XML Writer /// -pub fn export_ocel_xml<'a, 'b, W>( +pub fn export_ocel_xml<'b, W, O>( writer: impl Into>, - ocel: &'a OCEL, + ocel: &O, ) -> Result<(), quick_xml::Error> where W: Write + 'b, + O: ReadableOCEL + ?Sized, { let mut xml_writer = writer.into(); let writer: &mut Writer = xml_writer.to_xml_writer(); writer.write_event(Event::Decl(BytesDecl::new("1.0", Some("UTF-8"), None)))?; writer.create_element("log").write_inner_content(|w| { - // Write Object Types w.create_element("object-types").write_inner_content(|w| { - for ot in &ocel.object_types { + for ot in ocel.object_types() { w.create_element("object-type") .with_attributes(vec![("name", ot.name.as_str())]) .write_inner_content(|w| { @@ -39,9 +42,8 @@ where } OK })?; - // Write Event Types w.create_element("event-types").write_inner_content(|w| { - for et in &ocel.event_types { + for et in ocel.event_types() { w.create_element("event-type") .with_attributes(vec![("name", et.name.as_str())]) .write_inner_content(|w| { @@ -51,14 +53,12 @@ where } OK })?; - // Write Objects w.create_element("objects").write_inner_content(|w| { - for o in &ocel.objects { + for o in ocel.iter_objects() { w.create_element("object") .with_attribute(("id", o.id.as_str())) .with_attribute(("type", o.object_type.as_str())) .write_inner_content(|w| { - // Write Attributes w.create_element("attributes").write_inner_content(|w| { for oa in &o.attributes { w.create_element("attribute") @@ -73,23 +73,19 @@ where } OK })?; - // Write Relationships write_ocel_relationships(&o.relationships, w)?; OK })?; } OK })?; - - // Write Events w.create_element("events").write_inner_content(|w| { - for e in &ocel.events { + for e in ocel.iter_events() { w.create_element("event") .with_attribute(("id", e.id.as_str())) .with_attribute(("type", e.event_type.as_str())) .with_attribute(("time", e.time.to_rfc3339().as_str())) .write_inner_content(|w| { - // Write Attributes w.create_element("attributes").write_inner_content(|w| { for ea in &e.attributes { w.create_element("attribute") @@ -103,7 +99,6 @@ where } OK })?; - // Write Relationships write_ocel_relationships(&e.relationships, w)?; OK })?; @@ -117,7 +112,7 @@ where } fn write_ocel_type_attrs( - attrs: &Vec, + attrs: &[OCELTypeAttribute], w: &mut Writer, ) -> Result<(), std::io::Error> { w.create_element("attributes").write_inner_content(|w| { @@ -135,7 +130,7 @@ fn write_ocel_type_attrs( } fn write_ocel_relationships( - rels: &Vec, + rels: &[OCELRelationship], w: &mut Writer, ) -> Result<(), std::io::Error> { w.create_element("objects").write_inner_content(|w| { @@ -150,11 +145,12 @@ fn write_ocel_relationships( OK } -/// Export [`OCEL`] to a path -pub fn export_ocel_xml_path>( - ocel: &OCEL, - path: P, -) -> Result<(), quick_xml::Error> { +/// Export an OCEL to a path +pub fn export_ocel_xml_path(ocel: &O, path: P) -> Result<(), quick_xml::Error> +where + P: AsRef, + O: ReadableOCEL + ?Sized, +{ let file = File::create(path)?; export_ocel_xml(&mut Writer::new(BufWriter::new(file)), ocel) } diff --git a/process_mining/src/core/event_data/object_centric/ocel_xml/xml_ocel_import.rs b/process_mining/src/core/event_data/object_centric/ocel_xml/xml_ocel_import.rs index 51bca375..3fe9d2f2 100644 --- a/process_mining/src/core/event_data/object_centric/ocel_xml/xml_ocel_import.rs +++ b/process_mining/src/core/event_data/object_centric/ocel_xml/xml_ocel_import.rs @@ -1,21 +1,28 @@ use std::{ collections::HashMap, + convert::Infallible, io::{BufRead, BufReader}, }; +use chrono::{DateTime, FixedOffset}; use quick_xml::{events::BytesStart, Reader}; use serde::{Deserialize, Serialize}; -use crate::core::event_data::{ - object_centric::{ - io::OCELIOError, - ocel_struct::{ - OCELAttributeType, OCELAttributeValue, OCELEvent, OCELEventAttribute, OCELObject, - OCELObjectAttribute, OCELRelationship, OCELType, OCELTypeAttribute, OCEL, +use crate::core::{ + event_data::{ + object_centric::{ + appendable::AppendableOCEL, + io::OCELIOError, + ocel_struct::{ + OCELAttributeType, OCELAttributeValue, OCELEventAttribute, OCELObjectAttribute, + OCELRelationship, OCELType, OCELTypeAttribute, OCEL, + }, }, + timestamp_utils::parse_timestamp, }, - timestamp_utils::parse_timestamp, + io::read_xml_text_unescaped, }; + /// /// Options for OCEL Import /// @@ -59,19 +66,77 @@ enum Mode { None, } -fn read_to_string(x: &mut &[u8]) -> String { - if let Ok(x_str) = std::str::from_utf8(x) { - if let Ok(escaped) = quick_xml::escape::unescape(x_str) { - return escaped.to_string(); +impl From for OCELIOError { + fn from(x: Infallible) -> Self { + match x {} + } +} + +struct PartialEvent { + id: String, + event_type: String, + time: DateTime, + attributes: Vec, + relationships: Vec, +} + +struct PartialObject { + id: String, + object_type: String, + attributes: Vec, + relationships: Vec, +} + +/// Parse an `` tag and append a Null-valued attribute +/// to `current_object`. Skips on time parse failure (with optional warning). +fn append_object_attr_decl( + t: &BytesStart<'_>, + current_object: &mut Option, + options: &OCELImportOptions, +) -> Result<(), OCELIOError> { + let name = get_attribute_value(t, "name")?; + let time_str = get_attribute_value(t, "time")?; + match parse_timestamp(&time_str, options.date_format.as_deref(), options.verbose) { + Ok(time) => { + current_object + .as_mut() + .unwrap() + .attributes + .push(OCELObjectAttribute { + name, + value: OCELAttributeValue::Null, + time, + }); + } + Err(e) => { + if options.verbose { + eprintln!("Failed to parse time value of attribute: {e}. Will skip this attribute completely for now."); + } } - return x_str.to_string(); } - String::from_utf8_lossy(x).to_string() + Ok(()) +} + +/// Parse an `` tag and append a Null-valued attribute to `current_event`. +fn append_event_attr_decl( + t: &BytesStart<'_>, + current_event: &mut Option, +) -> Result<(), OCELIOError> { + let name = get_attribute_value(t, "name")?; + current_event + .as_mut() + .unwrap() + .attributes + .push(OCELEventAttribute { + name, + value: OCELAttributeValue::Null, + }); + Ok(()) } fn get_attribute_value(t: &BytesStart<'_>, key: &str) -> Result { match t.try_get_attribute(key)? { - Some(attr) => Ok(read_to_string(&mut attr.value.as_ref())), + Some(attr) => Ok(read_xml_text_unescaped(&mut attr.value.as_ref())), None => Err(quick_xml::Error::Io(std::sync::Arc::new( std::io::Error::new( std::io::ErrorKind::NotFound, @@ -128,445 +193,278 @@ fn parse_attribute_value( } /// -/// Import an [`OCEL`] XML file from the given reader +/// Import an OCEL XML stream into an [`AppendableOCEL`] /// -pub fn import_ocel_xml( - reader: &mut Reader, +/// Type declarations, events and objects are appended to `ocel` as they are +/// parsed. The caller is responsible for invoking [`AppendableOCEL::finalize`] +/// afterwards if the implementation requires it. +/// +pub fn import_ocel_xml_into( + reader: &mut Reader, + ocel: &mut A, options: OCELImportOptions, -) -> Result +) -> Result<(), OCELIOError> where - T: BufRead, + R: BufRead, + A: AppendableOCEL, + A::Error: Into, { reader.config_mut().trim_text(true); let mut buf: Vec = Vec::new(); - let mut current_mode: Mode = Mode::None; - let mut ocel = OCEL { - event_types: Vec::new(), - object_types: Vec::new(), - events: Vec::new(), - objects: Vec::new(), - }; - // Object Type, Attribute Name => Attribute Type + let mut current_ev_type: Option = None; + let mut current_ob_type: Option = None; + let mut current_event: Option = None; + let mut current_object: Option = None; + let mut object_attribute_types: HashMap<(String, String), OCELAttributeType> = HashMap::new(); - // Event Type, Attribute Name => Attribute Type let mut event_attribute_types: HashMap<(String, String), OCELAttributeType> = HashMap::new(); let mut has_object_or_event_types_decl = false; + loop { match reader.read_event_into(&mut buf) { Ok(r) => { match r { quick_xml::events::Event::Start(t) => match current_mode { - Mode::None => match t.name().as_ref() { - // Start log parsing - b"log" => current_mode = Mode::Log, - _ => {} // mut x => print_to_string(&mut x, current_mode, "EventStart"), - }, + Mode::None if t.name().as_ref() == b"log" => { + current_mode = Mode::Log; + } Mode::Log => match t.name().as_ref() { b"object-types" => { current_mode = Mode::ObjectTypes; - has_object_or_event_types_decl = true + has_object_or_event_types_decl = true; } b"event-types" => { current_mode = Mode::EventTypes; - has_object_or_event_types_decl = true + has_object_or_event_types_decl = true; } b"objects" => current_mode = Mode::Objects, b"events" => current_mode = Mode::Events, - _ => {} // mut x => print_to_string(&mut x, current_mode, "EventStart"), - }, - Mode::ObjectTypes => match t.name().as_ref() { - b"object-type" => { - let name = get_attribute_value(&t, "name")?; - ocel.object_types.push(OCELType { - name, - attributes: Vec::new(), - }); - current_mode = Mode::ObjectType - } - _ => {} // mut x => print_to_string(&mut x, current_mode, "EventStart"), - }, - Mode::ObjectType => match t.name().as_ref() { - b"attributes" => current_mode = Mode::ObjectTypeAttributes, - _ => {} - }, - Mode::EventType => match t.name().as_ref() { - b"attributes" => current_mode = Mode::EventTypeAttributes, - // mut x => print_to_string(&mut x, current_mode, "EventStart"), - _ => {} - }, - Mode::EventTypes => match t.name().as_ref() { - b"event-type" => { - let name = get_attribute_value(&t, "name")?; - ocel.event_types.push(OCELType { - name, - attributes: Vec::new(), - }); - current_mode = Mode::EventType - } - // mut x => print_to_string(&mut x, current_mode, "EventStart"), - _ => {} - }, - Mode::Objects => match t.name().as_ref() { - b"object" => { - let id = get_attribute_value(&t, "id")?; - let object_type = get_attribute_value(&t, "type")?; - ocel.objects.push(OCELObject { - id, - object_type, - attributes: Vec::new(), - relationships: Vec::new(), - }); - current_mode = Mode::Object - } - // mut x => print_to_string(&mut x, current_mode, "EventStart"), _ => {} }, + Mode::ObjectTypes if t.name().as_ref() == b"object-type" => { + let name = get_attribute_value(&t, "name")?; + current_ob_type = Some(OCELType { + name, + attributes: Vec::new(), + }); + current_mode = Mode::ObjectType; + } + Mode::ObjectType if t.name().as_ref() == b"attributes" => { + current_mode = Mode::ObjectTypeAttributes; + } + Mode::EventType if t.name().as_ref() == b"attributes" => { + current_mode = Mode::EventTypeAttributes; + } + Mode::EventTypes if t.name().as_ref() == b"event-type" => { + let name = get_attribute_value(&t, "name")?; + current_ev_type = Some(OCELType { + name, + attributes: Vec::new(), + }); + current_mode = Mode::EventType; + } + Mode::Objects if t.name().as_ref() == b"object" => { + let id = get_attribute_value(&t, "id")?; + let object_type = get_attribute_value(&t, "type")?; + current_object = Some(PartialObject { + id, + object_type, + attributes: Vec::new(), + relationships: Vec::new(), + }); + current_mode = Mode::Object; + } Mode::Object => match t.name().as_ref() { - b"attributes" => { - // Noop - } - b"objects" => { - // Begin O2O; Noop - } + b"attributes" | b"objects" => {} b"attribute" => { - let name = get_attribute_value(&t, "name")?; - let time_str = get_attribute_value(&t, "time")?; - let time = parse_timestamp( - &time_str, - options.date_format.as_deref(), - options.verbose, - ); - match time { - Ok(time_val) => { - ocel.objects.last_mut().unwrap().attributes.push( - OCELObjectAttribute { - name, - value: OCELAttributeValue::Null, - time: time_val, - }, - ) - } - Err(e) => { - if options.verbose { - eprintln!("Failed to parse time value of attribute: {e}. Will skip this attribute completely for now."); - } - } - } + append_object_attr_decl(&t, &mut current_object, &options)?; } - // mut x => print_to_string(&mut x, current_mode, "EventStart"), - _ => {} - }, - Mode::Events => match t.name().as_ref() { - b"event" => { - let id = get_attribute_value(&t, "id")?; - let event_type = get_attribute_value(&t, "type")?; - let time = get_attribute_value(&t, "time")?; - let time_val = match parse_timestamp( - &time, - options.date_format.as_deref(), - options.verbose, - ) { - Ok(t) => t, - Err(e) => { - return Err(OCELIOError::Xml(quick_xml::Error::Io( - std::sync::Arc::new(std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("Invalid date: {}", e), - )), - ))); - } - }; - ocel.events.push(OCELEvent { - id, - event_type, - attributes: Vec::new(), - relationships: Vec::new(), - time: time_val, - }); - current_mode = Mode::Event - } - // mut x => print_to_string(&mut x, current_mode, "EventStart"), _ => {} }, + Mode::Events if t.name().as_ref() == b"event" => { + let id = get_attribute_value(&t, "id")?; + let event_type = get_attribute_value(&t, "type")?; + let time_str = get_attribute_value(&t, "time")?; + let time = parse_timestamp( + &time_str, + options.date_format.as_deref(), + options.verbose, + ) + .map_err(|e| { + OCELIOError::Xml(quick_xml::Error::Io(std::sync::Arc::new( + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("Invalid date: {}", e), + ), + ))) + })?; + current_event = Some(PartialEvent { + id, + event_type, + time, + attributes: Vec::new(), + relationships: Vec::new(), + }); + current_mode = Mode::Event; + } Mode::Event => match t.name().as_ref() { - b"attributes" => { - // Noop - } + b"attributes" | b"objects" => {} b"attribute" => { - let name = get_attribute_value(&t, "name")?; - ocel.events.last_mut().unwrap().attributes.push( - OCELEventAttribute { - name, - value: OCELAttributeValue::Null, - }, - ) - } - b"objects" => { - // Event-to-Object relations start now + append_event_attr_decl(&t, &mut current_event)?; } _ => {} }, _ => {} }, quick_xml::events::Event::End(t) => match current_mode { - Mode::ObjectTypeAttributes => match t.name().as_ref() { - b"attributes" => current_mode = Mode::ObjectType, - _ => {} - }, - Mode::ObjectType => match t.name().as_ref() { - b"object-type" => current_mode = Mode::ObjectTypes, - _ => {} - }, - Mode::ObjectTypes => match t.name().as_ref() { - b"object-types" => { - // Finished parsing Object Types - current_mode = Mode::Log - } - _ => {} - }, - Mode::EventTypes => match t.name().as_ref() { - b"event-types" => { - // Finished parsing Object Types - current_mode = Mode::Log + Mode::ObjectTypeAttributes if t.name().as_ref() == b"attributes" => { + current_mode = Mode::ObjectType; + } + Mode::ObjectType if t.name().as_ref() == b"object-type" => { + if let Some(ot) = current_ob_type.take() { + ocel.declare_object_type(ot).map_err(Into::into)?; } - _ => {} - }, - Mode::EventType => match t.name().as_ref() { - b"event-type" => current_mode = Mode::EventTypes, - _ => {} - }, - Mode::EventTypeAttributes => match t.name().as_ref() { - b"attributes" => current_mode = Mode::EventType, - _ => {} - }, - Mode::Log => match t.name().as_ref() { - b"log" => { - // Finished parsing Object Types - current_mode = Mode::None + current_mode = Mode::ObjectTypes; + } + Mode::ObjectTypes if t.name().as_ref() == b"object-types" => { + current_mode = Mode::Log; + } + Mode::EventTypes if t.name().as_ref() == b"event-types" => { + current_mode = Mode::Log; + } + Mode::EventType if t.name().as_ref() == b"event-type" => { + if let Some(et) = current_ev_type.take() { + ocel.declare_event_type(et).map_err(Into::into)?; } - _ => {} - }, - Mode::Objects => match t.name().as_ref() { - b"objects" => current_mode = Mode::Log, - _ => {} - }, - Mode::Events => match t.name().as_ref() { - b"events" => current_mode = Mode::Log, - _ => {} - }, - Mode::Object => match t.name().as_ref() { - b"object" => current_mode = Mode::Objects, - b"attribute" => {} - b"attributes" => {} - b"objects" => { - // End O2O + current_mode = Mode::EventTypes; + } + Mode::EventTypeAttributes if t.name().as_ref() == b"attributes" => { + current_mode = Mode::EventType; + } + Mode::Log if t.name().as_ref() == b"log" => { + current_mode = Mode::None; + } + Mode::Objects if t.name().as_ref() == b"objects" => { + current_mode = Mode::Log; + } + Mode::Events if t.name().as_ref() == b"events" => { + current_mode = Mode::Log; + } + Mode::Object if t.name().as_ref() == b"object" => { + if let Some(o) = current_object.take() { + ocel.append_object( + o.id, + &o.object_type, + o.attributes, + o.relationships, + ) + .map_err(Into::into)?; } - _ => {} - }, - Mode::Event => match t.name().as_ref() { - b"event" => current_mode = Mode::Events, - b"objects" => { - // End of E20 Relations - // Noop + current_mode = Mode::Objects; + } + Mode::Event if t.name().as_ref() == b"event" => { + if let Some(e) = current_event.take() { + ocel.append_event( + e.id, + &e.event_type, + e.time, + e.attributes, + e.relationships, + ) + .map_err(Into::into)?; } - b"attribute" => {} - b"attributes" => {} - _ => {} - }, + current_mode = Mode::Events; + } _ => {} }, quick_xml::events::Event::Empty(t) => match current_mode { - Mode::ObjectTypeAttributes => match t.name().as_ref() { - b"attribute" => { - let name = get_attribute_value(&t, "name")?; - let value_type = get_attribute_value(&t, "type")?; - let object_type = &ocel.object_types.last().unwrap().name; - object_attribute_types.insert( - (object_type.clone(), name.clone()), - OCELAttributeType::from_type_str(&value_type), - ); - ocel.object_types - .last_mut() - .unwrap() - .attributes - .push(OCELTypeAttribute { name, value_type }) - } - // mut x => print_to_string(&mut x, current_mode, "EventEmpty"), - _ => {} - }, + Mode::ObjectTypeAttributes if t.name().as_ref() == b"attribute" => { + let name = get_attribute_value(&t, "name")?; + let value_type = get_attribute_value(&t, "type")?; + let ot = current_ob_type.as_mut().unwrap(); + object_attribute_types.insert( + (ot.name.clone(), name.clone()), + OCELAttributeType::from_type_str(&value_type), + ); + ot.attributes.push(OCELTypeAttribute { name, value_type }); + } Mode::Object => match t.name().as_ref() { - b"relationship" => { + b"relationship" | b"relobj" => { let object_id = get_attribute_value(&t, "object-id")?; let qualifier = get_attribute_value(&t, "qualifier")?; - let new_rel: OCELRelationship = OCELRelationship { - object_id, - qualifier, - }; - ocel.objects.last_mut().unwrap().relationships.push(new_rel); - } - // P2P log uses relobj instead of relationship? - // TODO: Remove once fixed - b"relobj" => { - let object_id = get_attribute_value(&t, "object-id")?; - let qualifier = get_attribute_value(&t, "qualifier")?; - let new_rel: OCELRelationship = OCELRelationship { - object_id, - qualifier, - }; - ocel.objects.last_mut().unwrap().relationships.push(new_rel); - } - b"objects" => { - // No O2O, that's fine! - } - b"attributes" => { - // No attributes, that's fine! + current_object.as_mut().unwrap().relationships.push( + OCELRelationship { + object_id, + qualifier, + }, + ); } - - // Empty attributes => null value (?) + b"attributes" | b"objects" => {} b"attribute" => { - let name = get_attribute_value(&t, "name")?; - let time_str = get_attribute_value(&t, "time")?; - let time = parse_timestamp( - &time_str, - options.date_format.as_deref(), - options.verbose, - ); - match time { - Ok(time_val) => { - ocel.objects.last_mut().unwrap().attributes.push( - OCELObjectAttribute { - name, - value: OCELAttributeValue::Null, - time: time_val, - }, - ) - } - Err(e) => { - if options.verbose { - eprintln!("Failed to parse time value of attribute: {e}. Will skip this attribute completely for now."); - } - } - } + append_object_attr_decl(&t, &mut current_object, &options)?; } _ => {} }, Mode::Event => match t.name().as_ref() { - b"attributes" => { - // Noop - } - b"objects" => { - // If they are empty => Noop - } - b"relationship" => { - let object_id = get_attribute_value(&t, "object-id")?; - let qualifier = get_attribute_value(&t, "qualifier")?; - let new_rel: OCELRelationship = OCELRelationship { - object_id, - qualifier, - }; - ocel.events.last_mut().unwrap().relationships.push(new_rel); - } - // Angular log uses object instead? - // TODO: Remove once example logs are updated - // Should use relationship instead - b"object" => { + b"attributes" | b"objects" => {} + b"relationship" | b"object" | b"relobj" => { let object_id = get_attribute_value(&t, "object-id")?; let qualifier = get_attribute_value(&t, "qualifier")?; - let new_rel: OCELRelationship = OCELRelationship { - object_id, - qualifier, - }; - ocel.events.last_mut().unwrap().relationships.push(new_rel); - } - - // P2P log uses relobj instead of relationship? - // TODO: Remove once fixed - b"relobj" => { - let object_id = get_attribute_value(&t, "object-id")?; - let qualifier = get_attribute_value(&t, "qualifier")?; - let new_rel: OCELRelationship = OCELRelationship { - object_id, - qualifier, - }; - ocel.events.last_mut().unwrap().relationships.push(new_rel); - } - // Empty attribute => Null value (?) - b"attribute" => { - let name = get_attribute_value(&t, "name")?; - ocel.events.last_mut().unwrap().attributes.push( - OCELEventAttribute { - name, - value: OCELAttributeValue::Null, + current_event.as_mut().unwrap().relationships.push( + OCELRelationship { + object_id, + qualifier, }, - ) - } - _ => {} - }, - Mode::ObjectType => match t.name().as_ref() { - b"attributes" => { - // No attributes, that's fine! - } - _ => {} - }, - Mode::EventType => match t.name().as_ref() { - b"attributes" => { - // No attributes, that's fine! + ); } - _ => {} - }, - Mode::EventTypeAttributes => match t.name().as_ref() { b"attribute" => { - let name = get_attribute_value(&t, "name")?; - let value_type = get_attribute_value(&t, "type")?; - let event_type = &ocel.event_types.last().unwrap().name; - event_attribute_types.insert( - (event_type.clone(), name.clone()), - OCELAttributeType::from_type_str(&value_type), - ); - ocel.event_types - .last_mut() - .unwrap() - .attributes - .push(OCELTypeAttribute { name, value_type }) + append_event_attr_decl(&t, &mut current_event)?; } _ => {} }, + Mode::ObjectType | Mode::EventType => { + // Empty tag, no-op + } + Mode::EventTypeAttributes if t.name().as_ref() == b"attribute" => { + let name = get_attribute_value(&t, "name")?; + let value_type = get_attribute_value(&t, "type")?; + let et = current_ev_type.as_mut().unwrap(); + event_attribute_types.insert( + (et.name.clone(), name.clone()), + OCELAttributeType::from_type_str(&value_type), + ); + et.attributes.push(OCELTypeAttribute { name, value_type }); + } _ => {} }, quick_xml::events::Event::Text(t) => match current_mode { Mode::Object => { - let str_val = read_to_string(&mut t.as_ref()); - let o = ocel.objects.last_mut().unwrap(); - let attribute = o.attributes.last_mut().unwrap(); - attribute.value = parse_attribute_value( + let str_val = read_xml_text_unescaped(&mut t.as_ref()); + let o = current_object.as_mut().unwrap(); + let attr = o.attributes.last_mut().unwrap(); + attr.value = parse_attribute_value( object_attribute_types - .get(&(o.object_type.clone(), attribute.name.clone())) + .get(&(o.object_type.clone(), attr.name.clone())) .unwrap_or(&OCELAttributeType::String), str_val, &options, ); - // parse_attribute_value } Mode::Event => { - let str_val = read_to_string(&mut t.as_ref()); - let e = ocel.events.last_mut().unwrap(); - let attribute = e.attributes.last_mut().unwrap(); - attribute.value = parse_attribute_value( + let str_val = read_xml_text_unescaped(&mut t.as_ref()); + let e = current_event.as_mut().unwrap(); + let attr = e.attributes.last_mut().unwrap(); + attr.value = parse_attribute_value( event_attribute_types - .get(&(e.event_type.clone(), attribute.name.clone())) + .get(&(e.event_type.clone(), attr.name.clone())) .unwrap_or(&OCELAttributeType::String), str_val, &options, ); } - _ => { - // Remove to not flood output for extended OCELs - // if options.verbose { - // println!("Got text in unexpected mode {current_mode:?}"); - // } - } + _ => {} }, quick_xml::events::Event::Eof => break, _ => {} @@ -574,10 +472,31 @@ where } Err(err) => return Err(err.into()), } + buf.clear(); } if !has_object_or_event_types_decl { return Err(OCELIOError::Other("No object or event types".to_string())); } + Ok(()) +} + +/// +/// Import an [`OCEL`] XML file from the given reader +/// +pub fn import_ocel_xml( + reader: &mut Reader, + options: OCELImportOptions, +) -> Result +where + T: BufRead, +{ + let mut ocel = OCEL { + event_types: Vec::new(), + object_types: Vec::new(), + events: Vec::new(), + objects: Vec::new(), + }; + import_ocel_xml_into(reader, &mut ocel, options)?; Ok(ocel) } diff --git a/process_mining/src/core/event_data/object_centric/readable.rs b/process_mining/src/core/event_data/object_centric/readable.rs new file mode 100644 index 00000000..02241ff9 --- /dev/null +++ b/process_mining/src/core/event_data/object_centric/readable.rs @@ -0,0 +1,148 @@ +//! Read-only OCEL view trait +use std::{borrow::Cow, collections::HashMap}; + +use chrono::{DateTime, FixedOffset}; + +use crate::core::event_data::object_centric::ocel_struct::{ + OCELAttributeValue, OCELEvent, OCELObject, OCELType, OCEL, +}; + +/// Read-only OCEL view used by exports. +pub trait ReadableOCEL { + /// Concrete id-keyed lookup type, returned by [`Self::lookup`]. + type Lookup<'a>: OCELLookup + where + Self: 'a; + /// Get declared event types + fn event_types(&self) -> &[OCELType]; + /// Get declared object types + fn object_types(&self) -> &[OCELType]; + /// Iterate all events + fn iter_events(&self) -> Box> + '_>; + /// Iterate all events in ascending order by timestamp. + fn iter_events_sorted_by_time(&self) -> Box> + '_> { + let mut events: Vec> = self.iter_events().collect(); + events.sort_by_key(|e| e.time); + Box::new(events.into_iter()) + } + /// Iterate all objects + fn iter_objects(&self) -> Box> + '_>; + /// Iterate events of a given type. Default impl filters [`Self::iter_events`]; + /// indexed backends should override for O(matches) instead of O(total events). + fn iter_events_of_type<'a>( + &'a self, + type_name: &'a str, + ) -> Box> + 'a> { + Box::new( + self.iter_events() + .filter(move |e| e.event_type == type_name), + ) + } + /// Iterate objects of a given type. Default impl filters [`Self::iter_objects`]; + /// indexed backends should override for O(matches) instead of O(total objects). + fn iter_objects_of_type<'a>( + &'a self, + type_name: &'a str, + ) -> Box> + 'a> { + Box::new( + self.iter_objects() + .filter(move |o| o.object_type == type_name), + ) + } + /// Build an id-keyed object lookup. + fn lookup(&self) -> Self::Lookup<'_>; +} + +/// Id-keyed access to an OCEL's objects. Built via [`ReadableOCEL::lookup`]. +pub trait OCELLookup { + /// Iterate object ids + fn iter_object_ids<'a>(&'a self) -> Box + 'a>; + /// Resolve `id` to a borrowed `&str` whose lifetime is tied to `self`. + fn get_id_borrow(&self, id: &str) -> Option<&str>; + /// Get the `object_type` name for the given object id. + fn object_type_of(&self, id: &str) -> Option<&str>; + /// Iterate `(attribute_name, value, time)` for the given object. + fn object_attributes<'a>( + &'a self, + id: &str, + ) -> Box)> + 'a>; + /// Iterate O2O relationships for the given object: `(target_object_id, qualifier)`. + fn object_relationships<'a>( + &'a self, + id: &str, + ) -> Box + 'a>; +} + +impl ReadableOCEL for OCEL { + type Lookup<'a> = OCELHashLookup<'a>; + fn event_types(&self) -> &[OCELType] { + &self.event_types + } + fn object_types(&self) -> &[OCELType] { + &self.object_types + } + fn iter_events(&self) -> Box> + '_> { + Box::new(self.events.iter().map(Cow::Borrowed)) + } + fn iter_events_sorted_by_time(&self) -> Box> + '_> { + let mut sorted: Vec<&OCELEvent> = self.events.iter().collect(); + sorted.sort_by_key(|e| e.time); + Box::new(sorted.into_iter().map(Cow::Borrowed)) + } + fn iter_objects(&self) -> Box> + '_> { + Box::new(self.objects.iter().map(Cow::Borrowed)) + } + fn lookup(&self) -> OCELHashLookup<'_> { + OCELHashLookup { + by_id: self.objects.iter().map(|o| (o.id.as_str(), o)).collect(), + ids: self.objects.iter().map(|o| o.id.as_str()).collect(), + } + } +} + +/// Lookup over a raw [`OCEL`], backed by a `HashMap` of id -> object reference. +#[derive(Debug)] +pub struct OCELHashLookup<'a> { + by_id: HashMap<&'a str, &'a OCELObject>, + /// Mirrors `OCEL.objects` insertion order; keeps `iter_object_ids` deterministic. + ids: Vec<&'a str>, +} + +impl<'a> OCELLookup for OCELHashLookup<'a> { + fn iter_object_ids<'b>(&'b self) -> Box + 'b> { + Box::new(self.ids.iter().copied()) + } + fn get_id_borrow(&self, id: &str) -> Option<&str> { + self.by_id.get_key_value(id).map(|(k, _)| *k) + } + fn object_type_of(&self, id: &str) -> Option<&str> { + self.by_id.get(id).map(|o| o.object_type.as_str()) + } + fn object_attributes<'b>( + &'b self, + id: &str, + ) -> Box)> + 'b> + { + match self.by_id.get(id) { + Some(o) => Box::new( + o.attributes + .iter() + .map(|a| (a.name.as_str(), &a.value, a.time)), + ), + None => Box::new(std::iter::empty()), + } + } + fn object_relationships<'b>( + &'b self, + id: &str, + ) -> Box + 'b> { + match self.by_id.get(id) { + Some(o) => Box::new( + o.relationships + .iter() + .map(|r| (r.object_id.as_str(), r.qualifier.as_str())), + ), + None => Box::new(std::iter::empty()), + } + } +} diff --git a/process_mining/src/core/event_data/tests/ocel_xml_import_tests.rs b/process_mining/src/core/event_data/tests/ocel_xml_import_tests.rs index 8e18e2b7..5d92e220 100644 --- a/process_mining/src/core/event_data/tests/ocel_xml_import_tests.rs +++ b/process_mining/src/core/event_data/tests/ocel_xml_import_tests.rs @@ -7,6 +7,7 @@ use std::{ use crate::{ core::event_data::object_centric::{ io::OCELIOError, + linked_ocel::{LinkedOCELAccess, SlimLinkedOCEL}, ocel_json::{import_ocel_json_path, import_ocel_json_slice}, ocel_xml::{ import_ocel_xml_path, xml_ocel_export::export_ocel_xml_path, @@ -14,6 +15,7 @@ use crate::{ }, }, test_utils::get_test_data_path, + Importable, }; fn get_ocel_file_bytes(name: &str) -> Vec { @@ -44,6 +46,45 @@ fn test_ocel_xml_import() { export_ocel_xml_path(&ocel, &export_path).unwrap(); } +#[test] +fn test_ocel_xml_import_json_ground_truth() { + let log_bytes = &get_ocel_file_bytes("order-management.xml"); + let ocel = import_ocel_xml_slice(log_bytes).unwrap(); + let locel = SlimLinkedOCEL::from_ocel(ocel); + let json_path = get_test_data_path() + .join("ocel") + .join("order-management.json"); + let locel_json = SlimLinkedOCEL::import_from_path(&json_path).unwrap(); + let xml_path = get_test_data_path() + .join("ocel") + .join("order-management.xml"); + let locel_xml = SlimLinkedOCEL::import_from_path(&xml_path).unwrap(); + for ob in locel.get_all_obs() { + let ob_id = locel.get_ob_id(ob); + let ob_json = locel_json.get_ob_by_id(ob_id).unwrap(); + let ob_xml = locel_xml.get_ob_by_id(ob_id).unwrap(); + let full_ob = locel.get_full_ob(ob).into_owned(); + let full_ob_json = locel_json.get_full_ob(ob_json).into_owned(); + let full_ob_xml = locel_xml.get_full_ob(ob_xml).into_owned(); + println!("Comparing object {ob_id}"); + assert_eq!(full_ob, full_ob_xml); + // Currently the JSON import does not necessarily parse attributes as the same type (e.g., float instead of int) + assert_eq!(full_ob.relationships, full_ob_json.relationships); + assert_eq!(full_ob.object_type, full_ob_json.object_type); + } + for ev in locel.get_all_evs() { + let ev_id = locel.get_ev_id(ev); + let ev_json = locel_json.get_ev_by_id(ev_id).unwrap(); + let ev_xml = locel_xml.get_ev_by_id(ev_id).unwrap(); + let full_ev = locel.get_full_ev(ev).into_owned(); + let full_ev_json = locel_json.get_full_ev(ev_json).into_owned(); + let full_ev_xml = locel_xml.get_full_ev(ev_xml).into_owned(); + println!("Comparing event {ev_id}"); + assert_eq!(full_ev, full_ev_xml); + assert_eq!(full_ev, full_ev_json); + } +} + #[test] fn test_xes_as_ocel_xml_import() { let xes_path = get_test_data_path().join("xes").join("small-example.xes"); @@ -198,6 +239,33 @@ fn test_ocel_pm4py_log() { ); } +#[test] +fn test_slim_xml_streaming_matches_via_ocel() { + let log_bytes = get_ocel_file_bytes("order-management.xml"); + let via_ocel = SlimLinkedOCEL::from_ocel(import_ocel_xml_slice(&log_bytes).unwrap()); + let streamed = SlimLinkedOCEL::import_from_reader(log_bytes.as_slice(), "xml").unwrap(); + assert_eq!(via_ocel.get_num_evs(), streamed.get_num_evs()); + assert_eq!(via_ocel.get_num_obs(), streamed.get_num_obs()); + + // Compare reconstructed OCELs end-to-end so that relationships, attributes, + // qualifiers, types, and ordering are all checked, not just counts. + use crate::test_utils::sort_ocel_for_equality_compare; + use std::collections::HashMap; + + let mut via = via_ocel.construct_ocel(); + let mut sm = streamed.construct_ocel(); + sort_ocel_for_equality_compare(&mut via); + sort_ocel_for_equality_compare(&mut sm); + assert_eq!(via.event_types, sm.event_types); + assert_eq!(via.object_types, sm.object_types); + let via_evs: HashMap<&str, _> = via.events.iter().map(|e| (e.id.as_str(), e)).collect(); + let sm_evs: HashMap<&str, _> = sm.events.iter().map(|e| (e.id.as_str(), e)).collect(); + assert_eq!(via_evs, sm_evs); + let via_obs: HashMap<&str, _> = via.objects.iter().map(|o| (o.id.as_str(), o)).collect(); + let sm_obs: HashMap<&str, _> = sm.objects.iter().map(|o| (o.id.as_str(), o)).collect(); + assert_eq!(via_obs, sm_obs); +} + #[test] fn test_ocel_pm4py_log_json() { let now = Instant::now(); diff --git a/process_mining/src/core/io.rs b/process_mining/src/core/io.rs index d73da9c4..0a235bfb 100644 --- a/process_mining/src/core/io.rs +++ b/process_mining/src/core/io.rs @@ -24,6 +24,17 @@ pub fn infer_format_from_path(path: &Path) -> Option { .map(|s| s.to_lowercase()) } +/// Decode an XML text node, unescaping entities and falling back to lossy UTF-8. +pub(crate) fn read_xml_text_unescaped(x: &mut &[u8]) -> String { + if let Ok(s) = std::str::from_utf8(x) { + if let Ok(unescaped) = quick_xml::escape::unescape(s) { + return unescaped.into_owned(); + } + return s.to_string(); + } + String::from_utf8_lossy(x).to_string() +} + #[derive(Debug, Clone, Serialize, Deserialize)] /// File Extension and MIME Type /// diff --git a/process_mining/src/core/process_models/case_centric/petri_net/pnml/import_pnml.rs b/process_mining/src/core/process_models/case_centric/petri_net/pnml/import_pnml.rs index 29817a72..b5aa730a 100644 --- a/process_mining/src/core/process_models/case_centric/petri_net/pnml/import_pnml.rs +++ b/process_mining/src/core/process_models/case_centric/petri_net/pnml/import_pnml.rs @@ -3,6 +3,7 @@ use std::{collections::HashMap, io::BufRead}; use uuid::Uuid; use crate::core::{ + io::read_xml_text_unescaped, process_models::case_centric::petri_net::petri_net_struct::{ArcType, Marking, PlaceID}, PetriNet, }; @@ -24,16 +25,6 @@ enum Mode { ArcInscription, } -fn read_to_string(x: &mut &[u8]) -> String { - if let Ok(x_str) = std::str::from_utf8(x) { - if let Ok(unescaped) = quick_xml::escape::unescape(x_str) { - return unescaped.into_owned(); - } - return x_str.to_string(); - } - String::from_utf8_lossy(x).to_string() -} - /// /// Error encountered while parsing PNML /// @@ -148,7 +139,7 @@ where b"place" => { if current_mode == Mode::FinalMarkingsMarking { // Final Marking place - let id_ref = read_to_string( + let id_ref = read_xml_text_unescaped( &mut b .try_get_attribute("idref") .unwrap_or_default() @@ -166,7 +157,7 @@ where .try_get_attribute("id") .unwrap_or_default() .ok_or(PNMLParseError::MissingKey("id"))?; - let place_id_str = read_to_string(&mut place_id.value.as_ref()); + let place_id_str = read_xml_text_unescaped(&mut place_id.value.as_ref()); let uuid = Uuid::new_v4(); current_id = Some(uuid); id_map.insert(place_id_str, uuid); @@ -179,14 +170,14 @@ where .try_get_attribute("id") .unwrap_or_default() .ok_or(PNMLParseError::MissingKey("id"))?; - let trans_id_str = read_to_string(&mut trans_id.value.as_ref()); + let trans_id_str = read_xml_text_unescaped(&mut trans_id.value.as_ref()); let uuid = Uuid::new_v4(); current_id = Some(uuid); id_map.insert(trans_id_str, uuid); pn.add_transition(Some(String::new()), Some(uuid)); } b"arc" => { - let source_id = read_to_string( + let source_id = read_xml_text_unescaped( &mut b .try_get_attribute("source") .unwrap_or_default() @@ -194,7 +185,7 @@ where .value .as_ref(), ); - let target_id = read_to_string( + let target_id = read_xml_text_unescaped( &mut b .try_get_attribute("target") .unwrap_or_default() @@ -277,7 +268,7 @@ where _ => {} }, quick_xml::events::Event::Text(t) => { - let text = read_to_string(&mut t.as_ref()); + let text = read_xml_text_unescaped(&mut t.as_ref()); match current_mode { Mode::TransitionName => { if let Some(trans) = current_id.and_then(|id| pn.transitions.get_mut(&id)) { @@ -318,6 +309,8 @@ where quick_xml::events::Event::Eof => break, _ => {} } + + buf.clear(); } if !encountered_pnml_tag { diff --git a/process_mining/src/core/process_models/object_centric/oc_declare/mod.rs b/process_mining/src/core/process_models/object_centric/oc_declare/mod.rs index 62c2b351..74427023 100644 --- a/process_mining/src/core/process_models/object_centric/oc_declare/mod.rs +++ b/process_mining/src/core/process_models/object_centric/oc_declare/mod.rs @@ -652,11 +652,11 @@ impl EventOrSynthetic { if matches!(self, EventOrSynthetic::Init(_)) { evs.min_by_key(|ev| locel.get_ev_time(*ev)) .copied() - .unwrap_or(0_usize.into()) + .unwrap_or(0_u32.into()) } else { evs.max_by_key(|ev| locel.get_ev_time(*ev)) .copied() - .unwrap_or(0_usize.into()) + .unwrap_or(0_u32.into()) } } } diff --git a/process_mining/src/discovery/case_centric/alphappp/full.rs b/process_mining/src/discovery/case_centric/alphappp/full.rs index 2ae6738e..e74d2848 100644 --- a/process_mining/src/discovery/case_centric/alphappp/full.rs +++ b/process_mining/src/discovery/case_centric/alphappp/full.rs @@ -296,6 +296,11 @@ pub fn cnds_to_names( cnd: &[(Vec, Vec)], ) -> Vec<(Vec, Vec)> { cnd.iter() - .map(|(a, b)| (log_proj.acts_to_names(a), log_proj.acts_to_names(b))) + .map(|(a, b)| { + ( + log_proj.acts_to_names_sorted(a), + log_proj.acts_to_names_sorted(b), + ) + }) .collect() } diff --git a/process_mining/src/discovery/object_centric/dfg.rs b/process_mining/src/discovery/object_centric/dfg.rs new file mode 100644 index 00000000..0b087a1e --- /dev/null +++ b/process_mining/src/discovery/object_centric/dfg.rs @@ -0,0 +1,54 @@ +//! Discover directly-follows graphs (DFG) from object-centric event data. + +use std::collections::HashMap; + +use macros_process_mining::register_binding; +use rayon::prelude::*; + +use crate::core::event_data::object_centric::linked_ocel::{ + slim_linked_ocel::ObjectIndex, LinkedOCELAccess, SlimLinkedOCEL, +}; + +use super::merge_count_maps; + +/// Get the directly-follows graph (DFG) for objects of the given object type. +/// +/// Each entry is `((from_activity, to_activity), count)`, counting adjacent pairs in each +/// object's timestamp-ordered activity trace. Sorted by count descending, ties broken by +/// `(from_activity, to_activity)`. +#[register_binding] +pub fn get_dfg_of_object_type( + ocel: &SlimLinkedOCEL, + ob_type: String, +) -> Vec<((String, String), usize)> { + let obs: Vec = ocel.get_obs_of_type(&ob_type).copied().collect(); + let counts: HashMap<(usize, usize), usize> = obs + .into_par_iter() + .fold(HashMap::new, |mut acc, ob| { + let mut iter = ob.get_obj_activity_trace_evtype_indices(ocel); + if let Some(mut prev) = iter.next() { + for next in iter { + *acc.entry((prev, next)).or_insert(0) += 1; + prev = next; + } + } + acc + }) + .reduce(HashMap::new, merge_count_maps); + let ev_type_names: Vec<&str> = + ::get_ev_types(ocel).collect(); + let mut result: Vec<((String, String), usize)> = counts + .into_iter() + .map(|((from, to), count)| { + ( + ( + ev_type_names[from].to_string(), + ev_type_names[to].to_string(), + ), + count, + ) + }) + .collect(); + result.sort_unstable_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + result +} diff --git a/process_mining/src/discovery/object_centric/mod.rs b/process_mining/src/discovery/object_centric/mod.rs index 85c877e8..0da37267 100644 --- a/process_mining/src/discovery/object_centric/mod.rs +++ b/process_mining/src/discovery/object_centric/mod.rs @@ -1,2 +1,17 @@ //! Object-centric Process Discovery Techniques +pub mod dfg; pub mod oc_declare; +pub mod variants; + +use std::collections::HashMap; + +/// Merge `b` into `a` by summing counts for matching keys. Used as the rayon reduce step. +pub(crate) fn merge_count_maps( + mut a: HashMap, + b: HashMap, +) -> HashMap { + for (k, v) in b { + *a.entry(k).or_insert(0) += v; + } + a +} diff --git a/process_mining/src/discovery/object_centric/variants.rs b/process_mining/src/discovery/object_centric/variants.rs new file mode 100644 index 00000000..232f4a93 --- /dev/null +++ b/process_mining/src/discovery/object_centric/variants.rs @@ -0,0 +1,48 @@ +//! Discover activity-trace variants from object-centric event data. + +use std::collections::HashMap; + +use macros_process_mining::register_binding; +use rayon::prelude::*; + +use crate::core::event_data::object_centric::linked_ocel::{ + slim_linked_ocel::ObjectIndex, LinkedOCELAccess, SlimLinkedOCEL, +}; + +use super::merge_count_maps; + +/// Get all activity-trace variants for objects of the given object type, with their occurrence counts +/// +/// Each entry is a tuple `(activity_trace, count)`, where `activity_trace` is the sequence of event types +/// connected to an object (ordered by event timestamp), and `count` is the number of objects of the +/// requested type that share that exact trace. Sorted by count descending, ties broken by the +/// activity trace. +#[register_binding] +pub fn get_variants_of_object_type( + ocel: &SlimLinkedOCEL, + ob_type: String, +) -> Vec<(Vec, usize)> { + let obs: Vec = ocel.get_obs_of_type(&ob_type).copied().collect(); + let counts: HashMap, usize> = obs + .into_par_iter() + .fold(HashMap::new, |mut acc, ob| { + let trace: Vec = ob.get_obj_activity_trace_evtype_indices(ocel).collect(); + *acc.entry(trace).or_insert(0) += 1; + acc + }) + .reduce(HashMap::new, merge_count_maps); + let ev_type_names: Vec<&str> = + ::get_ev_types(ocel).collect(); + let mut result: Vec<(Vec, usize)> = counts + .into_iter() + .map(|(trace_idx, count)| { + let trace: Vec = trace_idx + .into_iter() + .map(|i| ev_type_names[i].to_string()) + .collect(); + (trace, count) + }) + .collect(); + result.sort_unstable_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + result +} diff --git a/process_mining/src/lib.rs b/process_mining/src/lib.rs index 6f624a81..23c5cb65 100644 --- a/process_mining/src/lib.rs +++ b/process_mining/src/lib.rs @@ -18,6 +18,19 @@ pub use core::io::{Exportable, Importable}; // Re-export main structs for convenience pub use core::{EventLog, PetriNet, OCEL}; +// Re-export OCEL backend traits and the streaming entry points. +pub use core::event_data::object_centric::{ + appendable::AppendableOCEL, + ocel_json::import_ocel_json_into, + ocel_xml::xml_ocel_import::{import_ocel_xml_into, OCELImportOptions}, + readable::{OCELLookup, ReadableOCEL}, +}; + +// Re-exported so callers can construct the `Reader` / `Writer` arguments accepted by +// `import_ocel_xml_into` and `XMLWriterWrapper` without taking a direct `quick-xml` +// dependency. +pub use quick_xml::{Reader as XmlReader, Writer as XmlWriter}; + /// Bindings pub mod bindings; @@ -26,13 +39,38 @@ pub mod bindings; pub mod test_utils { use std::path::PathBuf; - /// Get the based path for test data. + use crate::OCEL; + + /// Get the base path for test data. /// /// Used for internal testing #[allow(unused)] pub fn get_test_data_path() -> PathBuf { std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("test_data") } + + /// Sort all collections inside an [`OCEL`] (events, objects, types, + /// relationships, attributes) by stable keys so two OCELs that are + /// structurally equivalent but inserted in different orders compare + /// equal via `assert_eq!`. + #[allow(unused)] + pub fn sort_ocel_for_equality_compare(ocel: &mut OCEL) { + ocel.events + .sort_by(|x, y| (x.time, &x.id).cmp(&(y.time, &y.id))); + ocel.objects.sort_by(|x, y| x.id.cmp(&y.id)); + ocel.event_types.sort_by(|x, y| x.name.cmp(&y.name)); + ocel.object_types.sort_by(|x, y| x.name.cmp(&y.name)); + for e in &mut ocel.events { + e.relationships + .sort_by(|x, y| (&x.object_id, &x.qualifier).cmp(&(&y.object_id, &y.qualifier))); + } + for o in &mut ocel.objects { + o.attributes + .sort_by(|x, y| (x.time, &x.name).cmp(&(y.time, &y.name))); + o.relationships + .sort_by(|x, y| (&x.object_id, &x.qualifier).cmp(&(&y.object_id, &y.qualifier))); + } + } } /// A wrapper for either an owned or mutable reference to a [`quick_xml::Writer`] diff --git a/r4pm/src/main.rs b/r4pm/src/main.rs index dcccb793..5cab6d20 100644 --- a/r4pm/src/main.rs +++ b/r4pm/src/main.rs @@ -140,6 +140,9 @@ fn main() -> ExitCode { let fn_args = serde_json::Value::Object(params); match bindings::call(binding, &fn_args, &state) { Ok(res) => { + // `call` returns JSON bytes; parse once for the CLI's structured handling. + let res: serde_json::Value = + serde_json::from_slice(&res).unwrap_or(serde_json::Value::Null); if let Some(output_path) = output_path { if let Some(id) = res.as_str() { let state_guard = state.items.read().unwrap();