Skip to content

Commit 41633b6

Browse files
molpopgenmolpopgen
authored
Add metadata type registration via derive macros. (#163)
1 parent 2f14352 commit 41633b6

File tree

7 files changed

+508
-237
lines changed

7 files changed

+508
-237
lines changed

‎Cargo.toml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,12 @@ chrono = {version = "0.4.19", optional = true}
2424
serde = {version = "1.0.118", features = ["derive"], optional = true}
2525
serde_json = {version = "1.0.67", optional = true}
2626
bincode = {version = "1.3.1", optional = true}
27+
tskit-derive = {version = "0.1.0", path = "tskit-derive", optional = true}
2728

2829
[dev-dependencies]
2930
clap = "~2.33.3"
3031
serde = {version = "1.0.118", features = ["derive"]}
32+
serde-pickle = "0.6"
3133
bincode = "1.3.1"
3234
rand = "0.8.3"
3335
rand_distr = "0.4.0"
@@ -39,8 +41,7 @@ pkg-config = "0.3"
3941

4042
[features]
4143
provenance = ["chrono"]
42-
serde_json_metadata = ["serde", "serde_json"]
43-
serde_bincode_metadata = ["serde", "bincode"]
44+
derive = ["tskit-derive", "serde", "serde_json", "bincode"]
4445

4546
[package.metadata.docs.rs]
4647
all-features = true

‎src/_macros.rs‎

Lines changed: 0 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -322,66 +322,6 @@ macro_rules! handle_metadata_return {
322322
};
323323
}
324324

325-
/// Implement [`crate::metadata::MetadataRoundtrip`]
326-
/// for a type using `serde_json`.
327-
///
328-
/// Requires the `serde_json_metadata` feature.
329-
#[cfg(any(doc, feature = "serde_json_metadata"))]
330-
#[macro_export]
331-
macro_rules! serde_json_metadata {
332-
($structname: ty) => {
333-
impl $crate::metadata::MetadataRoundtrip for $structname {
334-
fn encode(&self) -> Result<Vec<u8>, $crate::metadata::MetadataError> {
335-
match serde_json::to_string(self) {
336-
Ok(x) => Ok(x.as_bytes().to_vec()),
337-
Err(e) => {
338-
Err($crate::metadata::MetadataError::RoundtripError { value: Box::new(e) })
339-
}
340-
}
341-
}
342-
343-
fn decode(md: &[u8]) -> Result<Self, $crate::metadata::MetadataError> {
344-
let value: Result<Self, serde_json::Error> = serde_json::from_slice(md);
345-
match value {
346-
Ok(v) => Ok(v),
347-
Err(e) => {
348-
Err($crate::metadata::MetadataError::RoundtripError { value: Box::new(e) })
349-
}
350-
}
351-
}
352-
}
353-
};
354-
}
355-
356-
/// Implement [`crate::metadata::MetadataRoundtrip`]
357-
/// for a type using `bincode`.
358-
///
359-
/// Requires the `serde_bincode_metadata` feature.
360-
#[cfg(any(doc, feature = "serde_bincode_metadata"))]
361-
#[macro_export]
362-
macro_rules! serde_bincode_metadata {
363-
($structname: ty) => {
364-
impl $crate::metadata::MetadataRoundtrip for $structname {
365-
fn encode(&self) -> Result<Vec<u8>, tskit::metadata::MetadataError> {
366-
match bincode::serialize(&self) {
367-
Ok(x) => Ok(x),
368-
Err(e) => {
369-
Err($crate::metadata::MetadataError::RoundtripError { value: Box::new(e) })
370-
}
371-
}
372-
}
373-
fn decode(md: &[u8]) -> Result<Self, tskit::metadata::MetadataError> {
374-
match bincode::deserialize(md) {
375-
Ok(x) => Ok(x),
376-
Err(e) => {
377-
Err($crate::metadata::MetadataError::RoundtripError { value: Box::new(e) })
378-
}
379-
}
380-
}
381-
}
382-
};
383-
}
384-
385325
#[cfg(test)]
386326
mod test {
387327
use crate::error::TskitError;

‎src/lib.rs‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,18 @@
4545
//!
4646
//! * `provenance`
4747
//! * Enables [`provenance`]
48-
//! * `serde_json_metadata`
49-
//! * Enables [`serde_json_metadata`] macro.
50-
//! * `serde_bincode_metadata`
51-
//! * Enables [`serde_bincode_metadata`] macro.
48+
//! * `derive` enables the following derive macros:
49+
//! * [`MutationMetadata`]
50+
//! * [`IndividualMetadata`]
51+
//! * [`SiteMetadata`]
52+
//! * [`EdgeMetadata`]
53+
//! * [`NodeMetadata`]
54+
//! * [`MigrationMetadata`]
55+
//! * [`PopulationMetadata`]
5256
//!
57+
//! To see these derive macros in action, take a look
58+
//! [`here`](metadata).
59+
//!
5360
//! To add features to your `Cargo.toml` file:
5461
//!
5562
//! ```toml

‎src/metadata.rs‎

Lines changed: 176 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -1,88 +1,187 @@
11
//! Support for table row metadata
2+
//!
3+
//! Metadata refers to data that client code may need to associate
4+
//! with table rows, but the data are not necessary to perform algorithms
5+
//! on tables nor on trees.
6+
//!
7+
//! For complete details, see the data model descriptions
8+
//! [`here`](https://tskit.dev/tskit/docs/stable/)
9+
//!
10+
//! The most straightfoward way to implement metadata
11+
//! is to use the optional `derive` feature of `tskit`.
12+
//! This feature enables derive macros to convert
13+
//! your types to metadata types via [`serde`](https://docs.rs/serde).
14+
//!
15+
//! Note that you will need to add `serde` as a dependency of your
16+
//! package, as you will need its `Serialize` and `Deserialize`
17+
//! derive macros available.
18+
//!
19+
//! Without the derive macros provided by tskit, you must `impl` [`MetadataRoundtrip`]
20+
//! and the approprate table metadata tag marker for your type.
21+
//! An example of such "manual" metadata type registration is shown
22+
//! as the last example below.
23+
//!
24+
//! A technical details section follows the examples
25+
//!
26+
//! # Examples
27+
//!
28+
//! ## Mutation metadata encoded as JSON
29+
//!
30+
//! ```
31+
//! # #[cfg(feature = "derive")] {
32+
//! use tskit::handle_metadata_return;
33+
//! use tskit::TableAccess;
34+
//!
35+
//! #[derive(serde::Serialize, serde::Deserialize, tskit::metadata::MutationMetadata)]
36+
//! #[serializer("serde_json")]
37+
//! pub struct MyMutation {
38+
//! origin_time: i32,
39+
//! effect_size: f64,
40+
//! dominance: f64,
41+
//! }
42+
//!
43+
//! let mut tables = tskit::TableCollection::new(100.).unwrap();
44+
//! let mutation = MyMutation{origin_time: 100,
45+
//! effect_size: -1e-4,
46+
//! dominance: 0.25};
47+
//!
48+
//! // Add table row with metadata.
49+
//! let id = tables.add_mutation_with_metadata(0, 0, tskit::MutationId::NULL, 100., None,
50+
//! &mutation).unwrap();
51+
//!
52+
//! // Decode the metadata
53+
//! // The two unwraps are:
54+
//! // 1. Handle Errors vs Option.
55+
//! // 2. Handle the option for the case of no error.
56+
//! let decoded = tables.mutations().metadata::<MyMutation>(id).unwrap().unwrap();
57+
//! assert_eq!(mutation.origin_time, decoded.origin_time);
58+
//! match decoded.effect_size.partial_cmp(&mutation.effect_size) {
59+
//! Some(std::cmp::Ordering::Greater) => assert!(false),
60+
//! Some(std::cmp::Ordering::Less) => assert!(false),
61+
//! Some(std::cmp::Ordering::Equal) => (),
62+
//! None => panic!("bad comparison"),
63+
//! };
64+
//! match decoded.dominance.partial_cmp(&mutation.dominance) {
65+
//! Some(std::cmp::Ordering::Greater) => assert!(false),
66+
//! Some(std::cmp::Ordering::Less) => assert!(false),
67+
//! Some(std::cmp::Ordering::Equal) => (),
68+
//! None => panic!("bad comparison"),
69+
//! };
70+
//! # }
71+
//! ```
72+
//! ## Example: individual metadata implemented via newtypes
73+
//!
74+
//! This time, we use [`bincode`](https://docs.rs/bincode/) via `serde`.
75+
//!
76+
//! ```
77+
//! # #[cfg(feature = "derive")] {
78+
//! use tskit::TableAccess;
79+
//! #[derive(serde::Serialize, serde::Deserialize, PartialEq, PartialOrd)]
80+
//! struct GeneticValue(f64);
81+
//!
82+
//! #[derive(serde::Serialize, serde::Deserialize, tskit::metadata::IndividualMetadata)]
83+
//! #[serializer("bincode")]
84+
//! struct IndividualMetadata {
85+
//! genetic_value: GeneticValue,
86+
//! }
87+
//! let mut tables = tskit::TableCollection::new(100.).unwrap();
88+
//! let individual = IndividualMetadata {
89+
//! genetic_value: GeneticValue(0.0),
90+
//! };
91+
//! let id = tables.add_individual_with_metadata(0, &[], &[tskit::IndividualId::NULL], &individual).unwrap();
92+
//! let decoded = tables.individuals().metadata::<IndividualMetadata>(id).unwrap().unwrap();
93+
//! assert_eq!(decoded.genetic_value.partial_cmp(&individual.genetic_value).unwrap(), std::cmp::Ordering::Equal);
94+
//! # }
95+
//! ```
96+
//!
97+
//! ## Example: manual implementation of all of the traits.
98+
//!
99+
//! Okay, let's do things the hard way.
100+
//! We will use a serializer not supported by `tskit` right now.
101+
//! For fun, we'll use the Python [`pickle`](https://docs.rs/crate/serde-pickle/) format.
102+
//!
103+
//! ```
104+
//! use tskit::TableAccess;
105+
//!
106+
//! #[derive(serde::Serialize, serde::Deserialize)]
107+
//! struct Metadata {
108+
//! data: String,
109+
//! }
110+
//!
111+
//! // Manually implement the metadata round trip trait.
112+
//! // You must propogate any errors back via Box, else
113+
//! // risk a `panic!`.
114+
//! impl tskit::metadata::MetadataRoundtrip for Metadata {
115+
//! fn encode(&self) -> Result<Vec<u8>, tskit::metadata::MetadataError> {
116+
//! match serde_pickle::to_vec(self, true) {
117+
//! Ok(v) => Ok(v),
118+
//! Err(e) => Err(tskit::metadata::MetadataError::RoundtripError{ value: Box::new(e) }),
119+
//! }
120+
//! }
121+
//!
122+
//! fn decode(md: &[u8]) -> Result<Self, tskit::metadata::MetadataError> {
123+
//! match serde_pickle::from_slice(md) {
124+
//! Ok(x) => Ok(x),
125+
//! Err(e) => Err(tskit::metadata::MetadataError::RoundtripError{ value: Box::new(e) }),
126+
//! }
127+
//! }
128+
//! }
129+
//!
130+
//! // If we want this to be, say, node metadata, then we need to mark
131+
//! // it as such:
132+
//! impl tskit::metadata::NodeMetadata for Metadata {}
133+
//!
134+
//! // Ready to rock:
135+
//! let mut tables = tskit::TableCollection::new(1.).unwrap();
136+
//! let id = tables
137+
//! .add_node_with_metadata(
138+
//! 0,
139+
//! 0.0,
140+
//! tskit::PopulationId::NULL,
141+
//! tskit::IndividualId::NULL,
142+
//! &Metadata {
143+
//! data: "Bananas".to_string(),
144+
//! },
145+
//! )
146+
//! .unwrap();
147+
//!
148+
//! let decoded = tables.nodes().metadata::<Metadata>(id).unwrap().unwrap();
149+
//! assert_eq!(decoded.data, "Bananas".to_string());
150+
//! ```
151+
//!
152+
//! # Technial details and notes
153+
//!
154+
//! * The derive macros currently support two `serde` methods:
155+
//! `serde_json` and `bincode`.
156+
//! * A concept like "mutation metadata" is the combination of two traits:
157+
//! [`MetadataRoundtrip`] plus [`MutationMetadata`].
158+
//! The latter is a marker trait.
159+
//! The derive macros handle all of this "boiler plate" for you.
160+
//!
161+
//! ## Limitations/unknowns
162+
//!
163+
//! * We have not yet tested importing metadata encoded using `rust`
164+
//! into `Python` via the `tskit` `Python API`.
2165
3166
use crate::bindings::{tsk_id_t, tsk_size_t};
4167
use thiserror::Error;
5168

6-
/// Enable a type to be used as table metadata
7-
///
8-
/// See [`handle_metadata_return`] for a macro to help implement this trait,
9-
/// and its use in examples below.
10-
///
11-
/// We strongly recommend the use of the [serde](https://serde.rs/) ecosystem
12-
/// for row metadata.
13-
/// For many use cases, we imagine that
14-
/// [bincode](https://crates.io/crates/bincode) will be one of
15-
/// the more useful `serde`-related crates.
16-
///
17-
/// The library provides two macros to facilitate implementing metadata
18-
/// traits:
19-
///
20-
/// * [`serde_json_metadata`]
21-
/// * [`serde_bincode_metadata`]
22-
///
23-
/// These macros are optional features.
24-
/// The feature names are the same as the macro names
25-
///
26-
#[cfg_attr(
27-
feature = "provenance",
28-
doc = r##"
29-
# Examples
30-
31-
## Mutation metadata encoded as JSON
169+
#[cfg(feature = "derive")]
170+
#[doc(hidden)]
171+
pub extern crate tskit_derive;
32172

33-
```
34-
use tskit::handle_metadata_return;
35-
use tskit::TableAccess;
36-
37-
#[derive(serde::Serialize, serde::Deserialize)]
38-
pub struct MyMutation {
39-
origin_time: i32,
40-
effect_size: f64,
41-
dominance: f64,
42-
}
43-
44-
// Implement tskit::metadata::MetadataRoundtrip
45-
tskit::serde_json_metadata!(MyMutation);
46-
47-
impl tskit::metadata::MutationMetadata for MyMutation {}
48-
49-
let mut tables = tskit::TableCollection::new(100.).unwrap();
50-
let mutation = MyMutation{origin_time: 100,
51-
effect_size: -1e-4,
52-
dominance: 0.25};
53-
54-
// Add table row with metadata.
55-
tables.add_mutation_with_metadata(0, 0, tskit::MutationId::NULL, 100., None,
56-
&mutation).unwrap();
57-
58-
// Decode the metadata
59-
// The two unwraps are:
60-
// 1. Handle Errors vs Option.
61-
// 2. Handle the option for the case of no error.
62-
//
63-
// The .into() reflects the fact that metadata fetching
64-
// functions only take a strong ID type, and tskit-rust
65-
// adds Into<strong ID type> for i32 for all strong ID types.
66-
67-
let decoded = tables.mutations().metadata::<MyMutation>(0.into()).unwrap().unwrap();
68-
assert_eq!(mutation.origin_time, decoded.origin_time);
69-
match decoded.effect_size.partial_cmp(&mutation.effect_size) {
70-
Some(std::cmp::Ordering::Greater) => assert!(false),
71-
Some(std::cmp::Ordering::Less) => assert!(false),
72-
Some(std::cmp::Ordering::Equal) => (),
73-
None => panic!("bad comparison"),
173+
#[cfg(feature = "derive")]
174+
#[doc(hidden)]
175+
pub use tskit_derive::{
176+
EdgeMetadata, IndividualMetadata, MigrationMetadata, MutationMetadata, NodeMetadata,
177+
PopulationMetadata, SiteMetadata,
74178
};
75-
match decoded.dominance.partial_cmp(&mutation.dominance) {
76-
Some(std::cmp::Ordering::Greater) => assert!(false),
77-
Some(std::cmp::Ordering::Less) => assert!(false),
78-
Some(std::cmp::Ordering::Equal) => (),
79-
None => panic!("bad comparison"),
80-
};
81-
```
82-
"##
83-
)]
179+
180+
/// Trait marking a type as table metadata
84181
pub trait MetadataRoundtrip {
182+
/// Encode `self` as bytes
85183
fn encode(&self) -> Result<Vec<u8>, MetadataError>;
184+
/// Decond `Self` from bytes
86185
fn decode(md: &[u8]) -> Result<Self, MetadataError>
87186
where
88187
Self: Sized;
@@ -101,7 +200,7 @@ pub trait NodeMetadata: MetadataRoundtrip {}
101200
pub trait EdgeMetadata: MetadataRoundtrip {}
102201
///
103202
/// Marker trait indicating [`MetadataRoundtrip`]
104-
/// for the migratoin table of a [`TableCollection`](crate::TableCollection).
203+
/// for the migration table of a [`TableCollection`](crate::TableCollection).
105204
pub trait MigrationMetadata: MetadataRoundtrip {}
106205

107206
/// Marker trait indicating [`MetadataRoundtrip`]

0 commit comments

Comments
 (0)