Skip to content

Commit 88a30db

Browse files
authored
enhancement(config): add support for any serde_json-capable value for configurable metadata KV pairs (vectordotdev#15818)
1 parent 094e569 commit 88a30db

File tree

13 files changed

+143
-80
lines changed

13 files changed

+143
-80
lines changed

Cargo.lock

+2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

lib/vector-common/src/datetime.rs

+2-8
Original file line numberDiff line numberDiff line change
@@ -114,10 +114,7 @@ impl Configurable for TimeZone {
114114
fn generate_schema(gen: &mut SchemaGenerator) -> Result<SchemaObject, GenerateError> {
115115
let mut local_schema = generate_const_string_schema("local".to_string());
116116
let mut local_metadata = Metadata::<()>::with_description("System local timezone.");
117-
local_metadata.add_custom_attribute(CustomAttribute::KeyValue {
118-
key: "logical_name".to_string(),
119-
value: "Local".to_string(),
120-
});
117+
local_metadata.add_custom_attribute(CustomAttribute::kv("logical_name", "Local"));
121118
apply_metadata(&mut local_schema, local_metadata);
122119

123120
let mut tz_metadata = Metadata::with_title("A named timezone.");
@@ -126,10 +123,7 @@ impl Configurable for TimeZone {
126123
127124
[tzdb]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones"#,
128125
);
129-
tz_metadata.add_custom_attribute(CustomAttribute::KeyValue {
130-
key: "logical_name".to_string(),
131-
value: "Named".to_string(),
132-
});
126+
tz_metadata.add_custom_attribute(CustomAttribute::kv("logical_name", "Named"));
133127
let tz_schema = get_or_generate_schema::<Tz>(gen, tz_metadata)?;
134128

135129
Ok(generate_one_of_schema(&[local_schema, tz_schema]))

lib/vector-config-common/Cargo.toml

+2
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,7 @@ license = "MPL-2.0"
88
darling = { version = "0.13", default-features = false, features = ["suggestions"] }
99
proc-macro2 = { version = "1.0", default-features = false }
1010
schemars = { version = "0.8.11", default-features = false }
11+
serde = { version = "1.0", default-features = false }
12+
serde_json = { version = "1.0", default-features = false }
1113
syn = { version = "1.0", features = ["full", "extra-traits", "visit-mut", "visit"] }
1214
quote = { version = "1.0", default-features = false }

lib/vector-config-common/src/attributes.rs

+8-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
use std::fmt;
22

3+
use serde::Serialize;
4+
35
/// A custom attribute on a container, variant, or field.
46
///
57
/// Applied by using the `#[configurable(metadata(...))]` helper. Two forms are supported:
@@ -21,7 +23,10 @@ pub enum CustomAttribute {
2123
///
2224
/// Used for most metadata, where a given key could have many different possible values i.e. the status of a
2325
/// component (alpha, beta, stable, deprecated, etc).
24-
KeyValue { key: String, value: String },
26+
KeyValue {
27+
key: String,
28+
value: serde_json::Value,
29+
},
2530
}
2631

2732
impl CustomAttribute {
@@ -35,11 +40,11 @@ impl CustomAttribute {
3540
pub fn kv<K, V>(key: K, value: V) -> Self
3641
where
3742
K: fmt::Display,
38-
V: fmt::Display,
43+
V: Serialize,
3944
{
4045
Self::KeyValue {
4146
key: key.to_string(),
42-
value: value.to_string(),
47+
value: serde_json::to_value(value).expect("should not fail to serialize value to JSON"),
4348
}
4449
}
4550
}

lib/vector-config-macros/src/ast/container.rs

+2-3
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,13 @@ use syn::{
66
DeriveInput, ExprPath, GenericArgument, Generics, Ident, PathArguments, PathSegment, Type,
77
TypeParam,
88
};
9-
use vector_config_common::attributes::CustomAttribute;
109

1110
use super::{
1211
util::{
1312
err_serde_failed, get_serde_default_value, try_extract_doc_title_description,
1413
DarlingResultIterator,
1514
},
16-
Data, Field, Metadata, Style, Tagging, Variant,
15+
Data, Field, LazyCustomAttribute, Metadata, Style, Tagging, Variant,
1716
};
1817

1918
const ERR_NO_ENUM_TUPLES: &str = "enum variants cannot be tuples (multiple unnamed fields)";
@@ -369,7 +368,7 @@ impl<'a> Container<'a> {
369368
/// Attributes can take the shape of flags (`#[configurable(metadata(im_a_teapot))]`) or
370369
/// key/value pairs (`#[configurable(metadata(status = "beta"))]`) to allow rich, semantic
371370
/// metadata to be attached directly to containers.
372-
pub fn metadata(&self) -> impl Iterator<Item = CustomAttribute> {
371+
pub fn metadata(&self) -> impl Iterator<Item = LazyCustomAttribute> {
373372
self.attrs
374373
.metadata
375374
.clone()

lib/vector-config-macros/src/ast/field.rs

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
use darling::{util::Flag, FromAttributes};
22
use serde_derive_internals::ast as serde_ast;
33
use syn::{parse_quote, spanned::Spanned, ExprPath, Ident};
4-
use vector_config_common::{attributes::CustomAttribute, validation::Validation};
4+
use vector_config_common::validation::Validation;
55

66
use super::{
77
util::{
88
err_field_missing_description, find_delegated_serde_deser_ty, get_serde_default_value,
99
try_extract_doc_title_description,
1010
},
11-
Metadata,
11+
LazyCustomAttribute, Metadata,
1212
};
1313

1414
/// A field of a container.
@@ -207,7 +207,7 @@ impl<'a> Field<'a> {
207207
/// Attributes can take the shape of flags (`#[configurable(metadata(im_a_teapot))]`) or
208208
/// key/value pairs (`#[configurable(metadata(status = "beta"))]`) to allow rich, semantic
209209
/// metadata to be attached directly to fields.
210-
pub fn metadata(&self) -> impl Iterator<Item = CustomAttribute> {
210+
pub fn metadata(&self) -> impl Iterator<Item = LazyCustomAttribute> {
211211
self.attrs
212212
.metadata
213213
.clone()

lib/vector-config-macros/src/ast/mod.rs

+76-19
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
use darling::{error::Accumulator, util::path_to_string, FromMeta};
2+
use quote::ToTokens;
23
use serde_derive_internals::{ast as serde_ast, attr as serde_attr};
34

45
mod container;
@@ -8,9 +9,11 @@ mod variant;
89

910
pub use container::Container;
1011
pub use field::Field;
11-
use syn::NestedMeta;
12+
use syn::{Expr, NestedMeta};
1213
pub use variant::Variant;
13-
use vector_config_common::attributes::CustomAttribute;
14+
15+
const INVALID_VALUE_EXPR: &str =
16+
"got function call-style literal value but could not parse as expression";
1417

1518
/// The style of a data container, applying to both enum variants and structs.
1619
///
@@ -81,19 +84,19 @@ impl Tagging {
8184
/// This is typically added to the metadata for an enum's overall schema to better describe how
8285
/// the various subschemas relate to each other and how they're used on the Rust side, for the
8386
/// purpose of generating usable documentation from the schema.
84-
pub fn as_enum_metadata(&self) -> Vec<CustomAttribute> {
87+
pub fn as_enum_metadata(&self) -> Vec<LazyCustomAttribute> {
8588
match self {
86-
Self::External => vec![CustomAttribute::kv("docs::enum_tagging", "external")],
89+
Self::External => vec![LazyCustomAttribute::kv("docs::enum_tagging", "external")],
8790
Self::Internal { tag } => vec![
88-
CustomAttribute::kv("docs::enum_tagging", "internal"),
89-
CustomAttribute::kv("docs::enum_tag_field", tag),
91+
LazyCustomAttribute::kv("docs::enum_tagging", "internal"),
92+
LazyCustomAttribute::kv("docs::enum_tag_field", tag),
9093
],
9194
Self::Adjacent { tag, content } => vec![
92-
CustomAttribute::kv("docs::enum_tagging", "adjacent"),
93-
CustomAttribute::kv("docs::enum_tag_field", tag),
94-
CustomAttribute::kv("docs::enum_content_field", content),
95+
LazyCustomAttribute::kv("docs::enum_tagging", "adjacent"),
96+
LazyCustomAttribute::kv("docs::enum_tag_field", tag),
97+
LazyCustomAttribute::kv("docs::enum_content_field", content),
9598
],
96-
Self::None => vec![CustomAttribute::kv("docs::enum_tagging", "untagged")],
99+
Self::None => vec![LazyCustomAttribute::kv("docs::enum_tagging", "untagged")],
97100
}
98101
}
99102
}
@@ -120,14 +123,47 @@ pub enum Data<'a> {
120123
Struct(Style, Vec<Field<'a>>),
121124
}
122125

126+
/// A lazy version of `CustomAttribute`.
127+
///
128+
/// This is used to capture the value at the macro callsite without having to evaluate it, which
129+
/// lets us generate code where, for example, the value of a metadata key/value pair can be
130+
/// evaulated by an expression given in the attribute.
131+
///
132+
/// This is similar to how `serde` takes an expression for things like `#[serde(default =
133+
/// "exprhere")]`, and so on.
134+
#[derive(Clone, Debug)]
135+
pub enum LazyCustomAttribute {
136+
/// A standalone flag.
137+
Flag(String),
138+
139+
/// A key/value pair.
140+
KeyValue {
141+
key: String,
142+
value: proc_macro2::TokenStream,
143+
},
144+
}
145+
146+
impl LazyCustomAttribute {
147+
pub fn kv<K, V>(key: K, value: V) -> Self
148+
where
149+
K: std::fmt::Display,
150+
V: ToTokens,
151+
{
152+
Self::KeyValue {
153+
key: key.to_string(),
154+
value: value.to_token_stream(),
155+
}
156+
}
157+
}
158+
123159
/// Metadata items defined on containers, variants, or fields.
124160
#[derive(Clone, Debug)]
125161
pub struct Metadata {
126-
items: Vec<CustomAttribute>,
162+
items: Vec<LazyCustomAttribute>,
127163
}
128164

129165
impl Metadata {
130-
pub fn attributes(&self) -> impl Iterator<Item = CustomAttribute> {
166+
pub fn attributes(&self) -> impl Iterator<Item = LazyCustomAttribute> {
131167
self.items.clone().into_iter()
132168
}
133169
}
@@ -148,20 +184,41 @@ impl FromMeta for Metadata {
148184
.iter()
149185
.filter_map(|nmeta| match nmeta {
150186
NestedMeta::Meta(meta) => match meta {
151-
syn::Meta::Path(path) => Some(CustomAttribute::Flag(path_to_string(path))),
187+
syn::Meta::Path(path) => Some(LazyCustomAttribute::Flag(path_to_string(path))),
152188
syn::Meta::List(_) => {
153189
errors.push(darling::Error::unexpected_type("list").with_span(nmeta));
154190
None
155191
}
156192
syn::Meta::NameValue(nv) => match &nv.lit {
157-
syn::Lit::Str(s) => Some(CustomAttribute::KeyValue {
193+
// When dealing with a string literal, we check if it ends in `()`. If so,
194+
// we emit that as-is, leading to doing a function call and using the return
195+
// value of that function as the value for this key/value pair.
196+
//
197+
// Otherwise, we just treat the string literal normally.
198+
syn::Lit::Str(s) => {
199+
if s.value().ends_with("()") {
200+
if let Ok(expr) = s.parse::<Expr>() {
201+
Some(LazyCustomAttribute::KeyValue {
202+
key: path_to_string(&nv.path),
203+
value: expr.to_token_stream(),
204+
})
205+
} else {
206+
errors.push(
207+
darling::Error::custom(INVALID_VALUE_EXPR).with_span(nmeta),
208+
);
209+
None
210+
}
211+
} else {
212+
Some(LazyCustomAttribute::KeyValue {
213+
key: path_to_string(&nv.path),
214+
value: s.value().to_token_stream(),
215+
})
216+
}
217+
}
218+
lit => Some(LazyCustomAttribute::KeyValue {
158219
key: path_to_string(&nv.path),
159-
value: s.value(),
220+
value: lit.to_token_stream(),
160221
}),
161-
lit => {
162-
errors.push(darling::Error::unexpected_lit_type(lit));
163-
None
164-
}
165222
},
166223
},
167224
NestedMeta::Lit(_) => {

lib/vector-config-macros/src/ast/variant.rs

+2-3
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,10 @@ use darling::{error::Accumulator, util::Flag, FromAttributes};
22
use proc_macro2::Ident;
33
use serde_derive_internals::ast as serde_ast;
44
use syn::spanned::Spanned;
5-
use vector_config_common::attributes::CustomAttribute;
65

76
use super::{
87
util::{try_extract_doc_title_description, DarlingResultIterator},
9-
Field, Metadata, Style, Tagging,
8+
Field, LazyCustomAttribute, Metadata, Style, Tagging,
109
};
1110

1211
/// A variant in an enum.
@@ -157,7 +156,7 @@ impl<'a> Variant<'a> {
157156
/// Attributes can take the shape of flags (`#[configurable(metadata(im_a_teapot))]`) or
158157
/// key/value pairs (`#[configurable(metadata(status = "beta"))]`) to allow rich, semantic
159158
/// metadata to be attached directly to variants.
160-
pub fn metadata(&self) -> impl Iterator<Item = CustomAttribute> {
159+
pub fn metadata(&self) -> impl Iterator<Item = LazyCustomAttribute> {
161160
self.attrs
162161
.metadata
163162
.clone()

lib/vector-config-macros/src/configurable.rs

+13-11
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,9 @@ use syn::{
55
parse_macro_input, parse_quote, spanned::Spanned, token::Colon2, DeriveInput, ExprPath, Ident,
66
PathArguments, Type,
77
};
8-
use vector_config_common::{attributes::CustomAttribute, validation::Validation};
8+
use vector_config_common::validation::Validation;
99

10-
use crate::ast::{Container, Data, Field, Style, Tagging, Variant};
10+
use crate::ast::{Container, Data, Field, LazyCustomAttribute, Style, Tagging, Variant};
1111

1212
pub fn derive_configurable_impl(input: TokenStream) -> TokenStream {
1313
// Parse our input token stream as a derive input, and process the container, and the
@@ -518,7 +518,10 @@ fn generate_variant_metadata(
518518
// information.
519519
//
520520
// You can think of this as an enum-specific additional title.
521-
let logical_name_attrs = vec![CustomAttribute::kv("logical_name", variant.ident())];
521+
let logical_name_attrs = vec![LazyCustomAttribute::kv(
522+
"logical_name",
523+
variant.ident().to_string(),
524+
)];
522525
let variant_logical_name =
523526
get_metadata_custom_attributes(meta_ident, logical_name_attrs.into_iter());
524527

@@ -641,18 +644,17 @@ fn get_metadata_validation(
641644

642645
fn get_metadata_custom_attributes(
643646
meta_ident: &Ident,
644-
custom_attributes: impl Iterator<Item = CustomAttribute>,
647+
custom_attributes: impl Iterator<Item = LazyCustomAttribute>,
645648
) -> proc_macro2::TokenStream {
646649
let mapped_custom_attributes = custom_attributes
647650
.map(|attr| match attr {
648-
CustomAttribute::Flag(key) => quote! {
649-
#meta_ident.add_custom_attribute(::vector_config_common::attributes::CustomAttribute::Flag(#key.to_string()));
651+
LazyCustomAttribute::Flag(key) => quote! {
652+
#meta_ident.add_custom_attribute(::vector_config_common::attributes::CustomAttribute::flag(#key));
650653
},
651-
CustomAttribute::KeyValue { key, value } => quote! {
652-
#meta_ident.add_custom_attribute(::vector_config_common::attributes::CustomAttribute::KeyValue {
653-
key: #key.to_string(),
654-
value: #value.to_string(),
655-
});
654+
LazyCustomAttribute::KeyValue { key, value } => quote! {
655+
#meta_ident.add_custom_attribute(::vector_config_common::attributes::CustomAttribute::kv(
656+
#key, #value
657+
));
656658
},
657659
});
658660

lib/vector-config/Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ no-proxy = { version = "0.3.1", default-features = false, features = ["serializ
2020
num-traits = { version = "0.2.15", default-features = false }
2121
schemars = { version = "0.8.11", default-features = true, features = ["preserve_order"] }
2222
serde = { version = "1.0", default-features = false }
23-
serde_json = { version = "1.0.91", default-features = false }
23+
serde_json = { version = "1.0", default-features = false }
2424
serde_with = { version = "2.1.0", default-features = false, features = ["std"] }
2525
snafu = { version = "0.7.4", default-features = false }
2626
toml = { version = "0.5.10", default-features = false }

0 commit comments

Comments
 (0)