enhancement(config): add support for any serde_json-capable value for configurable metadata KV pairs (vectordotdev#15818)

Author: tobz

Cargo.lock

@@ -114,10 +114,7 @@ impl Configurable for TimeZone {
     fn generate_schema(gen: &mut SchemaGenerator) -> Result<SchemaObject, GenerateError> {
         let mut local_schema = generate_const_string_schema("local".to_string());
         let mut local_metadata = Metadata::<()>::with_description("System local timezone.");
-        local_metadata.add_custom_attribute(CustomAttribute::KeyValue {
-            key: "logical_name".to_string(),
-            value: "Local".to_string(),
-        });
+        local_metadata.add_custom_attribute(CustomAttribute::kv("logical_name", "Local"));
         apply_metadata(&mut local_schema, local_metadata);
 
         let mut tz_metadata = Metadata::with_title("A named timezone.");
@@ -126,10 +123,7 @@ impl Configurable for TimeZone {
 
 [tzdb]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones"#,
         );
-        tz_metadata.add_custom_attribute(CustomAttribute::KeyValue {
-            key: "logical_name".to_string(),
-            value: "Named".to_string(),
-        });
+        tz_metadata.add_custom_attribute(CustomAttribute::kv("logical_name", "Named"));
         let tz_schema = get_or_generate_schema::<Tz>(gen, tz_metadata)?;
 
         Ok(generate_one_of_schema(&[local_schema, tz_schema]))

lib/vector-common/src/datetime.rs

@@ -8,5 +8,7 @@ license = "MPL-2.0"
 darling = { version = "0.13", default-features = false, features = ["suggestions"] }
 proc-macro2 = { version = "1.0", default-features = false }
 schemars = { version = "0.8.11", default-features = false }
+serde = { version = "1.0", default-features = false }
+serde_json = { version = "1.0", default-features = false }
 syn = { version = "1.0", features = ["full", "extra-traits", "visit-mut", "visit"] }
 quote = { version = "1.0", default-features = false }

lib/vector-config-common/Cargo.toml

@@ -1,5 +1,7 @@
 use std::fmt;
 
+use serde::Serialize;
+
 /// A custom attribute on a container, variant, or field.
 ///
 /// Applied by using the `#[configurable(metadata(...))]` helper. Two forms are supported:
@@ -21,7 +23,10 @@ pub enum CustomAttribute {
     ///
     /// Used for most metadata, where a given key could have many different possible values i.e. the status of a
     /// component (alpha, beta, stable, deprecated, etc).
-    KeyValue { key: String, value: String },
+    KeyValue {
+        key: String,
+        value: serde_json::Value,
+    },
 }
 
 impl CustomAttribute {
@@ -35,11 +40,11 @@ impl CustomAttribute {
     pub fn kv<K, V>(key: K, value: V) -> Self
     where
         K: fmt::Display,
-        V: fmt::Display,
+        V: Serialize,
     {
         Self::KeyValue {
             key: key.to_string(),
-            value: value.to_string(),
+            value: serde_json::to_value(value).expect("should not fail to serialize value to JSON"),
         }
     }
 }

lib/vector-config-common/src/attributes.rs

@@ -6,14 +6,13 @@ use syn::{
     DeriveInput, ExprPath, GenericArgument, Generics, Ident, PathArguments, PathSegment, Type,
     TypeParam,
 };
-use vector_config_common::attributes::CustomAttribute;
 
 use super::{
     util::{
         err_serde_failed, get_serde_default_value, try_extract_doc_title_description,
         DarlingResultIterator,
     },
-    Data, Field, Metadata, Style, Tagging, Variant,
+    Data, Field, LazyCustomAttribute, Metadata, Style, Tagging, Variant,
 };
 
 const ERR_NO_ENUM_TUPLES: &str = "enum variants cannot be tuples (multiple unnamed fields)";
@@ -369,7 +368,7 @@ impl<'a> Container<'a> {
     /// Attributes can take the shape of flags (`#[configurable(metadata(im_a_teapot))]`) or
     /// key/value pairs (`#[configurable(metadata(status = "beta"))]`) to allow rich, semantic
     /// metadata to be attached directly to containers.
-    pub fn metadata(&self) -> impl Iterator<Item = CustomAttribute> {
+    pub fn metadata(&self) -> impl Iterator<Item = LazyCustomAttribute> {
         self.attrs
             .metadata
             .clone()

lib/vector-config-macros/src/ast/container.rs

@@ -1,14 +1,14 @@
 use darling::{util::Flag, FromAttributes};
 use serde_derive_internals::ast as serde_ast;
 use syn::{parse_quote, spanned::Spanned, ExprPath, Ident};
-use vector_config_common::{attributes::CustomAttribute, validation::Validation};
+use vector_config_common::validation::Validation;
 
 use super::{
     util::{
         err_field_missing_description, find_delegated_serde_deser_ty, get_serde_default_value,
         try_extract_doc_title_description,
     },
-    Metadata,
+    LazyCustomAttribute, Metadata,
 };
 
 /// A field of a container.
@@ -207,7 +207,7 @@ impl<'a> Field<'a> {
     /// Attributes can take the shape of flags (`#[configurable(metadata(im_a_teapot))]`) or
     /// key/value pairs (`#[configurable(metadata(status = "beta"))]`) to allow rich, semantic
     /// metadata to be attached directly to fields.
-    pub fn metadata(&self) -> impl Iterator<Item = CustomAttribute> {
+    pub fn metadata(&self) -> impl Iterator<Item = LazyCustomAttribute> {
         self.attrs
             .metadata
             .clone()

lib/vector-config-macros/src/ast/field.rs

@@ -1,4 +1,5 @@
 use darling::{error::Accumulator, util::path_to_string, FromMeta};
+use quote::ToTokens;
 use serde_derive_internals::{ast as serde_ast, attr as serde_attr};
 
 mod container;
@@ -8,9 +9,11 @@ mod variant;
 
 pub use container::Container;
 pub use field::Field;
-use syn::NestedMeta;
+use syn::{Expr, NestedMeta};
 pub use variant::Variant;
-use vector_config_common::attributes::CustomAttribute;
+
+const INVALID_VALUE_EXPR: &str =
+    "got function call-style literal value but could not parse as expression";
 
 /// The style of a data container, applying to both enum variants and structs.
 ///
@@ -81,19 +84,19 @@ impl Tagging {
     /// This is typically added to the metadata for an enum's overall schema to better describe how
     /// the various subschemas relate to each other and how they're used on the Rust side, for the
     /// purpose of generating usable documentation from the schema.
-    pub fn as_enum_metadata(&self) -> Vec<CustomAttribute> {
+    pub fn as_enum_metadata(&self) -> Vec<LazyCustomAttribute> {
         match self {
-            Self::External => vec![CustomAttribute::kv("docs::enum_tagging", "external")],
+            Self::External => vec![LazyCustomAttribute::kv("docs::enum_tagging", "external")],
             Self::Internal { tag } => vec![
-                CustomAttribute::kv("docs::enum_tagging", "internal"),
-                CustomAttribute::kv("docs::enum_tag_field", tag),
+                LazyCustomAttribute::kv("docs::enum_tagging", "internal"),
+                LazyCustomAttribute::kv("docs::enum_tag_field", tag),
             ],
             Self::Adjacent { tag, content } => vec![
-                CustomAttribute::kv("docs::enum_tagging", "adjacent"),
-                CustomAttribute::kv("docs::enum_tag_field", tag),
-                CustomAttribute::kv("docs::enum_content_field", content),
+                LazyCustomAttribute::kv("docs::enum_tagging", "adjacent"),
+                LazyCustomAttribute::kv("docs::enum_tag_field", tag),
+                LazyCustomAttribute::kv("docs::enum_content_field", content),
             ],
-            Self::None => vec![CustomAttribute::kv("docs::enum_tagging", "untagged")],
+            Self::None => vec![LazyCustomAttribute::kv("docs::enum_tagging", "untagged")],
         }
     }
 }
@@ -120,14 +123,47 @@ pub enum Data<'a> {
     Struct(Style, Vec<Field<'a>>),
 }
 
+/// A lazy version of `CustomAttribute`.
+///
+/// This is used to capture the value at the macro callsite without having to evaluate it, which
+/// lets us generate code where, for example, the value of a metadata key/value pair can be
+/// evaulated by an expression given in the attribute.
+///
+/// This is similar to how `serde` takes an expression for things like `#[serde(default =
+/// "exprhere")]`, and so on.
+#[derive(Clone, Debug)]
+pub enum LazyCustomAttribute {
+    /// A standalone flag.
+    Flag(String),
+
+    /// A key/value pair.
+    KeyValue {
+        key: String,
+        value: proc_macro2::TokenStream,
+    },
+}
+
+impl LazyCustomAttribute {
+    pub fn kv<K, V>(key: K, value: V) -> Self
+    where
+        K: std::fmt::Display,
+        V: ToTokens,
+    {
+        Self::KeyValue {
+            key: key.to_string(),
+            value: value.to_token_stream(),
+        }
+    }
+}
+
 /// Metadata items defined on containers, variants, or fields.
 #[derive(Clone, Debug)]
 pub struct Metadata {
-    items: Vec<CustomAttribute>,
+    items: Vec<LazyCustomAttribute>,
 }
 
 impl Metadata {
-    pub fn attributes(&self) -> impl Iterator<Item = CustomAttribute> {
+    pub fn attributes(&self) -> impl Iterator<Item = LazyCustomAttribute> {
         self.items.clone().into_iter()
     }
 }
@@ -148,20 +184,41 @@ impl FromMeta for Metadata {
             .iter()
             .filter_map(|nmeta| match nmeta {
                 NestedMeta::Meta(meta) => match meta {
-                    syn::Meta::Path(path) => Some(CustomAttribute::Flag(path_to_string(path))),
+                    syn::Meta::Path(path) => Some(LazyCustomAttribute::Flag(path_to_string(path))),
                     syn::Meta::List(_) => {
                         errors.push(darling::Error::unexpected_type("list").with_span(nmeta));
                         None
                     }
                     syn::Meta::NameValue(nv) => match &nv.lit {
-                        syn::Lit::Str(s) => Some(CustomAttribute::KeyValue {
+                        // When dealing with a string literal, we check if it ends in `()`. If so,
+                        // we emit that as-is, leading to doing a function call and using the return
+                        // value of that function as the value for this key/value pair.
+                        //
+                        // Otherwise, we just treat the string literal normally.
+                        syn::Lit::Str(s) => {
+                            if s.value().ends_with("()") {
+                                if let Ok(expr) = s.parse::<Expr>() {
+                                    Some(LazyCustomAttribute::KeyValue {
+                                        key: path_to_string(&nv.path),
+                                        value: expr.to_token_stream(),
+                                    })
+                                } else {
+                                    errors.push(
+                                        darling::Error::custom(INVALID_VALUE_EXPR).with_span(nmeta),
+                                    );
+                                    None
+                                }
+                            } else {
+                                Some(LazyCustomAttribute::KeyValue {
+                                    key: path_to_string(&nv.path),
+                                    value: s.value().to_token_stream(),
+                                })
+                            }
+                        }
+                        lit => Some(LazyCustomAttribute::KeyValue {
                             key: path_to_string(&nv.path),
-                            value: s.value(),
+                            value: lit.to_token_stream(),
                         }),
-                        lit => {
-                            errors.push(darling::Error::unexpected_lit_type(lit));
-                            None
-                        }
                     },
                 },
                 NestedMeta::Lit(_) => {

lib/vector-config-macros/src/ast/mod.rs

@@ -2,11 +2,10 @@ use darling::{error::Accumulator, util::Flag, FromAttributes};
 use proc_macro2::Ident;
 use serde_derive_internals::ast as serde_ast;
 use syn::spanned::Spanned;
-use vector_config_common::attributes::CustomAttribute;
 
 use super::{
     util::{try_extract_doc_title_description, DarlingResultIterator},
-    Field, Metadata, Style, Tagging,
+    Field, LazyCustomAttribute, Metadata, Style, Tagging,
 };
 
 /// A variant in an enum.
@@ -157,7 +156,7 @@ impl<'a> Variant<'a> {
     /// Attributes can take the shape of flags (`#[configurable(metadata(im_a_teapot))]`) or
     /// key/value pairs (`#[configurable(metadata(status = "beta"))]`) to allow rich, semantic
     /// metadata to be attached directly to variants.
-    pub fn metadata(&self) -> impl Iterator<Item = CustomAttribute> {
+    pub fn metadata(&self) -> impl Iterator<Item = LazyCustomAttribute> {
         self.attrs
             .metadata
             .clone()

lib/vector-config-macros/src/ast/variant.rs

@@ -5,9 +5,9 @@ use syn::{
     parse_macro_input, parse_quote, spanned::Spanned, token::Colon2, DeriveInput, ExprPath, Ident,
     PathArguments, Type,
 };
-use vector_config_common::{attributes::CustomAttribute, validation::Validation};
+use vector_config_common::validation::Validation;
 
-use crate::ast::{Container, Data, Field, Style, Tagging, Variant};
+use crate::ast::{Container, Data, Field, LazyCustomAttribute, Style, Tagging, Variant};
 
 pub fn derive_configurable_impl(input: TokenStream) -> TokenStream {
     // Parse our input token stream as a derive input, and process the container, and the
@@ -518,7 +518,10 @@ fn generate_variant_metadata(
     // information.
     //
     // You can think of this as an enum-specific additional title.
-    let logical_name_attrs = vec![CustomAttribute::kv("logical_name", variant.ident())];
+    let logical_name_attrs = vec![LazyCustomAttribute::kv(
+        "logical_name",
+        variant.ident().to_string(),
+    )];
     let variant_logical_name =
         get_metadata_custom_attributes(meta_ident, logical_name_attrs.into_iter());
 
@@ -641,18 +644,17 @@ fn get_metadata_validation(
 
 fn get_metadata_custom_attributes(
     meta_ident: &Ident,
-    custom_attributes: impl Iterator<Item = CustomAttribute>,
+    custom_attributes: impl Iterator<Item = LazyCustomAttribute>,
 ) -> proc_macro2::TokenStream {
     let mapped_custom_attributes = custom_attributes
         .map(|attr| match attr {
-            CustomAttribute::Flag(key) => quote! {
-                #meta_ident.add_custom_attribute(::vector_config_common::attributes::CustomAttribute::Flag(#key.to_string()));
+            LazyCustomAttribute::Flag(key) => quote! {
+                #meta_ident.add_custom_attribute(::vector_config_common::attributes::CustomAttribute::flag(#key));
             },
-            CustomAttribute::KeyValue { key, value } => quote! {
-                #meta_ident.add_custom_attribute(::vector_config_common::attributes::CustomAttribute::KeyValue {
-                    key: #key.to_string(),
-                    value: #value.to_string(),
-                });
+            LazyCustomAttribute::KeyValue { key, value } => quote! {
+                #meta_ident.add_custom_attribute(::vector_config_common::attributes::CustomAttribute::kv(
+                    #key, #value
+                ));
             },
         });
 

lib/vector-config-macros/src/configurable.rs

@@ -20,7 +20,7 @@ no-proxy = { version  = "0.3.1", default-features = false, features = ["serializ
 num-traits = { version = "0.2.15", default-features = false }
 schemars = { version = "0.8.11", default-features = true, features = ["preserve_order"] }
 serde = { version = "1.0", default-features = false }
-serde_json = { version = "1.0.91", default-features = false }
+serde_json = { version = "1.0", default-features = false }
 serde_with = { version = "2.1.0", default-features = false, features = ["std"] }
 snafu = { version = "0.7.4", default-features = false }
 toml = { version = "0.5.10", default-features = false }