vector_core/event/util/log/
all_fields.rs

1use std::{collections::btree_map, fmt::Write as _, iter, slice, sync::LazyLock};
2
3use regex::Regex;
4use serde::{Serialize, Serializer};
5use vrl::path::PathPrefix;
6
7use crate::event::{KeyString, ObjectMap, Value};
8
9static IS_VALID_PATH_SEGMENT: LazyLock<Regex> =
10    LazyLock::new(|| Regex::new(r"^[a-zA-Z0-9_]+$").unwrap());
11
12/// Iterates over all paths in form `a.b[0].c[1]` in alphabetical order
13/// and their corresponding values.
14pub fn all_fields(fields: &ObjectMap) -> FieldsIter<'_> {
15    FieldsIter::new(None, fields, true)
16}
17
18/// Iterates over all paths in form `a.b[0].c[1]` in alphabetical order and their corresponding
19/// values. Field names containing meta-characters are not quoted.
20pub fn all_fields_unquoted(fields: &ObjectMap) -> FieldsIter<'_> {
21    FieldsIter::new(None, fields, false)
22}
23
24/// Same functionality as `all_fields` but it prepends a character that denotes the
25/// path type.
26pub fn all_metadata_fields(fields: &ObjectMap) -> FieldsIter<'_> {
27    FieldsIter::new(Some(PathPrefix::Metadata), fields, true)
28}
29
30/// An iterator with a single "message" element
31pub fn all_fields_non_object_root(value: &Value) -> FieldsIter<'_> {
32    FieldsIter::non_object(value)
33}
34
35/// An iterator similar to `all_fields`, but instead of visiting each array element individually,
36/// it treats the entire array as a single value.
37pub fn all_fields_skip_array_elements(fields: &ObjectMap) -> FieldsIter<'_> {
38    FieldsIter::new_with_skip_array_elements(fields)
39}
40
41#[derive(Clone, Debug)]
42enum LeafIter<'a> {
43    Root((&'a Value, bool)),
44    Map(btree_map::Iter<'a, KeyString, Value>),
45    Array(iter::Enumerate<slice::Iter<'a, Value>>),
46}
47
48#[derive(Clone, Copy, Debug)]
49enum PathComponent<'a> {
50    Key(&'a KeyString),
51    Index(usize),
52}
53
54/// Performs depth-first traversal of the nested structure.
55///
56/// If a key maps to an empty collection, the key and the empty collection will be returned.
57#[derive(Clone)]
58pub struct FieldsIter<'a> {
59    /// If specified, this will be prepended to each path.
60    path_prefix: Option<PathPrefix>,
61    /// Stack of iterators used for the depth-first traversal.
62    stack: Vec<LeafIter<'a>>,
63    /// Path components from the root up to the top of the stack.
64    path: Vec<PathComponent<'a>>,
65    /// Treat array as a single value and don't traverse each element.
66    skip_array_elements: bool,
67    /// Surround invalid fields with quotes to make them parsable.
68    quote_invalid_fields: bool,
69}
70
71impl<'a> FieldsIter<'a> {
72    fn new(
73        path_prefix: Option<PathPrefix>,
74        fields: &'a ObjectMap,
75        quote_invalid_fields: bool,
76    ) -> FieldsIter<'a> {
77        FieldsIter {
78            path_prefix,
79            stack: vec![LeafIter::Map(fields.iter())],
80            path: vec![],
81            skip_array_elements: false,
82            quote_invalid_fields,
83        }
84    }
85
86    /// This is for backwards compatibility. An event where the root is not an object
87    /// will be treated as an object with a single "message" key
88    fn non_object(value: &'a Value) -> FieldsIter<'a> {
89        FieldsIter {
90            path_prefix: None,
91            stack: vec![LeafIter::Root((value, false))],
92            path: vec![],
93            skip_array_elements: false,
94            quote_invalid_fields: true,
95        }
96    }
97
98    fn new_with_skip_array_elements(fields: &'a ObjectMap) -> FieldsIter<'a> {
99        FieldsIter {
100            path_prefix: None,
101            stack: vec![LeafIter::Map(fields.iter())],
102            path: vec![],
103            skip_array_elements: true,
104            quote_invalid_fields: true,
105        }
106    }
107
108    fn push(&mut self, value: &'a Value, component: PathComponent<'a>) -> Option<&'a Value> {
109        match value {
110            Value::Object(map) if !map.is_empty() => {
111                self.stack.push(LeafIter::Map(map.iter()));
112                self.path.push(component);
113                None
114            }
115            Value::Array(array) if !array.is_empty() => {
116                if self.skip_array_elements {
117                    Some(value)
118                } else {
119                    self.stack.push(LeafIter::Array(array.iter().enumerate()));
120                    self.path.push(component);
121                    None
122                }
123            }
124            _ => Some(value),
125        }
126    }
127
128    fn pop(&mut self) {
129        self.stack.pop();
130        self.path.pop();
131    }
132
133    fn make_path(&mut self, component: PathComponent<'a>) -> KeyString {
134        let mut res = match self.path_prefix {
135            None => String::new(),
136            Some(prefix) => match prefix {
137                PathPrefix::Event => String::from("."),
138                PathPrefix::Metadata => String::from("%"),
139            },
140        };
141        let mut path_iter = self.path.iter().chain(iter::once(&component)).peekable();
142        loop {
143            match path_iter.next() {
144                None => break res.into(),
145                Some(PathComponent::Key(key)) => {
146                    if self.quote_invalid_fields && !IS_VALID_PATH_SEGMENT.is_match(key) {
147                        write!(res, "\"{key}\"").expect("write to String never fails");
148                    } else {
149                        res.push_str(key);
150                    }
151                }
152                Some(PathComponent::Index(index)) => {
153                    write!(res, "[{index}]").expect("write to String never fails");
154                }
155            }
156            if let Some(PathComponent::Key(_)) = path_iter.peek() {
157                res.push('.');
158            }
159        }
160    }
161}
162
163impl<'a> Iterator for FieldsIter<'a> {
164    type Item = (KeyString, &'a Value);
165
166    fn next(&mut self) -> Option<Self::Item> {
167        loop {
168            match self.stack.last_mut() {
169                None => return None,
170                Some(LeafIter::Map(map_iter)) => match map_iter.next() {
171                    None => self.pop(),
172                    Some((key, value)) => {
173                        if let Some(scalar_value) = self.push(value, PathComponent::Key(key)) {
174                            return Some((self.make_path(PathComponent::Key(key)), scalar_value));
175                        }
176                    }
177                },
178                Some(LeafIter::Array(array_iter)) => match array_iter.next() {
179                    None => self.pop(),
180                    Some((index, value)) => {
181                        if let Some(scalar_value) = self.push(value, PathComponent::Index(index)) {
182                            return Some((
183                                self.make_path(PathComponent::Index(index)),
184                                scalar_value,
185                            ));
186                        }
187                    }
188                },
189                Some(LeafIter::Root((value, visited))) => {
190                    let result = (!*visited).then(|| ("message".into(), *value));
191                    *visited = true;
192                    break result;
193                }
194            }
195        }
196    }
197}
198
199impl Serialize for FieldsIter<'_> {
200    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
201    where
202        S: Serializer,
203    {
204        serializer.collect_map(self.clone())
205    }
206}
207
208#[cfg(test)]
209mod test {
210    use serde_json::json;
211    use similar_asserts::assert_eq;
212
213    use super::{super::test::fields_from_json, *};
214
215    #[test]
216    fn keys_simple() {
217        let fields = fields_from_json(json!({
218            "field2": 3,
219            "field1": 4,
220            "field3": 5
221        }));
222        let expected: Vec<_> = vec![
223            ("field1", &Value::Integer(4)),
224            ("field2", &Value::Integer(3)),
225            ("field3", &Value::Integer(5)),
226        ]
227        .into_iter()
228        .map(|(k, v)| (k.into(), v))
229        .collect();
230
231        let collected: Vec<_> = all_fields(&fields).collect();
232        assert_eq!(collected, expected);
233    }
234
235    fn special_fields() -> ObjectMap {
236        fields_from_json(json!({
237                    "a-b": 1,
238                    "a*b": 2,
239                    "a b": 3,
240                    ".a .b*": 4,
241                    "\"a\"": 5,
242        }))
243    }
244
245    #[test]
246    fn keys_special_quoted() {
247        let fields = special_fields();
248        let mut collected: Vec<_> = all_fields(&fields).collect();
249        collected.sort_by(|(a, _), (b, _)| a.cmp(b));
250
251        let mut expected: Vec<(KeyString, &Value)> = vec![
252            ("\"a-b\"", &Value::Integer(1)),
253            ("\"a*b\"", &Value::Integer(2)),
254            ("\"a b\"", &Value::Integer(3)),
255            ("\".a .b*\"", &Value::Integer(4)),
256            ("\"\"a\"\"", &Value::Integer(5)),
257        ]
258        .into_iter()
259        .map(|(k, v)| (k.into(), v))
260        .collect();
261        // Compare without the leading `"` char so that the order is the same as the collected fields.
262        expected.sort_by(|(a, _), (b, _)| {
263            a.strip_prefix('"')
264                .unwrap_or(a)
265                .cmp(b.strip_prefix('"').unwrap_or(b))
266        });
267
268        assert_eq!(collected, expected);
269    }
270
271    #[test]
272    fn keys_special_unquoted() {
273        let fields = special_fields();
274        let mut collected: Vec<_> = all_fields_unquoted(&fields).collect();
275        collected.sort_by(|(a, _), (b, _)| a.cmp(b));
276
277        let mut expected: Vec<(KeyString, &Value)> = vec![
278            ("a-b", &Value::Integer(1)),
279            ("a*b", &Value::Integer(2)),
280            ("a b", &Value::Integer(3)),
281            (".a .b*", &Value::Integer(4)),
282            ("\"a\"", &Value::Integer(5)),
283        ]
284        .into_iter()
285        .map(|(k, v)| (k.into(), v))
286        .collect();
287        expected.sort_by(|(a, _), (b, _)| a.cmp(b));
288
289        assert_eq!(collected, expected);
290    }
291
292    #[test]
293    fn metadata_keys_simple() {
294        let fields = fields_from_json(json!({
295            "field_1": 1,
296            "field_0": 0,
297            "field_2": 2
298        }));
299        let expected: Vec<_> = vec![
300            ("%field_0", &Value::Integer(0)),
301            ("%field_1", &Value::Integer(1)),
302            ("%field_2", &Value::Integer(2)),
303        ]
304        .into_iter()
305        .map(|(k, v)| (k.into(), v))
306        .collect();
307
308        let collected: Vec<_> = all_metadata_fields(&fields).collect();
309        assert_eq!(collected, expected);
310    }
311
312    fn nested_fields() -> ObjectMap {
313        fields_from_json(json!({
314                    "a": {
315                        "b": {
316                            "c": 5
317                        },
318                        "a": 4,
319                        "array": [null, 3, {
320                            "x": 1
321                        }, [2]]
322                    },
323                    "a.b.c": 6,
324                    "d": {},
325                    "e": [],
326        }))
327    }
328
329    #[test]
330    fn keys_nested_quoted() {
331        let fields = nested_fields();
332        let expected: Vec<_> = vec![
333            ("a.a", Value::Integer(4)),
334            ("a.array[0]", Value::Null),
335            ("a.array[1]", Value::Integer(3)),
336            ("a.array[2].x", Value::Integer(1)),
337            ("a.array[3][0]", Value::Integer(2)),
338            ("a.b.c", Value::Integer(5)),
339            ("\"a.b.c\"", Value::Integer(6)),
340            ("d", Value::Object(ObjectMap::new())),
341            ("e", Value::Array(Vec::new())),
342        ]
343        .into_iter()
344        .map(|(k, v)| (k.into(), v))
345        .collect();
346
347        let collected: Vec<_> = all_fields(&fields).map(|(k, v)| (k, v.clone())).collect();
348        assert_eq!(collected, expected);
349    }
350
351    #[test]
352    fn keys_nested_unquoted() {
353        let fields = nested_fields();
354        let expected: Vec<_> = vec![
355            ("a.a", Value::Integer(4)),
356            ("a.array[0]", Value::Null),
357            ("a.array[1]", Value::Integer(3)),
358            ("a.array[2].x", Value::Integer(1)),
359            ("a.array[3][0]", Value::Integer(2)),
360            ("a.b.c", Value::Integer(5)),
361            ("a.b.c", Value::Integer(6)),
362            ("d", Value::Object(ObjectMap::new())),
363            ("e", Value::Array(Vec::new())),
364        ]
365        .into_iter()
366        .map(|(k, v)| (k.into(), v))
367        .collect();
368
369        let collected: Vec<_> = all_fields_unquoted(&fields)
370            .map(|(k, v)| (k, v.clone()))
371            .collect();
372        assert_eq!(collected, expected);
373    }
374
375    #[test]
376    fn test_non_object_root() {
377        let value = Value::Integer(3);
378        let collected: Vec<_> = all_fields_non_object_root(&value)
379            .map(|(k, v)| (k.into(), v.clone()))
380            .collect();
381        assert_eq!(collected, vec![("message".to_owned(), value)]);
382    }
383}
vector_core/event/util/log/all_fields.rs

vector_core/event/util/log/
all_fields.rs
