Skip to content

fix(stdlib): validate tdef against runtime value and fix incorrect type_defs #1823

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
thomasqueirozb wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

fix(stdlib): validate tdef against runtime value and fix incorrect type_defs #1823

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2438fd4
perf(stdlib): pre-compute capture info and use index-based lookup in …
JakubOnderka Jun 12, 2026
5c333ee
fix(stdlib): validate tdef against runtime value and fix incorrect ty…
thomasqueirozb Jun 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion benches/stdlib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -579,7 +579,7 @@ bench_function! {

str_too_long {
args: func_args![value: "foo", pattern: "foobar"],
want: Ok(value!(-1)),
want: Ok(value!(null)),
}

regex_matching_start {
Expand Down
3 changes: 3 additions & 0 deletions changelog.d/1812.fix.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Fixed documentation for `find` method that states that this function returns -1 in case `value` doesn't match `pattern`, but in reality returns `null`.

authors: JakubOnderka
9 changes: 9 additions & 0 deletions src/compiler/test_util.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,15 @@ macro_rules! test_function {
assert!(got_value == want, "assertion failed for `{}` case:\n got: {:?}\n wanted: {:?}", stringify!($case), got_value, want);
let got_tdef = expression.type_def(&state);
assert_eq!(got_tdef, $tdef);
if let Ok(ref value) = got_value {
assert!(
$tdef.kind().is_superset(&value.kind()).is_ok(),
"tdef for `{}` case does not contain the kind of the returned value: tdef={:?}, value={:?}",
stringify!($case),
$tdef,
value,
);
}
}
err@Err(_) => {
// Allow tests against compiler errors.
Expand Down
26 changes: 16 additions & 10 deletions src/stdlib/find.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,15 @@ impl Function for Find {
}

fn usage(&self) -> &'static str {
"Determines from left to right the start position of the first found element in `value` that matches `pattern`. Returns `-1` if not found."
"Determines from left to right the start position of the first found element in `value` that matches `pattern`. Returns null if not found."
}

fn category(&self) -> &'static str {
Category::String.as_ref()
}

fn return_kind(&self) -> u16 {
kind::INTEGER
kind::NULL | kind::INTEGER
}

fn parameters(&self) -> &'static [Parameter] {
Expand Down Expand Up @@ -153,7 +153,7 @@ impl FunctionExpression for FindFn {
}

fn type_def(&self, _: &state::TypeState) -> TypeDef {
TypeDef::integer().infallible()
TypeDef::null().or_integer().infallible()
}
}

Expand All @@ -171,43 +171,49 @@ mod tests {
str_matching_end {
args: func_args![value: "foobar", pattern: "bar"],
want: Ok(value!(3)),
tdef: TypeDef::integer().infallible(),
tdef: TypeDef::null().or_integer().infallible(),
}

str_matching_beginning {
args: func_args![value: "foobar", pattern: "foo"],
want: Ok(value!(0)),
tdef: TypeDef::integer().infallible(),
tdef: TypeDef::null().or_integer().infallible(),
}

str_matching_middle {
args: func_args![value: "foobar", pattern: "ob"],
want: Ok(value!(2)),
tdef: TypeDef::integer().infallible(),
tdef: TypeDef::null().or_integer().infallible(),
}

str_too_long {
args: func_args![value: "foo", pattern: "foobar"],
want: Ok(value!(null)),
tdef: TypeDef::integer().infallible(),
tdef: TypeDef::null().or_integer().infallible(),
}

regex_matching_end {
args: func_args![value: "foobar", pattern: Value::Regex(Regex::new("bar").unwrap().into())],
want: Ok(value!(3)),
tdef: TypeDef::integer().infallible(),
tdef: TypeDef::null().or_integer().infallible(),
}

regex_matching_start {
args: func_args![value: "foobar", pattern: Value::Regex(Regex::new("fo+z?").unwrap().into())],
want: Ok(value!(0)),
tdef: TypeDef::integer().infallible(),
tdef: TypeDef::null().or_integer().infallible(),
}

regex_no_match {
args: func_args![value: "foo", pattern: Value::Regex(Regex::new("foobar").unwrap().into())],
want: Ok(value!(null)),
tdef: TypeDef::null().or_integer().infallible(),
}

wrong_pattern {
args: func_args![value: "foobar", pattern: Value::Integer(42)],
want: Err("expected string or regex, got integer"),
tdef: TypeDef::integer().infallible(),
tdef: TypeDef::null().or_integer().infallible(),
}
];
}
17 changes: 17 additions & 0 deletions src/stdlib/merge.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,22 @@ impl FunctionExpression for MergeFn {
}

fn type_def(&self, state: &state::TypeState) -> TypeDef {
if let (Some(to_val), Some(from_val)) = (
self.to.resolve_constant(state),
self.from.resolve_constant(state),
) {
let deep = self
.deep
.as_ref()
.and_then(|d| d.resolve_constant(state))
.and_then(|v| v.as_boolean())
.unwrap_or(false);
if let (Value::Object(mut to_map), Value::Object(from_map)) = (to_val, from_val) {
merge_maps(&mut to_map, &from_map, deep);
return TypeDef::from(Kind::from(Value::Object(to_map)));
}
}

// TODO: this has a known bug when deep is true
// see: https://github.com/vectordotdev/vector/issues/13597
self.to
Expand Down Expand Up @@ -246,6 +262,7 @@ mod tests {
Field::from("key1") => Kind::bytes(),
Field::from("key2") => Kind::bytes(),
Field::from("child") => TypeDef::object(btreemap! {
Field::from("grandchild1") => Kind::bytes(),
Field::from("grandchild2") => Kind::boolean(),
}),
}),
Expand Down
156 changes: 128 additions & 28 deletions src/stdlib/parse_apache_log.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,44 +229,144 @@ impl FunctionExpression for ParseApacheLogFn {

fn kind_common() -> BTreeMap<Field, Kind> {
BTreeMap::from([
(Field::from("host"), Kind::bytes() | Kind::null()),
(Field::from("identity"), Kind::bytes() | Kind::null()),
(Field::from("user"), Kind::bytes() | Kind::null()),
(Field::from("timestamp"), Kind::timestamp() | Kind::null()),
(Field::from("message"), Kind::bytes() | Kind::null()),
(Field::from("method"), Kind::bytes() | Kind::null()),
(Field::from("path"), Kind::bytes() | Kind::null()),
(Field::from("protocol"), Kind::bytes() | Kind::null()),
(Field::from("status"), Kind::integer() | Kind::null()),
(Field::from("size"), Kind::integer() | Kind::null()),
(
Field::from("host"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
(
Field::from("identity"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
(
Field::from("user"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
(
Field::from("timestamp"),
Kind::timestamp() | Kind::null() | Kind::undefined(),
),
(
Field::from("message"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
(
Field::from("method"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
(
Field::from("path"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
(
Field::from("protocol"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
(
Field::from("status"),
Kind::integer() | Kind::null() | Kind::undefined(),
),
(
Field::from("size"),
Kind::integer() | Kind::null() | Kind::undefined(),
),
])
}

fn kind_combined() -> BTreeMap<Field, Kind> {
BTreeMap::from([
(Field::from("host"), Kind::bytes() | Kind::null()),
(Field::from("identity"), Kind::bytes() | Kind::null()),
(Field::from("user"), Kind::bytes() | Kind::null()),
(Field::from("timestamp"), Kind::timestamp() | Kind::null()),
(Field::from("message"), Kind::bytes() | Kind::null()),
(Field::from("method"), Kind::bytes() | Kind::null()),
(Field::from("path"), Kind::bytes() | Kind::null()),
(Field::from("protocol"), Kind::bytes() | Kind::null()),
(Field::from("status"), Kind::integer() | Kind::null()),
(Field::from("size"), Kind::integer() | Kind::null()),
(Field::from("referrer"), Kind::bytes() | Kind::null()),
(Field::from("agent"), Kind::bytes() | Kind::null()),
(
Field::from("host"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
(
Field::from("identity"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
(
Field::from("user"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
(
Field::from("timestamp"),
Kind::timestamp() | Kind::null() | Kind::undefined(),
),
(
Field::from("message"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
(
Field::from("method"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
(
Field::from("path"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
(
Field::from("protocol"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
(
Field::from("status"),
Kind::integer() | Kind::null() | Kind::undefined(),
),
(
Field::from("size"),
Kind::integer() | Kind::null() | Kind::undefined(),
),
(
Field::from("referrer"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
(
Field::from("agent"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
])
}

fn kind_error() -> BTreeMap<Field, Kind> {
BTreeMap::from([
(Field::from("timestamp"), Kind::timestamp() | Kind::null()),
(Field::from("module"), Kind::bytes() | Kind::null()),
(Field::from("severity"), Kind::bytes() | Kind::null()),
(Field::from("thread"), Kind::bytes() | Kind::null()),
(Field::from("port"), Kind::bytes() | Kind::null()),
(Field::from("message"), Kind::bytes() | Kind::null()),
(
Field::from("timestamp"),
Kind::timestamp() | Kind::null() | Kind::undefined(),
),
(
Field::from("module"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
(
Field::from("severity"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
(
Field::from("thread"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
(
Field::from("port"),
Kind::integer() | Kind::null() | Kind::undefined(),
),
(
Field::from("message"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
(
Field::from("pid"),
Kind::integer() | Kind::null() | Kind::undefined(),
),
(
Field::from("client"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
(
Field::from("message1"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
(
Field::from("message2"),
Kind::bytes() | Kind::null() | Kind::undefined(),
),
])
}

Expand Down
Loading