diff --git a/benches/stdlib.rs b/benches/stdlib.rs index d8b04e1b44..85d3eac6b0 100644 --- a/benches/stdlib.rs +++ b/benches/stdlib.rs @@ -579,7 +579,7 @@ bench_function! { str_too_long { args: func_args![value: "foo", pattern: "foobar"], - want: Ok(value!(-1)), + want: Ok(value!(null)), } regex_matching_start { diff --git a/changelog.d/1812.fix.md b/changelog.d/1812.fix.md new file mode 100644 index 0000000000..c01d55ab6b --- /dev/null +++ b/changelog.d/1812.fix.md @@ -0,0 +1,3 @@ +Fixed documentation for `find` method that states that this function returns -1 in case `value` doesn't match `pattern`, but in reality returns `null`. + +authors: JakubOnderka diff --git a/src/compiler/test_util.rs b/src/compiler/test_util.rs index b49e403459..18f474166e 100644 --- a/src/compiler/test_util.rs +++ b/src/compiler/test_util.rs @@ -135,6 +135,15 @@ macro_rules! test_function { assert!(got_value == want, "assertion failed for `{}` case:\n got: {:?}\n wanted: {:?}", stringify!($case), got_value, want); let got_tdef = expression.type_def(&state); assert_eq!(got_tdef, $tdef); + if let Ok(ref value) = got_value { + assert!( + $tdef.kind().is_superset(&value.kind()).is_ok(), + "tdef for `{}` case does not contain the kind of the returned value: tdef={:?}, value={:?}", + stringify!($case), + $tdef, + value, + ); + } } err@Err(_) => { // Allow tests against compiler errors. diff --git a/src/stdlib/find.rs b/src/stdlib/find.rs index b9cb6ddd2d..edf631c46f 100644 --- a/src/stdlib/find.rs +++ b/src/stdlib/find.rs @@ -31,7 +31,7 @@ impl Function for Find { } fn usage(&self) -> &'static str { - "Determines from left to right the start position of the first found element in `value` that matches `pattern`. Returns `-1` if not found." + "Determines from left to right the start position of the first found element in `value` that matches `pattern`. Returns null if not found." } fn category(&self) -> &'static str { @@ -39,7 +39,7 @@ impl Function for Find { } fn return_kind(&self) -> u16 { - kind::INTEGER + kind::NULL | kind::INTEGER } fn parameters(&self) -> &'static [Parameter] { @@ -153,7 +153,7 @@ impl FunctionExpression for FindFn { } fn type_def(&self, _: &state::TypeState) -> TypeDef { - TypeDef::integer().infallible() + TypeDef::null().or_integer().infallible() } } @@ -171,43 +171,49 @@ mod tests { str_matching_end { args: func_args![value: "foobar", pattern: "bar"], want: Ok(value!(3)), - tdef: TypeDef::integer().infallible(), + tdef: TypeDef::null().or_integer().infallible(), } str_matching_beginning { args: func_args![value: "foobar", pattern: "foo"], want: Ok(value!(0)), - tdef: TypeDef::integer().infallible(), + tdef: TypeDef::null().or_integer().infallible(), } str_matching_middle { args: func_args![value: "foobar", pattern: "ob"], want: Ok(value!(2)), - tdef: TypeDef::integer().infallible(), + tdef: TypeDef::null().or_integer().infallible(), } str_too_long { args: func_args![value: "foo", pattern: "foobar"], want: Ok(value!(null)), - tdef: TypeDef::integer().infallible(), + tdef: TypeDef::null().or_integer().infallible(), } regex_matching_end { args: func_args![value: "foobar", pattern: Value::Regex(Regex::new("bar").unwrap().into())], want: Ok(value!(3)), - tdef: TypeDef::integer().infallible(), + tdef: TypeDef::null().or_integer().infallible(), } regex_matching_start { args: func_args![value: "foobar", pattern: Value::Regex(Regex::new("fo+z?").unwrap().into())], want: Ok(value!(0)), - tdef: TypeDef::integer().infallible(), + tdef: TypeDef::null().or_integer().infallible(), + } + + regex_no_match { + args: func_args![value: "foo", pattern: Value::Regex(Regex::new("foobar").unwrap().into())], + want: Ok(value!(null)), + tdef: TypeDef::null().or_integer().infallible(), } wrong_pattern { args: func_args![value: "foobar", pattern: Value::Integer(42)], want: Err("expected string or regex, got integer"), - tdef: TypeDef::integer().infallible(), + tdef: TypeDef::null().or_integer().infallible(), } ]; } diff --git a/src/stdlib/merge.rs b/src/stdlib/merge.rs index 32847b8fb3..e56687c59a 100644 --- a/src/stdlib/merge.rs +++ b/src/stdlib/merge.rs @@ -134,6 +134,22 @@ impl FunctionExpression for MergeFn { } fn type_def(&self, state: &state::TypeState) -> TypeDef { + if let (Some(to_val), Some(from_val)) = ( + self.to.resolve_constant(state), + self.from.resolve_constant(state), + ) { + let deep = self + .deep + .as_ref() + .and_then(|d| d.resolve_constant(state)) + .and_then(|v| v.as_boolean()) + .unwrap_or(false); + if let (Value::Object(mut to_map), Value::Object(from_map)) = (to_val, from_val) { + merge_maps(&mut to_map, &from_map, deep); + return TypeDef::from(Kind::from(Value::Object(to_map))); + } + } + // TODO: this has a known bug when deep is true // see: https://github.com/vectordotdev/vector/issues/13597 self.to @@ -246,6 +262,7 @@ mod tests { Field::from("key1") => Kind::bytes(), Field::from("key2") => Kind::bytes(), Field::from("child") => TypeDef::object(btreemap! { + Field::from("grandchild1") => Kind::bytes(), Field::from("grandchild2") => Kind::boolean(), }), }), diff --git a/src/stdlib/parse_apache_log.rs b/src/stdlib/parse_apache_log.rs index cfcb1a8f1c..f676160f08 100644 --- a/src/stdlib/parse_apache_log.rs +++ b/src/stdlib/parse_apache_log.rs @@ -229,44 +229,144 @@ impl FunctionExpression for ParseApacheLogFn { fn kind_common() -> BTreeMap { BTreeMap::from([ - (Field::from("host"), Kind::bytes() | Kind::null()), - (Field::from("identity"), Kind::bytes() | Kind::null()), - (Field::from("user"), Kind::bytes() | Kind::null()), - (Field::from("timestamp"), Kind::timestamp() | Kind::null()), - (Field::from("message"), Kind::bytes() | Kind::null()), - (Field::from("method"), Kind::bytes() | Kind::null()), - (Field::from("path"), Kind::bytes() | Kind::null()), - (Field::from("protocol"), Kind::bytes() | Kind::null()), - (Field::from("status"), Kind::integer() | Kind::null()), - (Field::from("size"), Kind::integer() | Kind::null()), + ( + Field::from("host"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("identity"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("user"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("timestamp"), + Kind::timestamp() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("message"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("method"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("path"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("protocol"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("status"), + Kind::integer() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("size"), + Kind::integer() | Kind::null() | Kind::undefined(), + ), ]) } fn kind_combined() -> BTreeMap { BTreeMap::from([ - (Field::from("host"), Kind::bytes() | Kind::null()), - (Field::from("identity"), Kind::bytes() | Kind::null()), - (Field::from("user"), Kind::bytes() | Kind::null()), - (Field::from("timestamp"), Kind::timestamp() | Kind::null()), - (Field::from("message"), Kind::bytes() | Kind::null()), - (Field::from("method"), Kind::bytes() | Kind::null()), - (Field::from("path"), Kind::bytes() | Kind::null()), - (Field::from("protocol"), Kind::bytes() | Kind::null()), - (Field::from("status"), Kind::integer() | Kind::null()), - (Field::from("size"), Kind::integer() | Kind::null()), - (Field::from("referrer"), Kind::bytes() | Kind::null()), - (Field::from("agent"), Kind::bytes() | Kind::null()), + ( + Field::from("host"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("identity"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("user"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("timestamp"), + Kind::timestamp() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("message"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("method"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("path"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("protocol"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("status"), + Kind::integer() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("size"), + Kind::integer() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("referrer"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("agent"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), ]) } fn kind_error() -> BTreeMap { BTreeMap::from([ - (Field::from("timestamp"), Kind::timestamp() | Kind::null()), - (Field::from("module"), Kind::bytes() | Kind::null()), - (Field::from("severity"), Kind::bytes() | Kind::null()), - (Field::from("thread"), Kind::bytes() | Kind::null()), - (Field::from("port"), Kind::bytes() | Kind::null()), - (Field::from("message"), Kind::bytes() | Kind::null()), + ( + Field::from("timestamp"), + Kind::timestamp() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("module"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("severity"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("thread"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("port"), + Kind::integer() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("message"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("pid"), + Kind::integer() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("client"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("message1"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("message2"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), ]) } diff --git a/src/stdlib/parse_aws_alb_log.rs b/src/stdlib/parse_aws_alb_log.rs index f7b326046c..80cd62cdf7 100644 --- a/src/stdlib/parse_aws_alb_log.rs +++ b/src/stdlib/parse_aws_alb_log.rs @@ -200,45 +200,66 @@ fn inner_kind() -> BTreeMap { Kind::bytes() | Kind::null(), ), (Field::from("classification"), Kind::bytes() | Kind::null()), - (Field::from("client_host"), Kind::bytes()), + (Field::from("client_host"), Kind::bytes() | Kind::null()), (Field::from("domain_name"), Kind::bytes() | Kind::null()), - (Field::from("elb_status_code"), Kind::bytes()), - (Field::from("elb"), Kind::bytes()), + (Field::from("elb_status_code"), Kind::bytes() | Kind::null()), + (Field::from("elb"), Kind::bytes() | Kind::null()), (Field::from("error_reason"), Kind::bytes() | Kind::null()), ( Field::from("matched_rule_priority"), Kind::bytes() | Kind::null(), ), - (Field::from("received_bytes"), Kind::integer()), + ( + Field::from("received_bytes"), + Kind::integer() | Kind::null(), + ), (Field::from("redirect_url"), Kind::bytes() | Kind::null()), - (Field::from("request_creation_time"), Kind::bytes()), - (Field::from("request_method"), Kind::bytes()), - (Field::from("request_processing_time"), Kind::float()), - (Field::from("request_protocol"), Kind::bytes()), - (Field::from("request_url"), Kind::bytes()), - (Field::from("response_processing_time"), Kind::float()), - (Field::from("sent_bytes"), Kind::integer()), + ( + Field::from("request_creation_time"), + Kind::bytes() | Kind::null(), + ), + (Field::from("request_method"), Kind::bytes() | Kind::null()), + ( + Field::from("request_processing_time"), + Kind::float() | Kind::null(), + ), + ( + Field::from("request_protocol"), + Kind::bytes() | Kind::null(), + ), + (Field::from("request_url"), Kind::bytes() | Kind::null()), + ( + Field::from("response_processing_time"), + Kind::float() | Kind::null(), + ), + (Field::from("sent_bytes"), Kind::integer() | Kind::null()), (Field::from("ssl_cipher"), Kind::bytes() | Kind::null()), (Field::from("ssl_protocol"), Kind::bytes() | Kind::null()), - (Field::from("target_group_arn"), Kind::bytes()), + ( + Field::from("target_group_arn"), + Kind::bytes() | Kind::null(), + ), (Field::from("target_host"), Kind::bytes() | Kind::null()), ( Field::from("target_port_list"), - Kind::bytes() | Kind::null(), + Kind::array(Collection::from_unknown(Kind::bytes())), + ), + ( + Field::from("target_processing_time"), + Kind::float() | Kind::null(), ), - (Field::from("target_processing_time"), Kind::float()), ( Field::from("target_status_code_list"), - Kind::bytes() | Kind::null(), + Kind::array(Collection::from_unknown(Kind::bytes())), ), ( Field::from("target_status_code"), Kind::bytes() | Kind::null(), ), - (Field::from("timestamp"), Kind::bytes()), - (Field::from("trace_id"), Kind::bytes()), - (Field::from("type"), Kind::bytes()), - (Field::from("user_agent"), Kind::bytes()), + (Field::from("timestamp"), Kind::bytes() | Kind::null()), + (Field::from("trace_id"), Kind::bytes() | Kind::null()), + (Field::from("type"), Kind::bytes() | Kind::null()), + (Field::from("user_agent"), Kind::bytes() | Kind::null()), (Field::from("traceability_id"), Kind::bytes() | Kind::null()), ]) } diff --git a/src/stdlib/parse_aws_vpc_flow_log.rs b/src/stdlib/parse_aws_vpc_flow_log.rs index a1c174873b..2b2d0f49c9 100644 --- a/src/stdlib/parse_aws_vpc_flow_log.rs +++ b/src/stdlib/parse_aws_vpc_flow_log.rs @@ -171,44 +171,122 @@ impl FunctionExpression for ParseAwsVpcFlowLogFn { fn inner_kind() -> BTreeMap { BTreeMap::from([ - (Field::from("account_id"), Kind::bytes() | Kind::null()), - (Field::from("action"), Kind::bytes() | Kind::null()), - (Field::from("az_id"), Kind::bytes() | Kind::null()), - (Field::from("bytes"), Kind::integer() | Kind::null()), - (Field::from("dstaddr"), Kind::bytes() | Kind::null()), - (Field::from("dstport"), Kind::integer() | Kind::null()), - (Field::from("end"), Kind::integer() | Kind::null()), - (Field::from("flow_direction"), Kind::bytes() | Kind::null()), - (Field::from("instance_id"), Kind::bytes() | Kind::null()), - (Field::from("interface_id"), Kind::bytes() | Kind::null()), - (Field::from("log_status"), Kind::bytes() | Kind::null()), - (Field::from("packets"), Kind::integer() | Kind::null()), - (Field::from("pkt_dstaddr"), Kind::bytes() | Kind::null()), + ( + Field::from("account_id"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("action"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("az_id"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("bytes"), + Kind::integer() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("dstaddr"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("dstport"), + Kind::integer() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("end"), + Kind::integer() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("flow_direction"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("instance_id"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("interface_id"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("log_status"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("packets"), + Kind::integer() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("pkt_dstaddr"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), ( Field::from("pkt_dst_aws_service"), - Kind::bytes() | Kind::null(), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("pkt_srcaddr"), + Kind::bytes() | Kind::null() | Kind::undefined(), ), - (Field::from("pkt_srcaddr"), Kind::bytes() | Kind::null()), ( Field::from("pkt_src_aws_service"), - Kind::bytes() | Kind::null(), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("protocol"), + Kind::integer() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("region"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("srcaddr"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("srcport"), + Kind::integer() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("start"), + Kind::integer() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("sublocation_id"), + Kind::bytes() | Kind::null() | Kind::undefined(), ), - (Field::from("protocol"), Kind::integer() | Kind::null()), - (Field::from("region"), Kind::bytes() | Kind::null()), - (Field::from("srcaddr"), Kind::bytes() | Kind::null()), - (Field::from("srcport"), Kind::integer() | Kind::null()), - (Field::from("start"), Kind::integer() | Kind::null()), - (Field::from("sublocation_id"), Kind::bytes() | Kind::null()), ( Field::from("sublocation_type"), - Kind::bytes() | Kind::null(), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("subnet_id"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("tcp_flags"), + Kind::integer() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("traffic_path"), + Kind::integer() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("type"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("version"), + Kind::integer() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("vpc_id"), + Kind::bytes() | Kind::null() | Kind::undefined(), ), - (Field::from("subnet_id"), Kind::bytes() | Kind::null()), - (Field::from("tcp_flags"), Kind::integer() | Kind::null()), - (Field::from("traffic_path"), Kind::integer() | Kind::null()), - (Field::from("type"), Kind::bytes() | Kind::null()), - (Field::from("version"), Kind::integer() | Kind::null()), - (Field::from("vpc_id"), Kind::bytes() | Kind::null()), ]) } diff --git a/src/stdlib/parse_common_log.rs b/src/stdlib/parse_common_log.rs index e4d885e035..501352ce57 100644 --- a/src/stdlib/parse_common_log.rs +++ b/src/stdlib/parse_common_log.rs @@ -150,16 +150,46 @@ impl FunctionExpression for ParseCommonLogFn { fn inner_kind() -> BTreeMap { BTreeMap::from([ - (Field::from("host"), Kind::bytes() | Kind::null()), - (Field::from("identity"), Kind::bytes() | Kind::null()), - (Field::from("user"), Kind::bytes() | Kind::null()), - (Field::from("timestamp"), Kind::timestamp() | Kind::null()), - (Field::from("message"), Kind::bytes() | Kind::null()), - (Field::from("method"), Kind::bytes() | Kind::null()), - (Field::from("path"), Kind::bytes() | Kind::null()), - (Field::from("protocol"), Kind::bytes() | Kind::null()), - (Field::from("status"), Kind::integer() | Kind::null()), - (Field::from("size"), Kind::integer() | Kind::null()), + ( + Field::from("host"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("identity"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("user"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("timestamp"), + Kind::timestamp() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("message"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("method"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("path"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("protocol"), + Kind::bytes() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("status"), + Kind::integer() | Kind::null() | Kind::undefined(), + ), + ( + Field::from("size"), + Kind::integer() | Kind::null() | Kind::undefined(), + ), ]) } diff --git a/src/stdlib/parse_influxdb.rs b/src/stdlib/parse_influxdb.rs index e6ced348ff..b9310fe6fc 100644 --- a/src/stdlib/parse_influxdb.rs +++ b/src/stdlib/parse_influxdb.rs @@ -297,8 +297,8 @@ fn gauge_kind() -> Kind { fn metric_kind() -> BTreeMap { btreemap! { "name" => Kind::bytes(), - "tags" => tags_kind(), - "timestamp" => Kind::timestamp() | Kind::null(), + "tags" => tags_kind() | Kind::undefined(), + "timestamp" => Kind::timestamp() | Kind::null() | Kind::undefined(), "kind" => Kind::bytes(), "gauge" => gauge_kind(), } diff --git a/src/stdlib/parse_nginx_log.rs b/src/stdlib/parse_nginx_log.rs index f75e884a24..f7750b442f 100644 --- a/src/stdlib/parse_nginx_log.rs +++ b/src/stdlib/parse_nginx_log.rs @@ -315,14 +315,14 @@ impl FunctionExpression for ParseNginxLogFn { fn kind_combined() -> BTreeMap { BTreeMap::from([ ("client".into(), Kind::bytes()), - ("user".into(), Kind::bytes().or_null()), + ("user".into(), Kind::bytes().or_null().or_undefined()), ("timestamp".into(), Kind::timestamp()), ("request".into(), Kind::bytes()), ("status".into(), Kind::integer()), ("size".into(), Kind::integer()), - ("referer".into(), Kind::bytes().or_null()), - ("agent".into(), Kind::bytes().or_null()), - ("compression".into(), Kind::bytes().or_null()), + ("referer".into(), Kind::bytes().or_null().or_undefined()), + ("agent".into(), Kind::bytes().or_null().or_undefined()), + ("compression".into(), Kind::bytes().or_null().or_undefined()), ]) } @@ -344,9 +344,15 @@ fn kind_ingress_upstreaminfo() -> BTreeMap { Kind::bytes().or_undefined(), ), ("upstream_addr".into(), Kind::bytes()), - ("upstream_response_length".into(), Kind::integer()), - ("upstream_response_time".into(), Kind::float()), - ("upstream_status".into(), Kind::integer()), + ( + "upstream_response_length".into(), + Kind::integer().or_undefined(), + ), + ( + "upstream_response_time".into(), + Kind::float().or_undefined(), + ), + ("upstream_status".into(), Kind::integer().or_undefined()), ("req_id".into(), Kind::bytes()), ]) } @@ -371,16 +377,17 @@ fn kind_error() -> BTreeMap { ("severity".into(), Kind::bytes()), ("pid".into(), Kind::integer()), ("tid".into(), Kind::integer()), - ("cid".into(), Kind::integer()), + ("cid".into(), Kind::integer().or_undefined()), ("message".into(), Kind::bytes()), - ("excess".into(), Kind::float().or_null()), - ("zone".into(), Kind::bytes().or_null()), - ("client".into(), Kind::bytes().or_null()), - ("server".into(), Kind::bytes().or_null()), - ("request".into(), Kind::bytes().or_null()), - ("upstream".into(), Kind::bytes().or_null()), - ("host".into(), Kind::bytes().or_null()), - ("port".into(), Kind::bytes().or_null()), + ("excess".into(), Kind::float().or_null().or_undefined()), + ("zone".into(), Kind::bytes().or_null().or_undefined()), + ("client".into(), Kind::bytes().or_null().or_undefined()), + ("server".into(), Kind::bytes().or_null().or_undefined()), + ("request".into(), Kind::bytes().or_null().or_undefined()), + ("upstream".into(), Kind::bytes().or_null().or_undefined()), + ("host".into(), Kind::bytes().or_null().or_undefined()), + ("port".into(), Kind::bytes().or_null().or_undefined()), + ("referer".into(), Kind::bytes().or_null().or_undefined()), ]) } diff --git a/src/stdlib/parse_regex.rs b/src/stdlib/parse_regex.rs index b3e352b511..f5f454769a 100644 --- a/src/stdlib/parse_regex.rs +++ b/src/stdlib/parse_regex.rs @@ -199,9 +199,22 @@ impl FunctionExpression for ParseRegexFn { } } - fn type_def(&self, _: &state::TypeState) -> TypeDef { + fn type_def(&self, state: &state::TypeState) -> TypeDef { + let numeric_groups = self + .numeric_groups + .as_ref() + .and_then(|n| n.resolve_constant(state)) + .and_then(|v| v.as_boolean()) + .unwrap_or(false); + match &self.pattern { - ConstOrExpr::Const(regex) => TypeDef::object(util::regex_kind(regex)).fallible(), + ConstOrExpr::Const(regex) => { + if numeric_groups { + TypeDef::object(util::regex_kind(regex)).fallible() + } else { + TypeDef::object(util::named_group_kind(regex)).fallible() + } + } ConstOrExpr::Expr(_) => { TypeDef::object(Collection::from_unknown(Kind::bytes() | Kind::null())).fallible() } @@ -272,8 +285,6 @@ mod tests { want: Ok(value!({"number": "first"})), tdef: TypeDef::object(btreemap! { Field::from("number") => Kind::bytes(), - Field::from("0") => Kind::bytes() | Kind::null(), - Field::from("1") => Kind::bytes() | Kind::null(), }).fallible(), } @@ -293,15 +304,6 @@ mod tests { Field::from("path") => Kind::bytes(), Field::from("status") => Kind::bytes(), Field::from("bytes_out") => Kind::bytes(), - Field::from("0") => Kind::bytes() | Kind::null(), - Field::from("1") => Kind::bytes() | Kind::null(), - Field::from("2") => Kind::bytes() | Kind::null(), - Field::from("3") => Kind::bytes() | Kind::null(), - Field::from("4") => Kind::bytes() | Kind::null(), - Field::from("5") => Kind::bytes() | Kind::null(), - Field::from("6") => Kind::bytes() | Kind::null(), - Field::from("7") => Kind::bytes() | Kind::null(), - Field::from("8") => Kind::bytes() | Kind::null(), }).fallible(), } ]; diff --git a/src/stdlib/parse_regex_all.rs b/src/stdlib/parse_regex_all.rs index df642f57b7..7c2a6a97f0 100644 --- a/src/stdlib/parse_regex_all.rs +++ b/src/stdlib/parse_regex_all.rs @@ -204,12 +204,23 @@ impl FunctionExpression for ParseRegexAllFn { } } - fn type_def(&self, _: &state::TypeState) -> TypeDef { + fn type_def(&self, state: &state::TypeState) -> TypeDef { + let numeric_groups = self + .numeric_groups + .as_ref() + .and_then(|n| n.resolve_constant(state)) + .and_then(|v| v.as_boolean()) + .unwrap_or(false); + match &self.pattern { - ConstOrExpr::Const(regex) => TypeDef::array(Collection::from_unknown( - Kind::object(util::regex_kind(regex)).or_null(), - )) - .fallible(), + ConstOrExpr::Const(regex) => { + let kind = if numeric_groups { + util::regex_kind(regex) + } else { + util::named_group_kind(regex) + }; + TypeDef::array(Collection::from_unknown(Kind::object(kind).or_null())).fallible() + } ConstOrExpr::Expr(_) => TypeDef::array(Collection::from_unknown( Kind::object(Collection::from_unknown(Kind::bytes() | Kind::null())).or_null(), )) @@ -240,9 +251,6 @@ mod tests { tdef: TypeDef::array(Collection::from_unknown(Kind::null().or_object(btreemap! { Field::from("fruit") => Kind::bytes(), Field::from("veg") => Kind::bytes(), - Field::from("0") => Kind::bytes() | Kind::null(), - Field::from("1") => Kind::bytes() | Kind::null(), - Field::from("2") => Kind::bytes() | Kind::null(), }))).fallible(), } @@ -280,9 +288,6 @@ mod tests { tdef: TypeDef::array(Collection::from_unknown(Kind::null().or_object(btreemap! { Field::from("fruit") => Kind::bytes(), Field::from("veg") => Kind::bytes(), - Field::from("0") => Kind::bytes() | Kind::null(), - Field::from("1") => Kind::bytes() | Kind::null(), - Field::from("2") => Kind::bytes() | Kind::null(), }))).fallible(), } ]; diff --git a/src/stdlib/parse_syslog.rs b/src/stdlib/parse_syslog.rs index dd8916e623..c0088c8195 100644 --- a/src/stdlib/parse_syslog.rs +++ b/src/stdlib/parse_syslog.rs @@ -112,7 +112,7 @@ impl FunctionExpression for ParseSyslogFn { } fn type_def(&self, _: &state::TypeState) -> TypeDef { - TypeDef::object(inner_kind()).fallible() + TypeDef::object(inner_collection()).fallible() } } @@ -191,17 +191,29 @@ fn message_to_value(message: Message<&str>) -> Value { fn inner_kind() -> BTreeMap { BTreeMap::from([ ("message".into(), Kind::bytes()), - ("hostname".into(), Kind::bytes().or_null()), - ("severity".into(), Kind::bytes().or_null()), - ("facility".into(), Kind::bytes().or_null()), - ("appname".into(), Kind::bytes().or_null()), - ("msgid".into(), Kind::bytes().or_null()), - ("timestamp".into(), Kind::timestamp().or_null()), - ("procid".into(), Kind::bytes().or_integer().or_null()), - ("version".into(), Kind::integer().or_null()), + ("hostname".into(), Kind::bytes().or_null().or_undefined()), + ("severity".into(), Kind::bytes().or_null().or_undefined()), + ("facility".into(), Kind::bytes().or_null().or_undefined()), + ("appname".into(), Kind::bytes().or_null().or_undefined()), + ("msgid".into(), Kind::bytes().or_null().or_undefined()), + ( + "timestamp".into(), + Kind::timestamp().or_null().or_undefined(), + ), + ( + "procid".into(), + Kind::bytes().or_integer().or_null().or_undefined(), + ), + ("version".into(), Kind::integer().or_null().or_undefined()), ]) } +fn inner_collection() -> Collection { + let mut collection = Collection::from(inner_kind()); + collection.set_unknown(Kind::object(Collection::from_unknown(Kind::bytes()))); + collection +} + #[cfg(test)] mod tests { use crate::btreemap; @@ -230,13 +242,13 @@ mod tests { "message" => "Try to override the THX port, maybe it will reboot the neural interface!", "version" => 1, }), - tdef: TypeDef::object(inner_kind()).fallible(), + tdef: TypeDef::object(inner_collection()).fallible(), } invalid { args: func_args![value: "not much of a syslog message"], want: Err("unable to parse input as valid syslog message".to_string()), - tdef: TypeDef::object(inner_kind()).fallible(), + tdef: TypeDef::object(inner_collection()).fallible(), } haproxy { @@ -249,7 +261,7 @@ mod tests { "appname" => "haproxy", "procid" => 73411, }), - tdef: TypeDef::object(inner_kind()).fallible(), + tdef: TypeDef::object(inner_collection()).fallible(), } missing_pri { @@ -260,7 +272,7 @@ mod tests { "appname" => "haproxy", "procid" => 73411, }), - tdef: TypeDef::object(inner_kind()).fallible(), + tdef: TypeDef::object(inner_collection()).fallible(), } empty_sd_element { @@ -277,7 +289,7 @@ mod tests { "version" => 1, "empty" => btreemap! {}, }), - tdef: TypeDef::object(inner_kind()).fallible(), + tdef: TypeDef::object(inner_collection()).fallible(), } non_empty_sd_element { @@ -297,7 +309,7 @@ mod tests { }, "empty" => btreemap! {}, }), - tdef: TypeDef::object(inner_kind()).fallible(), + tdef: TypeDef::object(inner_collection()).fallible(), } empty_sd_value { @@ -317,7 +329,7 @@ mod tests { "x" => "", }, }), - tdef: TypeDef::object(inner_kind()).fallible(), + tdef: TypeDef::object(inner_collection()).fallible(), } non_structured_data_in_message { @@ -330,7 +342,7 @@ mod tests { "timestamp" => Utc.with_ymd_and_hms(Utc::now().year(), 6, 8, 11, 54, 8).unwrap(), "message" => "[Tue Jun 08 11:54:08.929301 2021] [php7:emerg] [pid 1374899] [client 95.223.77.60:41888] rest of message", }), - tdef: TypeDef::object(inner_kind()).fallible(), + tdef: TypeDef::object(inner_collection()).fallible(), } escapes_in_structured_data_quote { @@ -348,7 +360,7 @@ mod tests { "timestamp" => Utc.with_ymd_and_hms(2003, 10, 11, 22, 14, 15).unwrap().with_nanosecond(3_000_000).unwrap(), "version" => 1 }), - tdef: TypeDef::object(inner_kind()).fallible(), + tdef: TypeDef::object(inner_collection()).fallible(), } escapes_in_structured_data_slash { @@ -366,7 +378,7 @@ mod tests { "timestamp" => Utc.with_ymd_and_hms(2003, 10, 11, 22, 14, 15).unwrap().with_nanosecond(3_000_000).unwrap(), "version" => 1 }), - tdef: TypeDef::object(inner_kind()).fallible(), + tdef: TypeDef::object(inner_collection()).fallible(), } escapes_in_structured_data_bracket { @@ -384,7 +396,7 @@ mod tests { "timestamp" => Utc.with_ymd_and_hms(2003, 10, 11, 22, 14, 15).unwrap().with_nanosecond(3_000_000).unwrap(), "version" => 1 }), - tdef: TypeDef::object(inner_kind()).fallible(), + tdef: TypeDef::object(inner_collection()).fallible(), } ]; } diff --git a/src/stdlib/parse_tokens.rs b/src/stdlib/parse_tokens.rs index 7893d35f83..5a7c30b4b0 100644 --- a/src/stdlib/parse_tokens.rs +++ b/src/stdlib/parse_tokens.rs @@ -94,7 +94,7 @@ impl FunctionExpression for ParseTokensFn { } fn type_def(&self, _: &state::TypeState) -> TypeDef { - TypeDef::array(Collection::from_unknown(Kind::bytes())) + TypeDef::array(Collection::from_unknown(Kind::bytes() | Kind::null())) } } @@ -117,7 +117,7 @@ mod tests { "11881".into(), ]), - tdef: TypeDef::array(Collection::from_unknown(Kind::bytes())), + tdef: TypeDef::array(Collection::from_unknown(Kind::bytes() | Kind::null())), } ]; } diff --git a/src/stdlib/parse_xml.rs b/src/stdlib/parse_xml.rs index 47f70bdfea..542a52832a 100644 --- a/src/stdlib/parse_xml.rs +++ b/src/stdlib/parse_xml.rs @@ -218,7 +218,7 @@ fn type_def() -> TypeDef { } fn inner_kind() -> Kind { - Kind::object(Collection::any()) + Kind::any() } #[cfg(test)] diff --git a/src/stdlib/pop.rs b/src/stdlib/pop.rs index a8632c8050..960065b2c4 100644 --- a/src/stdlib/pop.rs +++ b/src/stdlib/pop.rs @@ -72,6 +72,10 @@ impl FunctionExpression for PopFn { } fn type_def(&self, state: &state::TypeState) -> TypeDef { + if let Some(Value::Array(mut arr)) = self.value.resolve_constant(state) { + arr.pop(); + return TypeDef::from(Kind::from(Value::Array(arr))); + } self.value .type_def(state) .fallible_unless(Kind::array(Collection::any())) @@ -108,7 +112,6 @@ mod tests { Index::from(2) => Kind::integer(), Index::from(3) => Kind::boolean(), Index::from(4) => Kind::float(), - Index::from(5) => Kind::bytes(), }), } @@ -119,7 +122,6 @@ mod tests { Index::from(0) => Kind::integer(), Index::from(1) => Kind::integer(), Index::from(2) => Kind::integer(), - Index::from(3) => Kind::integer(), }), } diff --git a/src/stdlib/round.rs b/src/stdlib/round.rs index 0d205c0ded..33bbea3f6b 100644 --- a/src/stdlib/round.rs +++ b/src/stdlib/round.rs @@ -113,8 +113,15 @@ impl FunctionExpression for RoundFn { round(precision, value) } - fn type_def(&self, _: &state::TypeState) -> TypeDef { - TypeDef::integer().infallible() + fn type_def(&self, state: &state::TypeState) -> TypeDef { + let kind = self.value.type_def(state).kind().clone(); + let contains_float = kind.contains_float(); + let contains_integer = kind.contains_integer(); + match (contains_float, contains_integer) { + (true, false) => TypeDef::float().infallible(), + (false, true) => TypeDef::integer().infallible(), + _ => TypeDef::float().or_integer().infallible(), + } } } @@ -128,13 +135,13 @@ mod tests { down { args: func_args![value: 1234.2], want: Ok(1234.0), - tdef: TypeDef::integer().infallible(), + tdef: TypeDef::float().infallible(), } up { args: func_args![value: 1234.8], want: Ok(1235.0), - tdef: TypeDef::integer().infallible(), + tdef: TypeDef::float().infallible(), } integer { @@ -148,7 +155,7 @@ mod tests { precision: 1 ], want: Ok(1234.4), - tdef: TypeDef::integer().infallible(), + tdef: TypeDef::float().infallible(), } bigger_precision { @@ -156,7 +163,7 @@ mod tests { precision: 4 ], want: Ok(1234.5679), - tdef: TypeDef::integer().infallible(), + tdef: TypeDef::float().infallible(), } huge { @@ -164,7 +171,7 @@ mod tests { precision: 5 ], want: Ok(9_876_543_210_123_456_789_098_765_432_101_234_567_890_987_654_321.987_65), - tdef: TypeDef::integer().infallible(), + tdef: TypeDef::float().infallible(), } ]; } diff --git a/src/stdlib/slice.rs b/src/stdlib/slice.rs index 8bf9d88df5..312dee2464 100644 --- a/src/stdlib/slice.rs +++ b/src/stdlib/slice.rs @@ -152,6 +152,31 @@ impl FunctionExpression for SliceFn { } fn type_def(&self, state: &state::TypeState) -> TypeDef { + // If all three parameters are compile-time constants, we can compute the exact result type. + if let (Some(value), Some(start_val)) = ( + self.value.resolve_constant(state), + self.start.resolve_constant(state), + ) { + let start = match start_val { + Value::Integer(i) => i, + _ => { + return TypeDef::from(Kind::never()) + .fallible() + .or_bytes() + .or_array(Collection::any()); + } + }; + let end = self + .end + .as_ref() + .and_then(|e| e.resolve_constant(state)) + .and_then(|v| v.as_integer()); + + if let Ok(result) = slice(start, end, value) { + return TypeDef::from(Kind::from(&result)).fallible(); + } + } + let td = TypeDef::from(Kind::never()).fallible(); match self.value.type_def(state) { @@ -276,7 +301,6 @@ mod tests { tdef: TypeDef::array(Collection::from_parts(BTreeMap::from([ (Index::from(0), Kind::integer()), (Index::from(1), Kind::integer()), - (Index::from(2), Kind::integer()), ]), Kind::undefined())).fallible(), } @@ -288,7 +312,6 @@ mod tests { tdef: TypeDef::array(Collection::from_parts(BTreeMap::from([ (Index::from(0), Kind::integer()), (Index::from(1), Kind::integer()), - (Index::from(2), Kind::integer()), ]), Kind::undefined())).fallible(), } @@ -297,10 +320,9 @@ mod tests { start: 1 ], want: Ok(value!(["ook", true])), - tdef: TypeDef::array(Collection::from_parts(BTreeMap::from([ - (Index::from(0), Kind::integer()), - (Index::from(1), Kind::bytes()), - (Index::from(2), Kind::boolean()), + tdef: TypeDef::array(Collection::from_parts(BTreeMap::from([ + (Index::from(0), Kind::bytes()), + (Index::from(1), Kind::boolean()), ]), Kind::undefined())).fallible(), } diff --git a/src/stdlib/util.rs b/src/stdlib/util.rs index 74fbc522ee..bfabbffbe1 100644 --- a/src/stdlib/util.rs +++ b/src/stdlib/util.rs @@ -93,6 +93,17 @@ pub(crate) fn regex_kind( inner_type } +pub(crate) fn named_group_kind( + regex: ®ex::Regex, +) -> std::collections::BTreeMap { + use crate::value::kind::Kind; + let mut inner_type = std::collections::BTreeMap::new(); + for name in regex.capture_names().flatten() { + inner_type.insert(name.to_owned().into(), Kind::bytes()); + } + inner_type +} + pub(crate) fn is_nullish(value: &Value) -> bool { match value { Value::Bytes(v) => { diff --git a/src/stdlib/xxhash.rs b/src/stdlib/xxhash.rs index aebb8762aa..870e86f3be 100644 --- a/src/stdlib/xxhash.rs +++ b/src/stdlib/xxhash.rs @@ -151,9 +151,9 @@ impl FunctionExpression for XxhashFn { .is_some_and(|variant| VALID_VARIANTS.contains(&variant.to_uppercase().as_str())); if valid_static_variant { - TypeDef::bytes().infallible() + TypeDef::bytes().or_integer().infallible() } else { - TypeDef::bytes().fallible() + TypeDef::bytes().or_integer().fallible() } } } @@ -169,61 +169,61 @@ mod tests { hash_xxh32_default { args: func_args![value: "foo"], want: Ok(value!(3_792_637_401_i64)), - tdef: TypeDef::bytes().infallible(), + tdef: TypeDef::bytes().or_integer().infallible(), } hash_xxh32 { args: func_args![value: "foo", variant: "XXH32"], want: Ok(value!(3_792_637_401_i64)), - tdef: TypeDef::bytes().infallible(), + tdef: TypeDef::bytes().or_integer().infallible(), } hash_xxh64 { args: func_args![value: "foo", variant: "XXH64"], want: Ok(value!(3_728_699_739_546_630_719_i64)), - tdef: TypeDef::bytes().infallible(), + tdef: TypeDef::bytes().or_integer().infallible(), } hash_xxh3_64 { args: func_args![value: "foo", variant: "XXH3-64"], want: Ok(value!(-6_093_828_362_558_603_894_i64)), - tdef: TypeDef::bytes().infallible(), + tdef: TypeDef::bytes().or_integer().infallible(), } hash_xxh3_128 { args: func_args![value: "foo", variant: "XXH3-128"], want: Ok(value!("161745101148472925293886522910304009610")), - tdef: TypeDef::bytes().infallible(), + tdef: TypeDef::bytes().or_integer().infallible(), } long_string_xxh32 { args: func_args![value: "vrl xxhash hash function"], want: Ok(value!(919_261_294_i64)), - tdef: TypeDef::bytes().infallible(), + tdef: TypeDef::bytes().or_integer().infallible(), } long_string_xxh64 { args: func_args![value: "vrl xxhash hash function", variant: "XXH64"], want: Ok(value!(7_826_295_616_420_964_813_i64)), - tdef: TypeDef::bytes().infallible(), + tdef: TypeDef::bytes().or_integer().infallible(), } long_string_xxh3_64 { args: func_args![value: "vrl xxhash hash function", variant: "XXH3-64"], want: Ok(value!(-7_714_906_473_624_552_998_i64)), - tdef: TypeDef::bytes().infallible(), + tdef: TypeDef::bytes().or_integer().infallible(), } long_string_xxh3_128 { args: func_args![value: "vrl xxhash hash function", variant: "XXH3-128"], want: Ok(value!("89621485359950851650871997518391357172")), - tdef: TypeDef::bytes().infallible(), + tdef: TypeDef::bytes().or_integer().infallible(), } hash_invalid_variant { args: func_args![value: "foo", variant: "XXH16"], want: Err("Variant must be either 'XXH32', 'XXH64', 'XXH3-64', or 'XXH3-128'"), - tdef: TypeDef::bytes().fallible(), + tdef: TypeDef::bytes().or_integer().fallible(), } ]; }