feat(compiler): enforce array element-kind constraints at compile time

changelog.d/1861.breaking.md

@@ -0,0 +1,17 @@
+Several stdlib functions now declare element-kind constraints on array parameters, enabling the compiler to detect element-type mismatches at compile time and automatically infer call-site infallibility.
+
+**Before:** passing a string-literal array required `!` because the compiler assumed it could fail:
+```
+join!(["sources", "transforms", "sinks"], separator: ", ")
+```
+
+**After:** the compiler proves the elements are strings, so `!` is unnecessary (and now an error):
+```
+join(["sources", "transforms", "sinks"], separator: ", ")
+```
+
+Passing the wrong element type (e.g. `join([1, 2, 3])`) is now a hard compile error instead of a runtime failure.
+
+Affected functions: `join`, `contains_all`, `tally`, `encode_key_value`, `encode_logfmt`, `ip_cidr_contains`, `parse_groks`.
+
+authors: pront

docs/generated/encode_key_value.json

@@ -65,7 +65,7 @@
     },
     {
       "title": "Encode with default delimiters (fields ordering)",
-      "source": "encode_key_value!(\n    {\n        \"ts\": \"2021-06-05T17:20:00Z\",\n        \"msg\": \"This is a message\",\n        \"lvl\": \"info\",\n        \"log_id\": 12345\n    },\n    [\"ts\", \"lvl\", \"msg\"]\n)\n",
+      "source": "encode_key_value(\n    {\n        \"ts\": \"2021-06-05T17:20:00Z\",\n        \"msg\": \"This is a message\",\n        \"lvl\": \"info\",\n        \"log_id\": 12345\n    },\n    [\"ts\", \"lvl\", \"msg\"]\n)\n",
       "return": "ts=2021-06-05T17:20:00Z lvl=info msg=\"This is a message\" log_id=12345"
     },
     {
@@ -75,7 +75,7 @@
     },
     {
       "title": "Encode with default delimiters (nested fields ordering)",
-      "source": "encode_key_value!(\n    {\n        \"agent\": {\"name\": \"foo\"},\n        \"log\": {\"file\": {\"path\": \"my.log\"}},\n        \"event\": \"log\"\n    },\n    [\"event\", \"log.file.path\", \"agent.name\"])\n",
+      "source": "encode_key_value(\n    {\n        \"agent\": {\"name\": \"foo\"},\n        \"log\": {\"file\": {\"path\": \"my.log\"}},\n        \"event\": \"log\"\n    },\n    [\"event\", \"log.file.path\", \"agent.name\"])\n",
       "return": "event=log log.file.path=my.log agent.name=foo"
     },
     {
@@ -90,7 +90,7 @@
     }
   ],
   "notices": [
-    "If `fields_ordering` is specified then the function is fallible else it is infallible."
+    "This function is fallible if `fields_ordering` contains non-string elements."
   ],
   "pure": true
 }

docs/generated/encode_logfmt.json

@@ -38,7 +38,7 @@
     },
     {
       "title": "Encode to logfmt (fields ordering)",
-      "source": "encode_logfmt!({\"ts\": \"2021-06-05T17:20:00Z\", \"msg\": \"This is a message\", \"lvl\": \"info\", \"log_id\": 12345}, [\"ts\", \"lvl\", \"msg\"])",
+      "source": "encode_logfmt({\"ts\": \"2021-06-05T17:20:00Z\", \"msg\": \"This is a message\", \"lvl\": \"info\", \"log_id\": 12345}, [\"ts\", \"lvl\", \"msg\"])",
       "return": "ts=2021-06-05T17:20:00Z lvl=info msg=\"This is a message\" log_id=12345"
     },
     {
@@ -48,7 +48,7 @@
     },
     {
       "title": "Encode to logfmt (nested fields ordering)",
-      "source": "encode_logfmt!({\"agent\": {\"name\": \"foo\"}, \"log\": {\"file\": {\"path\": \"my.log\"}}, \"event\": \"log\"}, [\"event\", \"log.file.path\", \"agent.name\"])",
+      "source": "encode_logfmt({\"agent\": {\"name\": \"foo\"}, \"log\": {\"file\": {\"path\": \"my.log\"}}, \"event\": \"log\"}, [\"event\", \"log.file.path\", \"agent.name\"])",
       "return": "event=log log.file.path=my.log agent.name=foo"
     }
   ],

docs/generated/join.json

@@ -29,12 +29,12 @@
   "examples": [
     {
       "title": "Join array (no separator)",
-      "source": "join!([\"bring\", \"us\", \"together\"])",
+      "source": "join([\"bring\", \"us\", \"together\"])",
       "return": "bringustogether"
     },
     {
       "title": "Join array (comma separator)",
-      "source": "join!([\"sources\", \"transforms\", \"sinks\"], separator: \", \")",
+      "source": "join([\"sources\", \"transforms\", \"sinks\"], separator: \", \")",
       "return": "sources, transforms, sinks"
     }
   ],

docs/generated/tally.json

@@ -21,7 +21,7 @@
   "examples": [
     {
       "title": "tally",
-      "source": "tally!([\"foo\", \"bar\", \"foo\", \"baz\"])",
+      "source": "tally([\"foo\", \"bar\", \"foo\", \"baz\"])",
       "return": {
         "foo": 2,
         "bar": 1,

lib/tests/tests/rfcs/8381/convert_object_to_specific_string_format.vrl

@@ -4,4 +4,4 @@
 strings = []
 for_each(.) -> |key, value| { strings = push(strings, key + "=" + encode_json(value)) }
 
-"{" + join!(strings, ",") + "}"
+"{" + join(strings, ",") + "}"

src/compiler/function.rs

@@ -262,6 +262,18 @@ pub struct Parameter {
     /// error.
     pub kind: u16,
 
+    /// For array parameters, the type kind(s) allowed for the array's elements.
+    ///
+    /// When [`kind`] includes [`kind::ARRAY`] and this is not [`kind::ANY`], the
+    /// compiler restricts the array's element kind accordingly:
+    /// - If the argument's element kind is a subset, the call is infallible.
+    /// - If the argument's element kind is unknown or a superset, the call is
+    ///   automatically marked fallible.
+    ///
+    /// Defaults to [`kind::ANY`] (no element-type constraint). Ignored when
+    /// [`kind`] does not include [`kind::ARRAY`].
+    pub element_kind: u16,
+
     /// Whether or not this is a required parameter.
     ///
     /// If it isn't, the function can be called without errors, even if the
@@ -292,6 +304,7 @@ impl Parameter {
         Self {
             keyword,
             kind,
+            element_kind: kind::ANY,
             required: true,
             description,
             default: None,
@@ -305,6 +318,7 @@ impl Parameter {
         Self {
             keyword,
             kind,
+            element_kind: kind::ANY,
             required: false,
             description,
             default: None,
@@ -326,55 +340,67 @@ impl Parameter {
         self
     }
 
+    /// For an array parameter, restrict the kind of its elements.
+    ///
+    /// The compiler uses this to infer call-site fallibility: a subset match is
+    /// infallible; a superset or unknown match is fallible; a disjoint match is
+    /// a compile error. Ignored when [`kind`] does not include [`kind::ARRAY`].
+    #[must_use]
+    pub const fn with_element_kind(mut self, element_kind: u16) -> Self {
+        self.element_kind = element_kind;
+        self
+    }
+
     #[allow(arithmetic_overflow)]
     #[must_use]
     pub fn kind(&self) -> Kind {
-        let mut kind = Kind::never();
-
-        let n = self.kind;
-
-        if (n & kind::BYTES) == kind::BYTES {
-            kind.add_bytes();
-        }
-
-        if (n & kind::INTEGER) == kind::INTEGER {
-            kind.add_integer();
-        }
-
-        if (n & kind::FLOAT) == kind::FLOAT {
-            kind.add_float();
-        }
+        let mut kind = kind_from_bits(self.kind);
 
-        if (n & kind::BOOLEAN) == kind::BOOLEAN {
-            kind.add_boolean();
+        if (self.kind & kind::ARRAY) == kind::ARRAY && self.element_kind != kind::ANY {
+            let element = kind_from_bits(self.element_kind);
+            kind.add_array(Collection::from_unknown(element));
         }
 
-        if (n & kind::OBJECT) == kind::OBJECT {
-            kind.add_object(Collection::any());
-        }
-
-        if (n & kind::ARRAY) == kind::ARRAY {
-            kind.add_array(Collection::any());
-        }
-
-        if (n & kind::TIMESTAMP) == kind::TIMESTAMP {
-            kind.add_timestamp();
-        }
-
-        if (n & kind::REGEX) == kind::REGEX {
-            kind.add_regex();
-        }
-
-        if (n & kind::NULL) == kind::NULL {
-            kind.add_null();
-        }
+        kind
+    }
+}
 
-        if (n & kind::UNDEFINED) == kind::UNDEFINED {
-            kind.add_undefined();
-        }
+#[allow(arithmetic_overflow)]
+fn kind_from_bits(n: u16) -> Kind {
+    let mut kind = Kind::never();
 
-        kind
+    if (n & kind::BYTES) == kind::BYTES {
+        kind.add_bytes();
+    }
+    if (n & kind::INTEGER) == kind::INTEGER {
+        kind.add_integer();
     }
+    if (n & kind::FLOAT) == kind::FLOAT {
+        kind.add_float();
+    }
+    if (n & kind::BOOLEAN) == kind::BOOLEAN {
+        kind.add_boolean();
+    }
+    if (n & kind::OBJECT) == kind::OBJECT {
+        kind.add_object(Collection::any());
+    }
+    if (n & kind::ARRAY) == kind::ARRAY {
+        kind.add_array(Collection::any());
+    }
+    if (n & kind::TIMESTAMP) == kind::TIMESTAMP {
+        kind.add_timestamp();
+    }
+    if (n & kind::REGEX) == kind::REGEX {
+        kind.add_regex();
+    }
+    if (n & kind::NULL) == kind::NULL {
+        kind.add_null();
+    }
+    if (n & kind::UNDEFINED) == kind::UNDEFINED {
+        kind.add_undefined();
+    }
+
+    kind
 }
 
 // -----------------------------------------------------------------------------

src/docs/mod.rs

@@ -130,6 +130,7 @@ pub fn build_function_doc(func: &dyn Function) -> FunctionDoc {
                 description,
                 default,
                 enum_variants,
+                element_kind: _,
             } = param;
 
             let name = keyword.trim().to_string();

src/stdlib/contains_all.rs

@@ -46,7 +46,8 @@ impl Function for ContainsAll {
                 "substrings",
                 kind::ARRAY,
                 "An array of substrings to search for in `value`.",
-            ),
+            )
+            .with_element_kind(kind::BYTES),
             Parameter::optional(
                 "case_sensitive",
                 kind::BOOLEAN,
@@ -114,11 +115,11 @@ impl FunctionExpression for ContainsAllFn {
         contains_all(&value, substrings, case_sensitive)
     }
 
-    fn type_def(&self, state: &TypeState) -> TypeDef {
-        let substring_type_def = self.substrings.type_def(state);
-        let collection = substring_type_def.as_array().expect("must be an array");
-        let bytes_collection = Collection::from_unknown(Kind::bytes());
-        TypeDef::boolean().maybe_fallible(bytes_collection.is_superset(collection).is_err())
+    fn type_def(&self, _state: &TypeState) -> TypeDef {
+        // Element-kind constraint on `substrings` is declared via
+        // `Parameter::with_element_kind`, which drives compiler-level
+        // fallibility inference at the call site.
+        TypeDef::boolean()
     }
 }
 
@@ -138,11 +139,16 @@ mod tests {
             tdef: TypeDef::boolean().infallible(),
         }
 
+        // The function body's `type_def` is now unconditionally infallible;
+        // element-kind fallibility for `substrings` is inferred by the compiler
+        // at the call site via `Parameter::with_element_kind`. See the
+        // `compiler_flags_non_bytes_element_as_fallible` test for end-to-end
+        // verification.
         substring_type {
             args: func_args![value: value!("The Needle In The Haystack"),
                              substrings: value!([1, 2])],
             want: Err("expected string, got integer"),
-            tdef: TypeDef::boolean().fallible(),
+            tdef: TypeDef::boolean(),
         }
 
         yes {
@@ -182,4 +188,36 @@ mod tests {
             tdef: TypeDef::boolean().infallible(),
         }
     ];
+
+    /// End-to-end check: `Parameter::with_element_kind(kind::BYTES)` should drive
+    /// the compiler to mark a `contains_all` call fallible when it cannot prove
+    /// the array elements are strings.
+    #[test]
+    fn compiler_flags_non_bytes_element_as_fallible() {
+        let fns = vec![Box::new(ContainsAll) as Box<dyn crate::compiler::Function>];
+
+        // All string literals: element kind is a subset of `bytes` -> infallible.
+        let src = r#"contains_all("The Needle", ["Needle", "Hay"])"#;
+        let res = crate::compiler::compile(src, &fns).expect("compiles");
+        assert!(
+            !res.program.info().fallible,
+            "call with string literals should be infallible"
+        );
+
+        // Element kind cannot be proven a subset of `bytes` -> fallible.
+        // [1, 2] has element kind integer, which is provably disjoint from
+        // bytes. The compiler now rejects this outright — even `??` cannot
+        // silence a provably-impossible argument.
+        let src = r#"contains_all("The Needle", [1, 2]) ?? false"#;
+        assert!(
+            crate::compiler::compile(src, &fns).is_err(),
+            "[1, 2] is provably not array<bytes>; must be a compile error even with `??`"
+        );
+
+        let src = r#"contains_all("The Needle", [1, 2])"#;
+        assert!(
+            crate::compiler::compile(src, &fns).is_err(),
+            "unhandled provably-wrong call should fail to compile"
+        );
+    }
 }

src/stdlib/encode_key_value.rs

@@ -37,7 +37,8 @@ static DEFAULT_FLATTEN_BOOLEAN: Value = Value::Boolean(false);
 const PARAMETERS: &[Parameter] = &[
     Parameter::required("value", kind::OBJECT, "The value to convert to a string."),
     Parameter::optional("fields_ordering", kind::ARRAY, "The ordering of fields to preserve. Any fields not in this list are listed unordered, after all ordered fields.")
-        .default(&DEFAULT_FIELDS_ORDERING),
+        .default(&DEFAULT_FIELDS_ORDERING)
+        .with_element_kind(kind::BYTES),
     Parameter::optional("key_value_delimiter", kind::BYTES, "The string that separates the key from the value.")
         .default(&DEFAULT_KEY_VALUE_DELIMITER),
     Parameter::optional("field_delimiter", kind::BYTES, "The string that separates each key-value pair.")
@@ -71,7 +72,7 @@ impl Function for EncodeKeyValue {
     }
 
     fn notices(&self) -> &'static [&'static str] {
-        &["If `fields_ordering` is specified then the function is fallible else it is infallible."]
+        &["This function is fallible if `fields_ordering` contains non-string elements."]
     }
 
     fn parameters(&self) -> &'static [Parameter] {
@@ -119,7 +120,7 @@ impl Function for EncodeKeyValue {
             example! {
                 title: "Encode with default delimiters (fields ordering)",
                 source: indoc! {r#"
-                    encode_key_value!(
+                    encode_key_value(
                         {
                             "ts": "2021-06-05T17:20:00Z",
                             "msg": "This is a message",
@@ -147,7 +148,7 @@ impl Function for EncodeKeyValue {
             example! {
                 title: "Encode with default delimiters (nested fields ordering)",
                 source: indoc! {r#"
-                    encode_key_value!(
+                    encode_key_value(
                         {
                             "agent": {"name": "foo"},
                             "log": {"file": {"path": "my.log"}},
@@ -231,7 +232,7 @@ impl FunctionExpression for EncodeKeyValueFn {
     }
 
     fn type_def(&self, _: &state::TypeState) -> TypeDef {
-        TypeDef::bytes().maybe_fallible(self.fields.is_some())
+        TypeDef::bytes()
     }
 }
 
@@ -427,7 +428,7 @@ mod tests {
                 fields_ordering: value!(["lvl", "msg"])
             ],
             want: Ok(r#"lvl=info msg="This is a log message" log_id=12345"#),
-            tdef: TypeDef::bytes().fallible(),
+            tdef: TypeDef::bytes(),
         }
 
         nested_fields_ordering {
@@ -446,7 +447,7 @@ mod tests {
                 fields_ordering:  value!(["event", "log.file.path", "agent.name"])
             ],
             want: Ok("event=log log.file.path=encode_key_value.rs agent.name=vector"),
-            tdef: TypeDef::bytes().fallible(),
+            tdef: TypeDef::bytes(),
         }
 
         fields_ordering_invalid_field_type {
@@ -463,7 +464,7 @@ mod tests {
                         got: Kind::integer(),
                         expected: Kind::bytes()
                     })),
-            tdef: TypeDef::bytes().fallible(),
+            tdef: TypeDef::bytes(),
         }
     ];
 }