feat(codecs, databricks_zerobus sink): Unity Catalog VARIANT support

Cargo.toml

@@ -352,7 +352,7 @@ prost-reflect = { workspace = true, optional = true }
 prost-types = { workspace = true, optional = true }
 
 # Databricks Zerobus
-databricks-zerobus-ingest-sdk = { version = "2.3.1", optional = true, features = ["arrow-flight"] }
+databricks-zerobus-ingest-sdk = { version = "2.4.0", optional = true, features = ["arrow-flight"] }
 
 # GCP
 goauth = { version = "0.16.0", optional = true }
@@ -373,9 +373,9 @@ greptimedb-ingester = { version = "0.17.0", default-features = false, optional =
 # External libs
 arc-swap = { workspace = true, default-features = false, optional = true }
 async-compression = { workspace = true, features = ["zstd"], optional = true }
-arrow = { version = "58.2.0", default-features = false, features = ["ipc"], optional = true }
-arrow-schema = { version = "58.2.0", default-features = false, optional = true }
-parquet = { version = "58.2.0", default-features = false, features = [
+arrow = { version = "59.1.0", default-features = false, features = ["ipc"], optional = true }
+arrow-schema = { version = "59.1.0", default-features = false, optional = true }
+parquet = { version = "59.1.0", default-features = false, features = [
   "arrow",
   "snap",
   "zstd",

LICENSE-3rdparty.csv

@@ -390,7 +390,6 @@ inotify,https://github.com/hannobraun/inotify,ISC,"Hanno Braun <mail@hannobraun.
 inotify-sys,https://github.com/hannobraun/inotify-sys,ISC,Hanno Braun <hb@hannobraun.de>
 inout,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers
 instability,https://github.com/ratatui-org/instability,MIT,"Stephen M. Coakley <me@stephencoakley.com>, Joshka"
-integer-encoding,https://github.com/dermesser/integer-encoding-rs,MIT,Lewin Bormann <lbo@spheniscida.de>
 inventory,https://github.com/dtolnay/inventory,MIT OR Apache-2.0,David Tolnay <dtolnay@gmail.com>
 ipconfig,https://github.com/liranringel/ipconfig,MIT OR Apache-2.0,Liran Ringel <liranringel@gmail.com>
 ipcrypt-rs,https://github.com/jedisct1/rust-ipcrypt2,ISC,Frank Denis <github@pureftpd.org>
@@ -549,6 +548,8 @@ parking,https://github.com/smol-rs/parking,Apache-2.0 OR MIT,"Stjepan Glavina <s
 parking_lot,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras <amanieu@gmail.com>
 parking_lot_core,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras <amanieu@gmail.com>
 parquet,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow <dev@arrow.apache.org>
+parquet-variant,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow <dev@arrow.apache.org>
+parquet-variant-json,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow <dev@arrow.apache.org>
 parse-size,https://github.com/kennytm/parse-size,MIT,kennytm <kennytm@gmail.com>
 paste,https://github.com/dtolnay/paste,MIT OR Apache-2.0,David Tolnay <dtolnay@gmail.com>
 pastey,https://github.com/as1100k/pastey,MIT OR Apache-2.0,"Aditya Kumar <git@adityais.dev>, David Tolnay <dtolnay@gmail.com>"
@@ -777,7 +778,6 @@ terminal_size,https://github.com/eminence/terminal-size,MIT OR Apache-2.0,Andrew
 thiserror,https://github.com/dtolnay/thiserror,MIT OR Apache-2.0,David Tolnay <dtolnay@gmail.com>
 thiserror-impl,https://github.com/dtolnay/thiserror,MIT OR Apache-2.0,David Tolnay <dtolnay@gmail.com>
 thread_local,https://github.com/Amanieu/thread_local-rs,MIT OR Apache-2.0,Amanieu d'Antras <amanieu@gmail.com>
-thrift,https://github.com/apache/thrift/tree/master/lib/rs,Apache-2.0,Apache Thrift Developers <dev@thrift.apache.org>
 tikv-jemalloc-sys,https://github.com/tikv/jemallocator,MIT OR Apache-2.0,"Alex Crichton <alex@alexcrichton.com>, Gonzalo Brito Gadeschi <gonzalobg88@gmail.com>, The TiKV Project Developers"
 tikv-jemallocator,https://github.com/tikv/jemallocator,MIT OR Apache-2.0,"Alex Crichton <alex@alexcrichton.com>, Gonzalo Brito Gadeschi <gonzalobg88@gmail.com>, Simon Sapin <simon.sapin@exyr.org>, Steven Fackler <sfackler@gmail.com>, The TiKV Project Developers"
 time,https://github.com/time-rs/time,MIT OR Apache-2.0,"Jacob Pratt <open-source@jhpratt.dev>, Time contributors"

changelog.d/zerobus_ingest_sdk_2_4_0.enhancement.md

@@ -0,0 +1,3 @@
+The `databricks_zerobus` sink now supports Unity Catalog `VARIANT` columns.
+
+authors: yorickvanzweeden

lib/codecs/Cargo.toml

@@ -14,14 +14,16 @@ path = "tests/bin/generate-avro-fixtures.rs"
 
 [dependencies]
 apache-avro = { version = "0.20.0", default-features = false }
-arrow = { version = "58.2.0", default-features = false, features = ["ipc", "json"], optional = true }
-parquet = { version = "58.2.0", default-features = false, features = [
+arrow = { version = "59.1.0", default-features = false, features = ["ipc", "json"], optional = true }
+parquet = { version = "59.1.0", default-features = false, features = [
     "arrow",
     "snap",
     "zstd",
     "lz4",
     "flate2-rust_backened",
 ], optional = true }
+parquet-variant = { version = "59.1.0", default-features = false, optional = true }
+parquet-variant-json = { version = "59.1.0", default-features = false, optional = true }
 async-trait.workspace = true
 bytes.workspace = true
 chrono.workspace = true
@@ -33,6 +35,7 @@ derive_more = { version = "2.1.1", optional = true, features = ["from", "display
 dyn-clone = { version = "1", default-features = false }
 flate2.workspace = true
 futures.workspace = true
+hex = { version = "0.4.3", default-features = false, optional = true }
 influxdb-line-protocol = { version = "2", default-features = false }
 lookup = { package = "vector-lookup", path = "../vector-lookup", default-features = false, features = ["test"] }
 memchr = { version = "2", default-features = false }
@@ -77,7 +80,13 @@ uuid.workspace = true
 vrl.workspace = true
 
 [features]
-arrow = ["dep:arrow", "arrow/chrono-tz"]
+arrow = [
+    "dep:arrow",
+    "dep:hex",
+    "dep:parquet-variant",
+    "dep:parquet-variant-json",
+    "arrow/chrono-tz",
+]
 parquet = ["dep:parquet", "arrow"]
 opentelemetry = ["dep:opentelemetry-proto"]
 syslog = ["dep:syslog_loose", "dep:strum", "dep:derive_more", "dep:serde-aux", "dep:toml"]