fix(clp-package)!: Restore zstd-compressed job configs

components/api-server/src/error.rs

@@ -1,3 +1,5 @@
+//! Error types for the API server.
+
 use aws_sdk_s3::{error::SdkError, primitives::ByteStreamError};
 use num_enum::TryFromPrimitive;
 use thiserror::Error;
@@ -80,9 +82,9 @@ impl From<ByteStreamError> for ClientError {
 impl From<clp_rust_utils::Error> for ClientError {
     fn from(value: clp_rust_utils::Error) -> Self {
         match value {
-            clp_rust_utils::Error::MsgpackEncode(_) | clp_rust_utils::Error::SerdeYaml(_) => {
-                Self::MalformedData
-            }
+            clp_rust_utils::Error::MsgpackEncode(_)
+            | clp_rust_utils::Error::SerdeYaml(_)
+            | clp_rust_utils::Error::Zstd(_) => Self::MalformedData,
             clp_rust_utils::Error::Io(error) => error.into(),
             clp_rust_utils::Error::Sqlx(error) => error.into(),
             clp_rust_utils::Error::TelemetryExporterBuildError(error) => Self::Telemetry(error),

components/clp-package-utils/clp_package_utils/scripts/native/compress.py

@@ -7,8 +7,6 @@
 import time
 from contextlib import closing
 
-import brotli
-import msgpack
 from clp_py_utils.clp_config import (
     AwsAuthentication,
     CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
@@ -19,6 +17,7 @@
 from clp_py_utils.pretty_size import pretty_size
 from clp_py_utils.s3_utils import parse_s3_url
 from clp_py_utils.sql_adapter import SqlAdapter
+from clp_py_utils.zstd_msgpack import serialize as serialize_zstd_msgpack
 from job_orchestration.scheduler.constants import (
     CompressionJobCompletionStatus,
     CompressionJobStatus,
@@ -120,9 +119,8 @@ def handle_job(sql_adapter: SqlAdapter, clp_io_config: ClpIoConfig, no_progress_
         closing(db.cursor(dictionary=True)) as db_cursor,
     ):
         try:
-            compressed_clp_io_config = brotli.compress(
-                msgpack.packb(clp_io_config.model_dump(exclude_none=True, exclude_unset=True)),
-                quality=4,
+            compressed_clp_io_config = serialize_zstd_msgpack(
+                clp_io_config.model_dump(exclude_none=True, exclude_unset=True)
             )
             db_cursor.execute(
                 f"INSERT INTO {COMPRESSION_JOBS_TABLE_NAME} (clp_config) VALUES (%s)",

components/clp-package-utils/pyproject.toml

@@ -6,7 +6,6 @@ authors = [{name = "YScope Inc.", email = "dev@yscope.com"}]
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
-    "Brotli>=1.2.0",
     "click>=8.3.1",
     "msgpack>=1.1.2",
     "psutil>=7.2.1",