feat(compression-coordinator): Introduce max_concurrent_tasks config to limit job-handler concurrency.

components/clp-py-utils/clp_py_utils/clp_config.py

@@ -761,6 +761,7 @@ class PollingBackoff(BaseModel):
 class CompressionCoordinator(BaseModel):
     resource_group: SpiderResourceGroup = SpiderResourceGroup(name="compression-coordinator")
     job_polling_interval_millisecs: PositiveInt = 100
+    max_concurrent_tasks: PositiveInt = 1000
     result_polling: PollingBackoff = PollingBackoff(
         init_backoff_millisecs=100, max_backoff_millisecs=1000
     )

components/clp-rust-utils/src/clp_config/package/config.rs

@@ -1,5 +1,6 @@
 use std::num::NonZeroU32;
 use std::num::NonZeroU64;
+use std::num::NonZeroUsize;
 use std::path::Path;
 use std::path::PathBuf;
 
@@ -482,6 +483,7 @@ impl Default for Telemetry {
 pub struct CompressionCoordinator {
     pub resource_group: SpiderResourceGroup,
     pub job_polling_interval_millisecs: NonZeroU64,
+    pub max_concurrent_tasks: NonZeroUsize,
     pub result_polling: PollingBackoff,
     pub compression_task_max_retry: u32,
     pub commit_task_max_retry: u32,
@@ -500,6 +502,8 @@ impl Default for CompressionCoordinator {
             },
             job_polling_interval_millisecs: NonZeroU64::new(100)
                 .expect("default jobs poll delay should not be zero"),
+            max_concurrent_tasks: NonZeroUsize::new(1000)
+                .expect("default maximum number of concurrent tasks should not be zero"),
             result_polling: PollingBackoff {
                 init_backoff_millisecs: NonZeroU64::new(100)
                     .expect("default result polling init backoff should not be zero"),