diff --git a/components/clp-package-utils/uv.lock b/components/clp-package-utils/uv.lock index e43742a55b..c441f1c310 100644 --- a/components/clp-package-utils/uv.lock +++ b/components/clp-package-utils/uv.lock @@ -544,7 +544,6 @@ dependencies = [ { name = "pydantic" }, { name = "pymongo" }, { name = "pyyaml" }, - { name = "yscope-spider-py" }, ] [package.metadata] @@ -558,7 +557,6 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.12.5" }, { name = "pymongo", specifier = ">=4.16.0" }, { name = "pyyaml", specifier = ">=6.0.3" }, - { name = "yscope-spider-py", specifier = "==0.2.0" }, ] [package.metadata.requires-dev] @@ -591,15 +589,6 @@ redis = [ { name = "redis" }, ] -[[package]] -name = "lark" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/da/34/28fff3ab31ccff1fd4f6c7c7b0ceb2b6968d8ea4950663eadcb5720591a0/lark-1.3.1.tar.gz", hash = "sha256:b426a7a6d6d53189d318f2b6236ab5d6429eaf09259f1ca33eb716eed10d2905", size = 382732, upload-time = "2025-10-27T18:25:56.653Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/82/3d/14ce75ef66813643812f3093ab17e46d3a206942ce7376d31ec2d36229e7/lark-1.3.1-py3-none-any.whl", hash = "sha256:c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12", size = 113151, upload-time = "2025-10-27T18:25:54.882Z" }, -] - [[package]] name = "levenshtein" version = "0.27.3" @@ -848,19 +837,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/f2/08ace4142eb281c12701fc3b93a10795e4d4dc7f753911d836675050f886/msgpack-1.1.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d99ef64f349d5ec3293688e91486c5fdb925ed03807f64d98d205d2713c60b46", size = 70868, upload-time = "2025-10-08T09:15:44.959Z" }, ] -[[package]] -name = "msgpack-types" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "msgpack" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/e2/fa5fd255886a05c188e3038b8456d52805a855cf20cf06cbda83beb0fb47/msgpack_types-0.7.0.tar.gz", hash = "sha256:dadd3c51a7a869c2dabf59a97bdcdca951c453f310d73d471e104366f5c4bc87", size = 6408, upload-time = "2026-01-13T23:47:19.242Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/5b/911af9f1f68f4e30474dc5923966656a57d6bc5b1f3b836362739a57d961/msgpack_types-0.7.0-py3-none-any.whl", hash = "sha256:b07de919b771a782cdfd49d4c7b6a085661b55036d616fda1c01e3ca78cce459", size = 8229, upload-time = "2026-01-13T23:47:17.444Z" }, -] - [[package]] name = "mypy" version = "1.19.1" @@ -1786,18 +1762,3 @@ sdist = { url = "https://files.pythonhosted.org/packages/64/6e/62daec357285b927e wheels = [ { url = "https://files.pythonhosted.org/packages/f2/3e/45583b67c2ff08ad5a582d316fcb2f11d6cf0a50c7707ac09d212d25bc98/wcwidth-0.5.0-py3-none-any.whl", hash = "sha256:1efe1361b83b0ff7877b81ba57c8562c99cf812158b778988ce17ec061095695", size = 93772, upload-time = "2026-01-27T01:31:43.432Z" }, ] - -[[package]] -name = "yscope-spider-py" -version = "0.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "lark" }, - { name = "mariadb" }, - { name = "msgpack" }, - { name = "msgpack-types" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2b/1d/4ff0569fc95f6210a4bff258bf070f09b39a6eb60e6772d3dad3f7eb9279/yscope_spider_py-0.2.0.tar.gz", hash = "sha256:43c819ad4e0a135e3ae63ca55d849b6b3015bcf9b982c337d5cf10ea07b4e947", size = 25554, upload-time = "2025-11-14T04:30:33.533Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/ac/9eb7fa22ac02641faf57532e44326c471840ebf125270cb4bfad0968f534/yscope_spider_py-0.2.0-py3-none-any.whl", hash = "sha256:a5d1e1c2182e81321cc3482bcca5b46998922c6d5ccfc5bbd1cc6326388435d0", size = 34481, upload-time = "2025-11-14T04:30:32.463Z" }, -] diff --git a/components/clp-py-utils/uv.lock b/components/clp-py-utils/uv.lock index 9255789e12..c2b0f42786 100644 --- a/components/clp-py-utils/uv.lock +++ b/components/clp-py-utils/uv.lock @@ -544,7 +544,6 @@ dependencies = [ { name = "pydantic" }, { name = "pymongo" }, { name = "pyyaml" }, - { name = "yscope-spider-py" }, ] [package.metadata] @@ -558,7 +557,6 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.12.5" }, { name = "pymongo", specifier = ">=4.16.0" }, { name = "pyyaml", specifier = ">=6.0.3" }, - { name = "yscope-spider-py", specifier = "==0.2.0" }, ] [package.metadata.requires-dev] @@ -591,15 +589,6 @@ redis = [ { name = "redis" }, ] -[[package]] -name = "lark" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/da/34/28fff3ab31ccff1fd4f6c7c7b0ceb2b6968d8ea4950663eadcb5720591a0/lark-1.3.1.tar.gz", hash = "sha256:b426a7a6d6d53189d318f2b6236ab5d6429eaf09259f1ca33eb716eed10d2905", size = 382732, upload-time = "2025-10-27T18:25:56.653Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/82/3d/14ce75ef66813643812f3093ab17e46d3a206942ce7376d31ec2d36229e7/lark-1.3.1-py3-none-any.whl", hash = "sha256:c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12", size = 113151, upload-time = "2025-10-27T18:25:54.882Z" }, -] - [[package]] name = "levenshtein" version = "0.27.3" @@ -848,19 +837,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/f2/08ace4142eb281c12701fc3b93a10795e4d4dc7f753911d836675050f886/msgpack-1.1.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d99ef64f349d5ec3293688e91486c5fdb925ed03807f64d98d205d2713c60b46", size = 70868, upload-time = "2025-10-08T09:15:44.959Z" }, ] -[[package]] -name = "msgpack-types" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "msgpack" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/e2/fa5fd255886a05c188e3038b8456d52805a855cf20cf06cbda83beb0fb47/msgpack_types-0.7.0.tar.gz", hash = "sha256:dadd3c51a7a869c2dabf59a97bdcdca951c453f310d73d471e104366f5c4bc87", size = 6408, upload-time = "2026-01-13T23:47:19.242Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/5b/911af9f1f68f4e30474dc5923966656a57d6bc5b1f3b836362739a57d961/msgpack_types-0.7.0-py3-none-any.whl", hash = "sha256:b07de919b771a782cdfd49d4c7b6a085661b55036d616fda1c01e3ca78cce459", size = 8229, upload-time = "2026-01-13T23:47:17.444Z" }, -] - [[package]] name = "mypy" version = "1.19.1" @@ -1786,18 +1762,3 @@ sdist = { url = "https://files.pythonhosted.org/packages/64/6e/62daec357285b927e wheels = [ { url = "https://files.pythonhosted.org/packages/f2/3e/45583b67c2ff08ad5a582d316fcb2f11d6cf0a50c7707ac09d212d25bc98/wcwidth-0.5.0-py3-none-any.whl", hash = "sha256:1efe1361b83b0ff7877b81ba57c8562c99cf812158b778988ce17ec061095695", size = 93772, upload-time = "2026-01-27T01:31:43.432Z" }, ] - -[[package]] -name = "yscope-spider-py" -version = "0.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "lark" }, - { name = "mariadb" }, - { name = "msgpack" }, - { name = "msgpack-types" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2b/1d/4ff0569fc95f6210a4bff258bf070f09b39a6eb60e6772d3dad3f7eb9279/yscope_spider_py-0.2.0.tar.gz", hash = "sha256:43c819ad4e0a135e3ae63ca55d849b6b3015bcf9b982c337d5cf10ea07b4e947", size = 25554, upload-time = "2025-11-14T04:30:33.533Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/ac/9eb7fa22ac02641faf57532e44326c471840ebf125270cb4bfad0968f534/yscope_spider_py-0.2.0-py3-none-any.whl", hash = "sha256:a5d1e1c2182e81321cc3482bcca5b46998922c6d5ccfc5bbd1cc6326388435d0", size = 34481, upload-time = "2025-11-14T04:30:32.463Z" }, -] diff --git a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py b/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py deleted file mode 100644 index a34062e3cc..0000000000 --- a/components/job-orchestration/job_orchestration/executor/compress/spider_compress.py +++ /dev/null @@ -1,54 +0,0 @@ -import json - -from clp_py_utils.clp_logging import get_logger -from spider_py import Int64, TaskContext -from structlog.contextvars import bound_contextvars - -from job_orchestration.executor.compress.compression_task import compression_entry_point - -# Setup logging -logger = get_logger("spider_compression") - - -def compress( - _: TaskContext, - job_id: Int64, - task_id: Int64, - clp_io_config_json: bytes, - paths_to_compress_json: bytes, - clp_metadata_db_connection_config_json: bytes, -) -> bytes: - """ - Compresses files using the general compression entry point. - :param _: Spider's task context. Not used in the function. - :param job_id: - :param task_id: - :param clp_io_config_json: A JSON string representation of - `job_orchestration.scheduler.constants.ClpIoConfig`. - :param paths_to_compress_json: A JSON string representation of - `job_orchestration.scheduler.constants.PathToCompress`. - :param clp_metadata_db_connection_config_json: A JSON string representation of - `clp_py_utils.clp_config.Database`. - :return: A JSON string representation of - `job_orchestration.scheduler.constants.CompressionTaskResult`, encoded as a list of - `spider_py.Int8`. - """ - job_id_int = int(job_id) - task_id_int = int(task_id) - with bound_contextvars(job_id=job_id_int, task_id=task_id_int): - try: - result_as_json_str = json.dumps( - compression_entry_point( - job_id_int, - task_id_int, - clp_io_config_json.decode("utf-8"), - paths_to_compress_json.decode("utf-8"), - json.loads(clp_metadata_db_connection_config_json.decode("utf-8")), - logger, - ) - ) - except Exception: - logger.exception("Compression task failed with an unexpected exception.") - raise - - return result_as_json_str.encode("utf-8") diff --git a/components/job-orchestration/job_orchestration/executor/start-spider-worker.py b/components/job-orchestration/job_orchestration/executor/start-spider-worker.py deleted file mode 100644 index 6e1caf629e..0000000000 --- a/components/job-orchestration/job_orchestration/executor/start-spider-worker.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python3 - -"""Script to start multiple Spider workers.""" - -import argparse -import logging -import os -import pathlib -import signal -import subprocess -import sys -import threading - -# Setup logging -# Create logger -logger = logging.getLogger("start-spider-worker") -logger.setLevel(logging.INFO) -# Setup console logging -logging_console_handler = logging.StreamHandler() -logging_formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s") -logging_console_handler.setFormatter(logging_formatter) -logger.addHandler(logging_console_handler) - -SPIDER_WORKER_TERM_TIMEOUT_SECONDS = 5 - -stop_event = threading.Event() - - -def handle_sigterm(signum, frame) -> None: - """ - Signal handler for SIGTERM. - Stops all running workers gracefully. - - :param signum: The signal number. - :param frame: The current stack frame. - """ - logger.info("Received SIGTERM, stopping workers...") - stop_event.set() - - -def parse_args() -> argparse.Namespace: - """ - Parses command line arguments. - :return: The parsed arguments. - """ - parser = argparse.ArgumentParser() - parser.add_argument( - "--num-workers", type=int, default=1, help="Number of concurrent Spider workers." - ) - parser.add_argument("--storage-url", type=str, required=True, help="Spider storage URL.") - parser.add_argument("--host", type=str, required=True, help="The worker host.") - return parser.parse_args() - - -def main() -> None: - """Main function to start multiple Spider workers.""" - # Parse arguments - args = parse_args() - num_workers = args.num_workers - if num_workers < 1: - logger.error("Number of concurrent workers must be at least 1.") - sys.exit(1) - storage_url = args.storage_url - host = args.host - - clp_home = os.getenv("CLP_HOME", "/opt/clp") - spider_worker_path = pathlib.Path(clp_home) / "bin" / "spider_worker" - if not spider_worker_path.exists(): - logger.error("spider_worker not found at %s.", spider_worker_path) - sys.exit(1) - - # Start multiple spider workers - processes = [] - try: - for i in range(num_workers): - proc = subprocess.Popen( - [spider_worker_path, "--storage_url", storage_url, "--host", host] - ) - processes.append(proc) - logger.info("Started Spider worker %d/%d (PID: %d).", i + 1, num_workers, proc.pid) - except OSError: - logger.exception("Failed to start Spider worker.") - for proc in processes: - logger.info("Terminating worker process %d.", proc.pid) - proc.terminate() - wait_exit(processes) - sys.exit(1) - - signal.signal(signal.SIGTERM, handle_sigterm) - - # Wait for SIGTERM - stop_event.wait() - - # Wait for termination with timeout - exit_code = wait_exit(processes) - sys.exit(exit_code) - - -def wait_exit(processes: list[subprocess.Popen]) -> int: - """ - Waits for all processes to exit. - If the process does not exit within the timeout, it is killed. - - :param processes: List of subprocess.Popen objects. - :return: Exit code of the first non-zero exit code, or 0 if all processes exited successfully. - """ - exit_code = 0 - for proc in processes: - try: - code = proc.wait(timeout=SPIDER_WORKER_TERM_TIMEOUT_SECONDS) - if code != 0 and exit_code == 0: - exit_code = code - except subprocess.TimeoutExpired: - logger.warning( - "Worker process %d did not terminate in time, sending SIGKILL.", proc.pid - ) - proc.kill() - proc.wait() - return exit_code - - -if __name__ == "__main__": - main() diff --git a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py b/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py deleted file mode 100644 index 97af8a0ffb..0000000000 --- a/components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py +++ /dev/null @@ -1,47 +0,0 @@ -from __future__ import annotations - -import json -from typing import Any - -import spider_py -from spider_py.client.job import Job - -from job_orchestration.executor.compress.spider_compress import compress -from job_orchestration.scheduler.compress.task_manager.task_manager import TaskManager -from job_orchestration.scheduler.task_result import CompressionTaskResult - - -class SpiderTaskManager(TaskManager): - class ResultHandle(TaskManager.ResultHandle): - def __init__(self, spider_job: Job) -> None: - self._spider_job: Job = spider_job - - def get_result(self, timeout: float = 0.1) -> list[CompressionTaskResult] | None: - job_results = self._spider_job.get_results() - if job_results is None: - return None - if not isinstance(job_results, tuple): - return [CompressionTaskResult.model_validate_json(job_results.decode("utf-8"))] - return [ - CompressionTaskResult.model_validate_json(task_result.decode("utf-8")) - for task_result in job_results - ] - - def __init__(self, storage_url: str) -> None: - self._driver = spider_py.Driver(storage_url) - - def submit(self, task_params: list[dict[str, Any]]) -> TaskManager.ResultHandle: - job = spider_py.group( - [compress for _ in range(len(task_params))], - ) - job_args = [] - for task_param in task_params: - job_args.append(spider_py.Int64(task_param["job_id"])) - job_args.append(spider_py.Int64(task_param["task_id"])) - job_args.append(task_param["clp_io_config_json"].encode("utf-8")) - job_args.append(task_param["paths_to_compress_json"].encode("utf-8")) - job_args.append( - json.dumps(task_param["clp_metadata_db_connection_config"]).encode("utf-8") - ) - submitted_job = self._driver.submit_jobs([job], [job_args])[0] - return SpiderTaskManager.ResultHandle(submitted_job) diff --git a/components/job-orchestration/pyproject.toml b/components/job-orchestration/pyproject.toml index c1c89b823e..11db067aa4 100644 --- a/components/job-orchestration/pyproject.toml +++ b/components/job-orchestration/pyproject.toml @@ -15,7 +15,6 @@ dependencies = [ "pydantic>=2.12.5", "pymongo>=4.16.0", "PyYAML>=6.0.3", - "yscope-spider-py==0.2.0", ] [build-system] diff --git a/components/job-orchestration/uv.lock b/components/job-orchestration/uv.lock index 7c4ebdca66..7bc1dfdcd2 100644 --- a/components/job-orchestration/uv.lock +++ b/components/job-orchestration/uv.lock @@ -534,7 +534,6 @@ dependencies = [ { name = "pydantic" }, { name = "pymongo" }, { name = "pyyaml" }, - { name = "yscope-spider-py" }, ] [package.dev-dependencies] @@ -558,7 +557,6 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.12.5" }, { name = "pymongo", specifier = ">=4.16.0" }, { name = "pyyaml", specifier = ">=6.0.3" }, - { name = "yscope-spider-py", specifier = "==0.2.0" }, ] [package.metadata.requires-dev] @@ -591,15 +589,6 @@ redis = [ { name = "redis" }, ] -[[package]] -name = "lark" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/da/34/28fff3ab31ccff1fd4f6c7c7b0ceb2b6968d8ea4950663eadcb5720591a0/lark-1.3.1.tar.gz", hash = "sha256:b426a7a6d6d53189d318f2b6236ab5d6429eaf09259f1ca33eb716eed10d2905", size = 382732, upload-time = "2025-10-27T18:25:56.653Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/82/3d/14ce75ef66813643812f3093ab17e46d3a206942ce7376d31ec2d36229e7/lark-1.3.1-py3-none-any.whl", hash = "sha256:c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12", size = 113151, upload-time = "2025-10-27T18:25:54.882Z" }, -] - [[package]] name = "levenshtein" version = "0.27.3" @@ -848,19 +837,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/f2/08ace4142eb281c12701fc3b93a10795e4d4dc7f753911d836675050f886/msgpack-1.1.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d99ef64f349d5ec3293688e91486c5fdb925ed03807f64d98d205d2713c60b46", size = 70868, upload-time = "2025-10-08T09:15:44.959Z" }, ] -[[package]] -name = "msgpack-types" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "msgpack" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/e2/fa5fd255886a05c188e3038b8456d52805a855cf20cf06cbda83beb0fb47/msgpack_types-0.7.0.tar.gz", hash = "sha256:dadd3c51a7a869c2dabf59a97bdcdca951c453f310d73d471e104366f5c4bc87", size = 6408, upload-time = "2026-01-13T23:47:19.242Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/5b/911af9f1f68f4e30474dc5923966656a57d6bc5b1f3b836362739a57d961/msgpack_types-0.7.0-py3-none-any.whl", hash = "sha256:b07de919b771a782cdfd49d4c7b6a085661b55036d616fda1c01e3ca78cce459", size = 8229, upload-time = "2026-01-13T23:47:17.444Z" }, -] - [[package]] name = "mypy" version = "1.19.1" @@ -1786,18 +1762,3 @@ sdist = { url = "https://files.pythonhosted.org/packages/64/6e/62daec357285b927e wheels = [ { url = "https://files.pythonhosted.org/packages/f2/3e/45583b67c2ff08ad5a582d316fcb2f11d6cf0a50c7707ac09d212d25bc98/wcwidth-0.5.0-py3-none-any.whl", hash = "sha256:1efe1361b83b0ff7877b81ba57c8562c99cf812158b778988ce17ec061095695", size = 93772, upload-time = "2026-01-27T01:31:43.432Z" }, ] - -[[package]] -name = "yscope-spider-py" -version = "0.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "lark" }, - { name = "mariadb" }, - { name = "msgpack" }, - { name = "msgpack-types" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2b/1d/4ff0569fc95f6210a4bff258bf070f09b39a6eb60e6772d3dad3f7eb9279/yscope_spider_py-0.2.0.tar.gz", hash = "sha256:43c819ad4e0a135e3ae63ca55d849b6b3015bcf9b982c337d5cf10ea07b4e947", size = 25554, upload-time = "2025-11-14T04:30:33.533Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/ac/9eb7fa22ac02641faf57532e44326c471840ebf125270cb4bfad0968f534/yscope_spider_py-0.2.0-py3-none-any.whl", hash = "sha256:a5d1e1c2182e81321cc3482bcca5b46998922c6d5ccfc5bbd1cc6326388435d0", size = 34481, upload-time = "2025-11-14T04:30:32.463Z" }, -] diff --git a/docs/src/dev-docs/tooling-containers.md b/docs/src/dev-docs/tooling-containers.md index 209bdcfa54..bf3a97af7e 100644 --- a/docs/src/dev-docs/tooling-containers.md +++ b/docs/src/dev-docs/tooling-containers.md @@ -167,11 +167,6 @@ The container image sets the following environment variables and configurations, /bin/bash Default shell for the default user. - - $LD_LIBRARY_PATH - ${CLP_HOME}/lib - Library path to include any shared libraries in the CLP package. - $PATH ${CLP_HOME}/sbin
diff --git a/integration-tests/uv.lock b/integration-tests/uv.lock index 0435a4626d..2921b12c14 100644 --- a/integration-tests/uv.lock +++ b/integration-tests/uv.lock @@ -1033,7 +1033,6 @@ dependencies = [ { name = "pydantic" }, { name = "pymongo" }, { name = "pyyaml" }, - { name = "yscope-spider-py" }, ] [package.metadata] @@ -1047,7 +1046,6 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.12.5" }, { name = "pymongo", specifier = ">=4.16.0" }, { name = "pyyaml", specifier = ">=6.0.3" }, - { name = "yscope-spider-py", specifier = "==0.2.0" }, ] [package.metadata.requires-dev] @@ -1149,15 +1147,6 @@ redis = [ { name = "redis" }, ] -[[package]] -name = "lark" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/da/34/28fff3ab31ccff1fd4f6c7c7b0ceb2b6968d8ea4950663eadcb5720591a0/lark-1.3.1.tar.gz", hash = "sha256:b426a7a6d6d53189d318f2b6236ab5d6429eaf09259f1ca33eb716eed10d2905", size = 382732, upload-time = "2025-10-27T18:25:56.653Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/82/3d/14ce75ef66813643812f3093ab17e46d3a206942ce7376d31ec2d36229e7/lark-1.3.1-py3-none-any.whl", hash = "sha256:c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12", size = 113151, upload-time = "2025-10-27T18:25:54.882Z" }, -] - [[package]] name = "levenshtein" version = "0.27.3" @@ -1535,19 +1524,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/f2/08ace4142eb281c12701fc3b93a10795e4d4dc7f753911d836675050f886/msgpack-1.1.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d99ef64f349d5ec3293688e91486c5fdb925ed03807f64d98d205d2713c60b46", size = 70868, upload-time = "2025-10-08T09:15:44.959Z" }, ] -[[package]] -name = "msgpack-types" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "msgpack" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/e2/fa5fd255886a05c188e3038b8456d52805a855cf20cf06cbda83beb0fb47/msgpack_types-0.7.0.tar.gz", hash = "sha256:dadd3c51a7a869c2dabf59a97bdcdca951c453f310d73d471e104366f5c4bc87", size = 6408, upload-time = "2026-01-13T23:47:19.242Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/5b/911af9f1f68f4e30474dc5923966656a57d6bc5b1f3b836362739a57d961/msgpack_types-0.7.0-py3-none-any.whl", hash = "sha256:b07de919b771a782cdfd49d4c7b6a085661b55036d616fda1c01e3ca78cce459", size = 8229, upload-time = "2026-01-13T23:47:17.444Z" }, -] - [[package]] name = "mypy" version = "1.19.1" @@ -3186,21 +3162,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" }, ] -[[package]] -name = "yscope-spider-py" -version = "0.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "lark" }, - { name = "mariadb" }, - { name = "msgpack" }, - { name = "msgpack-types" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2b/1d/4ff0569fc95f6210a4bff258bf070f09b39a6eb60e6772d3dad3f7eb9279/yscope_spider_py-0.2.0.tar.gz", hash = "sha256:43c819ad4e0a135e3ae63ca55d849b6b3015bcf9b982c337d5cf10ea07b4e947", size = 25554, upload-time = "2025-11-14T04:30:33.533Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/ac/9eb7fa22ac02641faf57532e44326c471840ebf125270cb4bfad0968f534/yscope_spider_py-0.2.0-py3-none-any.whl", hash = "sha256:a5d1e1c2182e81321cc3482bcca5b46998922c6d5ccfc5bbd1cc6326388435d0", size = 34481, upload-time = "2025-11-14T04:30:32.463Z" }, -] - [[package]] name = "zipp" version = "3.23.0" diff --git a/taskfile.yaml b/taskfile.yaml index 385d57bd0b..235c18ef36 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -37,7 +37,6 @@ vars: G_PYTHON_LIBS_DIR: "{{.G_BUILD_DIR}}/python-libs" G_PYTHON_WHEELS_BUILD_DIR: "{{.G_BUILD_DIR}}/python-wheels" G_RUST_BUILD_DIR: "{{.G_BUILD_DIR}}/rust-targets" - G_SPIDER_BUILD_DIR: "{{.G_BUILD_DIR}}/spider" G_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/webui" # Taskfile paths @@ -262,7 +261,6 @@ tasks: package-build-deps: deps: - "core" - - "deps:spider" - "init" - "python-libs" - "rust" diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 5a93077f24..b1b6a5e8ee 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -80,55 +80,6 @@ tasks: - task: "zlib" - task: "zstd" - spider: - deps: - - "spider-dep" - platforms: ["linux"] - preconditions: - # NOTE: Spider is only supported on Ubuntu at present. - - >- - source /etc/os-release && [[ "$ID" == "ubuntu" ]] - cmds: - - task: "yscope-dev-utils:cmake:install-remote-tar" - vars: - CMAKE_PACKAGE_NAME: "spider" - TAR_SHA256: "ef24c3271565864e630ac2c650fc46c62f0dd74d1a4c896fbdc67e5f37d9177e" - TAR_URL: "https://github.com/y-scope/spider/archive/32711f9.tar.gz" - WORK_DIR: "{{.G_SPIDER_BUILD_DIR}}" - CMAKE_JOBS: "{{.G_CPP_MAX_PARALLELISM_PER_BUILD_TASK}}" - CMAKE_GEN_ARGS: - - "-C {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/all-spider.cmake" - - "-DCMAKE_BUILD_TYPE=Release" - - "-DCMAKE_INSTALL_MESSAGE=LAZY" - - "-DSPIDER_BUILD_TESTING=OFF" - - spider-dep: - internal: true - deps: - - "utils:clean-outdated-cpp-checksum-files" - - "utils:init" - cmds: - - task: "yscope-dev-utils:cmake:install-deps-and-generate-settings" - vars: - CMAKE_SETTINGS_DIR: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}" - CMAKE_SETTINGS_FILE: "{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/all-spider.cmake" - DEP_TASK: "spider-all-parallel" - - task: "utils:combine-cpp-checksum-files" - - spider-all-parallel: - internal: true - run: "once" - deps: - - task: "absl" - - task: "antlr-jar" - - task: "antlr-runtime" - - task: "boost" - - task: "fmt" - - task: "mariadb-connector-cpp" - - task: "msgpack-cxx" - - task: "spdlog" - - task: "ystdlib" - absl: internal: true run: "once" @@ -367,29 +318,6 @@ tasks: TARBALL_SHA256: "537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b" TARBALL_URL: "https://github.com/lz4/lz4/releases/download/v1.10.0/lz4-1.10.0.tar.gz" - mariadb-connector-cpp: - platforms: ["linux"] - preconditions: - # NOTE: The MariaDB connector is only required for building Spider, which is only supported on - # Ubuntu. `INSTALL_LAYOUT` is currently hardcoded to "DEB". To support other Linux distros: - # - Update the precondition accordingly. - # - Make `INSTALL_LAYOUT` configurable. - - >- - source /etc/os-release && [[ "$ID" == "ubuntu" ]] - internal: true - run: "once" - cmds: - - task: "utils:install-remote-cmake-lib" - vars: - CMAKE_GEN_ARGS: - - "-DCMAKE_BUILD_TYPE=Release" - - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - - "-DINSTALL_LAYOUT=DEB" - LIB_NAME: "mariadb-connector-cpp" - TARBALL_URL: "https://github.com/mariadb-corporation/mariadb-connector-cpp/archive/refs/\ - tags/1.1.5.tar.gz" - TARBALL_SHA256: "0e3dfe9f2bc3f7bb6f7c159009556290064a7c23402ea08019fa8aebfc3ff2c9" - microsoft.gsl: internal: true run: "once" diff --git a/tools/docker-images/clp-package/Dockerfile b/tools/docker-images/clp-package/Dockerfile index 0516b95bb8..7bc85e6a4e 100644 --- a/tools/docker-images/clp-package/Dockerfile +++ b/tools/docker-images/clp-package/Dockerfile @@ -19,8 +19,7 @@ COPY --link --from=base / / ARG UID=1000 ENV CLP_HOME="/opt/clp" -ENV LD_LIBRARY_PATH="${CLP_HOME}/lib" \ - PATH="${CLP_HOME}/sbin:${CLP_HOME}/bin:${PATH}" \ +ENV PATH="${CLP_HOME}/sbin:${CLP_HOME}/bin:${PATH}" \ PYTHONPATH="${CLP_HOME}/lib/python3/site-packages" \ USER="clp-user" @@ -39,10 +38,6 @@ COPY --link --chown=${UID} ./build/core/clp-s bin/ COPY --link --chown=${UID} ./build/core/indexer bin/ COPY --link --chown=${UID} ./build/core/log-converter bin/ COPY --link --chown=${UID} ./build/core/reducer-server bin/ -COPY --link --chown=${UID} ./build/deps/cpp/mariadb-connector-cpp-install/lib/*/libmariadbcpp.so* \ - lib/ -COPY --link --chown=${UID} ./build/spider/spider-build/src/spider/spider_scheduler bin/ -COPY --link --chown=${UID} ./build/spider/spider-build/src/spider/spider_worker bin/ COPY --link --chown=${UID} ./build/nodejs/bin/node bin/node COPY --link --chown=${UID} ./build/python-libs/ lib/python3/site-packages/ COPY --link --chown=${UID} ./build/rust-targets/release/api_server bin/