diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..1d4830e --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.10.17 diff --git a/Makefile b/Makefile index 27f5dc1..acfb590 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build +.PHONY: build check clean test publish service-build service-run service-test build: @echo "build" @@ -19,3 +19,22 @@ test: publish: @echo "publish" twine upload dist/* + +# --- HTTP /validate service --------------------------------------------------- +# Build the Docker image. Build context is the repository root so that the +# Dockerfile can `pip install .[service]`. +SERVICE_IMAGE ?= morphic-util-service:latest + +service-build: + @echo "building $(SERVICE_IMAGE)" + docker build -f service/Dockerfile -t $(SERVICE_IMAGE) . + +# Run the service locally with hot reload (no Docker). Requires the service +# extra to be installed: `pip install -e .[service]`. +service-run: + @echo "running service on http://localhost:8000" + uvicorn service.app:app --host 0.0.0.0 --port 8000 --reload + +service-test: + @echo "running service tests" + pytest service/tests diff --git a/ait/commons/util/command/submit_file.py b/ait/commons/util/command/submit_file.py index 04efb9b..dbc27bf 100644 --- a/ait/commons/util/command/submit_file.py +++ b/ait/commons/util/command/submit_file.py @@ -14,6 +14,7 @@ from ait.commons.util.spreadsheet_util import SpreadsheetSubmitter, ValidationError, \ merge_library_preparation_sequencing_file, merge_cell_line_and_differentiated_cell_line, \ merge_differentiated_cell_line_and_library_preparation, SubmissionError, process_library_preparations +from ait.commons.util.spreadsheet_validate import SpreadsheetValidator # Define a class for handling submission of a command file @@ -321,132 +322,16 @@ def _handle_expression_alterations(self, return created_expression_alterations def _parse_spreadsheet(self, parser): + """Delegate to the shared SpreadsheetValidator so the CLI and the + HTTP /validate service share one parsing path. self.validation_errors + is mutated in place to preserve existing CLI behavior.""" try: - # Determine the necessary sheet names - tab_names = parser.list_sheets() - - cell_line_sheet_name = next( - (name for name in ["Cell line", "Clonal cell line"] if name in tab_names), None - ) - - differentiated_cell_line_sheet_name = next( - (name for name in ["Differentiated cell line", "Differentiated product"] if name in tab_names), None - ) - - undifferentiated_cell_line_sheet_name = ( - "Undifferentiated product" if "Undifferentiated product" in tab_names else None + return SpreadsheetValidator._parse_spreadsheet( + parser, + action=self.action, + context=self.context, + validation_errors=self.validation_errors, ) - - undifferentiated_cell_lines = [] - undifferentiated_cell_lines_df = None - - differentiated_cell_lines = [] - differentiated_cell_lines_df = None - - differentiated = False - - # Validate the presence of required sheets - if not cell_line_sheet_name: - self.validation_errors.append("Spreadsheet must contain a " - "'Cell line' or 'Clonal cell line' sheet.") - - if not (differentiated_cell_line_sheet_name or undifferentiated_cell_line_sheet_name): - self.validation_errors.append( - "Spreadsheet must contain a " - "'Differentiated cell line', 'Undifferentiated product', " - "or 'Differentiated product' sheet." - ) - - # Parse different sections of the spreadsheet - expression_alterations, expression_alterations_df = parser.get_expression_alterations( - 'Expression alteration', self.action, self.validation_errors, - context=self.context - ) - - cell_lines, cell_lines_df, parent_cell_line_names = parser.get_cell_lines( - cell_line_sheet_name, self.action, self.validation_errors, context=self.context - ) - - if differentiated_cell_line_sheet_name: - differentiated_cell_lines, differentiated_cell_lines_df = parser.get_differentiated_cell_lines( - differentiated_cell_line_sheet_name, self.action, self.validation_errors - ) - - if undifferentiated_cell_line_sheet_name: - undifferentiated_cell_lines, undifferentiated_cell_lines_df = parser.get_undifferentiated_cell_lines( - undifferentiated_cell_line_sheet_name, self.action, self.validation_errors - ) - - # Check for errors and merge data - if differentiated_cell_lines and undifferentiated_cell_lines: - self.validation_errors.append( - "A spreadsheet cannot contain rows in both differentiated and undifferentiated cell lines/ products" - ) - - if differentiated_cell_lines: - differentiated = True - merge_cell_line_and_differentiated_cell_line(cell_lines, differentiated_cell_lines, - self.validation_errors, context=self.context) - - if undifferentiated_cell_lines and not differentiated: - merge_cell_line_and_differentiated_cell_line(cell_lines, undifferentiated_cell_lines, - self.validation_errors, context=self.context) - - library_preparations_result = parser.get_library_preparations( - 'Library preparation', differentiated, self.action, self.validation_errors) - - if not isinstance(library_preparations_result, tuple) or len(library_preparations_result) != 2: - raise ValueError("Unexpected return from get_library_preparations()") - - library_preparations, library_preparations_df = library_preparations_result - - # Handle N:1 relationships for differentiated products in library preparation - for lp in library_preparations: - if "differentiated_biomaterial_id" in lp.__dict__: - differentiated_ids = lp.differentiated_biomaterial_id.split("|") - lp.differentiated_biomaterial_id = differentiated_ids - - if differentiated_cell_lines: - if self.context == "unperturbed_multiple": - # Use the new processing that creates a LP process and links the clone and differentiated product - process_library_preparations(cell_lines, differentiated_cell_lines, library_preparations, self.validation_errors) - else: - # Use the original merge function for differentiated cell lines (for MSK, JAX, etc.) - merge_differentiated_cell_line_and_library_preparation(differentiated_cell_lines, - library_preparations, self.validation_errors, cell_lines=cell_lines) - elif undifferentiated_cell_lines and not differentiated: - if self.context == "unperturbed_multiple": - process_library_preparations(cell_lines, undifferentiated_cell_lines, library_preparations, self.validation_errors) - else: - merge_differentiated_cell_line_and_library_preparation(undifferentiated_cell_lines, - library_preparations, self.validation_errors, cell_lines=cell_lines) - - sequencing_files, sequencing_files_df = parser.get_sequencing_files( - 'Sequence file', self.action, self.validation_errors - ) - - merge_library_preparation_sequencing_file(library_preparations, sequencing_files, self.validation_errors) - - # Return the parsed data as a dictionary - return { - "expression_alterations": expression_alterations, - "expression_alterations_df": expression_alterations_df, - "cell_lines": cell_lines, - "cell_lines_df": cell_lines_df, - "parent_cell_line_names": parent_cell_line_names, - "differentiated_cell_lines": differentiated_cell_lines, - "differentiated_cell_lines_df": differentiated_cell_lines_df, - "undifferentiated_cell_lines": undifferentiated_cell_lines, - "undifferentiated_cell_lines_df": undifferentiated_cell_lines_df, - "library_preparations": library_preparations, - "library_preparations_df": library_preparations_df, - "sequencing_files": sequencing_files, - "sequencing_files_df": sequencing_files_df, - "differentiated": differentiated, - "cell_line_sheet_name": cell_line_sheet_name, - "differentiated_cell_line_sheet_name": differentiated_cell_line_sheet_name, - "undifferentiated_cell_line_sheet_name": undifferentiated_cell_line_sheet_name - } except Exception as e: print(f"Exception occurred:", e) diff --git a/ait/commons/util/spreadsheet_util.py b/ait/commons/util/spreadsheet_util.py index 798abdb..f6fbf85 100644 --- a/ait/commons/util/spreadsheet_util.py +++ b/ait/commons/util/spreadsheet_util.py @@ -765,7 +765,7 @@ def list_sheets(self): return [sheet_name.strip() for sheet_name in xls.sheet_names] def input_file_to_data_frames(self, sheet_name, action): - if action.upper() == 'MODIFY': + if action and action.upper() == 'MODIFY': skip_rows = 0 else: skip_rows = 3 diff --git a/ait/commons/util/spreadsheet_validate.py b/ait/commons/util/spreadsheet_validate.py new file mode 100644 index 0000000..b3bf2e4 --- /dev/null +++ b/ait/commons/util/spreadsheet_validate.py @@ -0,0 +1,195 @@ +""" +Standalone spreadsheet validation/parsing logic shared by: + + * CmdSubmitFile (the CLI submit-file command) + * The HTTP /validate service (service/app.py) + +The CLI passes its own action/context/validation_errors so existing behavior +is preserved (validation_errors is mutated in place). The HTTP service can +call this with action=None, context=None, validation_errors=None and read +the errors back from the returned dict. +""" + +from .spreadsheet_util import ( + merge_cell_line_and_differentiated_cell_line, + merge_differentiated_cell_line_and_library_preparation, + merge_library_preparation_sequencing_file, + process_library_preparations, + ValidationError, +) + + +class SpreadsheetValidator: + @staticmethod + def _parse_spreadsheet(parser, action=None, context=None, validation_errors=None): + """ + Parse a spreadsheet (already opened via SpreadsheetSubmitter) into a + structured dict, accumulating non-fatal validation errors in + ``validation_errors`` (mutated in place if provided). + + Parameters + ---------- + parser : SpreadsheetSubmitter + The opened spreadsheet. + action : str | None + ADD / MODIFY / DELETE for the CLI; None for the HTTP validator. + context : str | None + Optional ingestion context (e.g. ``pooled_differentiated``, + ``unperturbed_multiple``). + validation_errors : list | None + Existing errors list to append to. If None, a new list is created. + + Returns + ------- + dict + Parsed sections plus ``errors`` and ``sheets`` keys for callers + (the HTTP service) that don't have a separate handle on the + mutated ``validation_errors`` list. + """ + if validation_errors is None: + validation_errors = [] + + try: + tab_names = parser.list_sheets() + + cell_line_sheet_name = next( + (name for name in ["Cell line", "Clonal cell line"] if name in tab_names), None + ) + + differentiated_cell_line_sheet_name = next( + (name for name in ["Differentiated cell line", "Differentiated product"] if name in tab_names), None + ) + + undifferentiated_cell_line_sheet_name = ( + "Undifferentiated product" if "Undifferentiated product" in tab_names else None + ) + + undifferentiated_cell_lines = [] + undifferentiated_cell_lines_df = None + + differentiated_cell_lines = [] + differentiated_cell_lines_df = None + + differentiated = False + + # Validate the presence of required sheets + if not cell_line_sheet_name: + validation_errors.append( + "Spreadsheet must contain a 'Cell line' or 'Clonal cell line' sheet." + ) + + if not (differentiated_cell_line_sheet_name or undifferentiated_cell_line_sheet_name): + validation_errors.append( + "Spreadsheet must contain a " + "'Differentiated cell line', 'Undifferentiated product', " + "or 'Differentiated product' sheet." + ) + + # Parse different sections of the spreadsheet + expression_alterations, expression_alterations_df = parser.get_expression_alterations( + 'Expression alteration', action, validation_errors, + context=context + ) + + cell_lines, cell_lines_df, parent_cell_line_names = parser.get_cell_lines( + cell_line_sheet_name, action, validation_errors, context=context + ) + + if differentiated_cell_line_sheet_name: + differentiated_cell_lines, differentiated_cell_lines_df = parser.get_differentiated_cell_lines( + differentiated_cell_line_sheet_name, action, validation_errors + ) + + if undifferentiated_cell_line_sheet_name: + undifferentiated_cell_lines, undifferentiated_cell_lines_df = parser.get_undifferentiated_cell_lines( + undifferentiated_cell_line_sheet_name, action, validation_errors + ) + + # Check for errors and merge data + if differentiated_cell_lines and undifferentiated_cell_lines: + validation_errors.append( + "A spreadsheet cannot contain rows in both differentiated and undifferentiated cell lines/ products" + ) + + if differentiated_cell_lines: + differentiated = True + merge_cell_line_and_differentiated_cell_line( + cell_lines, differentiated_cell_lines, validation_errors, context=context + ) + + if undifferentiated_cell_lines and not differentiated: + merge_cell_line_and_differentiated_cell_line( + cell_lines, undifferentiated_cell_lines, validation_errors, context=context + ) + + library_preparations_result = parser.get_library_preparations( + 'Library preparation', differentiated, action, validation_errors + ) + + if not isinstance(library_preparations_result, tuple) or len(library_preparations_result) != 2: + raise ValueError("Unexpected return from get_library_preparations()") + + library_preparations, library_preparations_df = library_preparations_result + + # Handle N:1 relationships for differentiated products in library preparation + for lp in library_preparations: + if "differentiated_biomaterial_id" in lp.__dict__: + differentiated_ids = lp.differentiated_biomaterial_id.split("|") + lp.differentiated_biomaterial_id = differentiated_ids + + if differentiated_cell_lines: + if context == "unperturbed_multiple": + process_library_preparations( + cell_lines, differentiated_cell_lines, library_preparations, validation_errors + ) + else: + merge_differentiated_cell_line_and_library_preparation( + differentiated_cell_lines, library_preparations, validation_errors, + cell_lines=cell_lines + ) + elif undifferentiated_cell_lines and not differentiated: + if context == "unperturbed_multiple": + process_library_preparations( + cell_lines, undifferentiated_cell_lines, library_preparations, validation_errors + ) + else: + merge_differentiated_cell_line_and_library_preparation( + undifferentiated_cell_lines, library_preparations, validation_errors, + cell_lines=cell_lines + ) + + sequencing_files, sequencing_files_df = parser.get_sequencing_files( + 'Sequence file', action, validation_errors + ) + + merge_library_preparation_sequencing_file( + library_preparations, sequencing_files, validation_errors + ) + + # Return parsed data as a dictionary. The CLI ignores the extra + # 'errors' / 'sheets' keys (it reads self.validation_errors); the + # HTTP service reads them directly from this dict. + return { + "sheets": tab_names, + "errors": validation_errors, + "expression_alterations": expression_alterations, + "expression_alterations_df": expression_alterations_df, + "cell_lines": cell_lines, + "cell_lines_df": cell_lines_df, + "parent_cell_line_names": parent_cell_line_names, + "differentiated_cell_lines": differentiated_cell_lines, + "differentiated_cell_lines_df": differentiated_cell_lines_df, + "undifferentiated_cell_lines": undifferentiated_cell_lines, + "undifferentiated_cell_lines_df": undifferentiated_cell_lines_df, + "library_preparations": library_preparations, + "library_preparations_df": library_preparations_df, + "sequencing_files": sequencing_files, + "sequencing_files_df": sequencing_files_df, + "differentiated": differentiated, + "cell_line_sheet_name": cell_line_sheet_name, + "differentiated_cell_line_sheet_name": differentiated_cell_line_sheet_name, + "undifferentiated_cell_line_sheet_name": undifferentiated_cell_line_sheet_name, + } + except Exception: + # Re-raise; caller decides how to surface (CLI prints, HTTP returns 400/500) + raise diff --git a/service/.dockerignore b/service/.dockerignore new file mode 100644 index 0000000..635306f --- /dev/null +++ b/service/.dockerignore @@ -0,0 +1,10 @@ +.git +.venv +.vscode +__pycache__ +*.pyc +*.pyo +build +dist +*.egg-info +.pytest_cache diff --git a/service/Dockerfile b/service/Dockerfile new file mode 100644 index 0000000..1b0cc2f --- /dev/null +++ b/service/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.10-slim + +WORKDIR /app + +# Install build deps for any wheels that need compiling (pandas/numpy etc.) +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +# Copy the package source and install morphic-util plus the service extras. +# Build context for this Dockerfile is the repository root, e.g.: +# docker build -f service/Dockerfile -t morphic-util-service . +COPY . /app + +RUN pip install --no-cache-dir --upgrade pip \ + && pip install --no-cache-dir ".[service]" + +EXPOSE 8000 + +CMD ["uvicorn", "service.app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/service/__init__.py b/service/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/service/app.py b/service/app.py new file mode 100644 index 0000000..cefb81a --- /dev/null +++ b/service/app.py @@ -0,0 +1,140 @@ +"""HTTP /validate service for morphic-util spreadsheets. + +Shares the parsing/validation logic with the CLI via +``ait.commons.util.spreadsheet_validate.SpreadsheetValidator``. + +Run locally: + uvicorn service.app:app --host 0.0.0.0 --port 8000 + +Or via Docker (see service/Dockerfile).""" + +from tempfile import NamedTemporaryFile +from typing import Any + +import numpy as np +import pandas as pd +import uvicorn +from fastapi import FastAPI, File, HTTPException, UploadFile +from fastapi.encoders import jsonable_encoder +from fastapi.responses import JSONResponse + +from ait.commons.util.spreadsheet_util import ( + SpreadsheetSubmitter, + ValidationError, + SubmissionError, # noqa: F401 (re-exported for parity with the CLI surface) +) +from ait.commons.util.spreadsheet_validate import SpreadsheetValidator + + +def convert_to_json_serializable(obj: Any) -> Any: + """Recursively convert pandas/numpy objects to JSON-serializable forms.""" + if isinstance(obj, dict): + return {k: convert_to_json_serializable(v) for k, v in obj.items()} + if isinstance(obj, (list, tuple)): + return [convert_to_json_serializable(item) for item in obj] + if isinstance(obj, pd.DataFrame): + return obj.replace({np.nan: None, np.inf: None, -np.inf: None}).to_dict('records') + if isinstance(obj, pd.Series): + return obj.replace({np.nan: None, np.inf: None, -np.inf: None}).to_dict() + if isinstance(obj, np.ndarray): + return convert_to_json_serializable(obj.tolist()) + if isinstance(obj, (np.integer, np.int64)): + return int(obj) + if isinstance(obj, (np.floating, np.float64)): + if np.isnan(obj) or np.isinf(obj): + return None + return float(obj) + try: + if pd.isna(obj): + return None + except (TypeError, ValueError): + pass + return obj + + +app = FastAPI(title="morphic-util validator", version="1.0") + + +@app.get("/health") +def health(): + return {"status": "ok"} + + +@app.post("/validate") +async def validate_xlsx(file: UploadFile = File(...)): + if not file.filename or not file.filename.endswith(".xlsx"): + raise HTTPException(status_code=400, detail="Only .xlsx files are supported.") + + contents = await file.read() + serialized_data: dict = {"file_name": file.filename} + + try: + with NamedTemporaryFile(delete=True, suffix=".xlsx") as tmp: + tmp.write(contents) + tmp.flush() + + try: + submitter = SpreadsheetSubmitter(tmp.name) + parsed_data = SpreadsheetValidator._parse_spreadsheet(submitter) + + is_valid = not parsed_data.get("errors") + + serialized_data = convert_to_json_serializable(parsed_data) + serialized_data["file_name"] = file.filename + + # Strip metadata-only keys from the data payload + parsed_data_no_info = { + k: v for k, v in parsed_data.items() if k not in ("errors", "sheets") + } + serialized_data_no_info = convert_to_json_serializable(parsed_data_no_info) + + if not is_valid: + return JSONResponse(status_code=400, content=jsonable_encoder({ + "valid": False, + "file_name": serialized_data.get("file_name"), + "errors": serialized_data.get("errors", []), + "message": "Spreadsheet validation failed", + "sheets_found": serialized_data.get("sheets", []), + })) + + return JSONResponse(content=jsonable_encoder({ + "valid": True, + "file_name": serialized_data.get("file_name"), + "sheets_found": serialized_data.get("sheets", []), + "errors": serialized_data.get("errors", []), + "data": serialized_data_no_info, + })) + + except ValidationError as ve: + return JSONResponse(status_code=400, content={ + "valid": False, + "file_name": serialized_data.get("file_name"), + "errors": getattr(ve, "errors", [str(ve)]), + "message": "Validation failed", + }) + except AttributeError as ae: + return JSONResponse(status_code=400, content={ + "valid": False, + "file_name": serialized_data.get("file_name"), + "errors": [str(ae)], + "message": "Spreadsheet is missing required sheets or fields", + }) + except Exception as e: + return JSONResponse(status_code=400, content={ + "valid": False, + "file_name": serialized_data.get("file_name"), + "errors": [str(e)], + "message": "Validation failed with an unexpected error", + }) + except Exception as e: + return JSONResponse(status_code=500, content={ + "valid": False, + "file_name": serialized_data.get("file_name"), + "errors": [str(e)], + "message": "Server error", + }) + + +# For local dev: `python -m service.app` +if __name__ == "__main__": + uvicorn.run("service.app:app", host="0.0.0.0", port=8000, reload=True) diff --git a/service/requirements.txt b/service/requirements.txt new file mode 100644 index 0000000..c6dfa17 --- /dev/null +++ b/service/requirements.txt @@ -0,0 +1,3 @@ +fastapi +uvicorn[standard] +python-multipart diff --git a/service/tests/__init__.py b/service/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/service/tests/test_app.py b/service/tests/test_app.py new file mode 100644 index 0000000..2c8a217 --- /dev/null +++ b/service/tests/test_app.py @@ -0,0 +1,76 @@ +"""Minimal pytest coverage for the /validate HTTP service. + +These tests intentionally avoid depending on a real morphic spreadsheet +fixture. They verify: + + * non-.xlsx uploads are rejected with 400 + * the health endpoint responds + * an .xlsx that's missing the required morphic sheets returns a + structured 400 (validation error), not a 500 (server error) + +For deeper coverage, add fixtures with real spreadsheet content and test +the happy path against SpreadsheetValidator directly. +""" + +import io + +import pytest +from fastapi.testclient import TestClient +from openpyxl import Workbook + +from service.app import app + + +@pytest.fixture +def client(): + return TestClient(app) + + +@pytest.fixture +def empty_xlsx_bytes(): + """An .xlsx with one empty 'Random' sheet — no morphic sheets present.""" + wb = Workbook() + ws = wb.active + ws.title = "Random" + ws["A1"] = "placeholder" + + buf = io.BytesIO() + wb.save(buf) + buf.seek(0) + return buf.read() + + +def test_health(client): + response = client.get("/health") + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + +def test_rejects_non_xlsx(client): + response = client.post( + "/validate", + files={"file": ("notes.txt", b"hello world", "text/plain")}, + ) + assert response.status_code == 400 + body = response.json() + assert "xlsx" in body["detail"].lower() + + +def test_missing_required_sheets_returns_400_with_errors(client, empty_xlsx_bytes): + response = client.post( + "/validate", + files={ + "file": ( + "empty.xlsx", + empty_xlsx_bytes, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ) + }, + ) + # Must be a structured validation failure, never a 500. + assert response.status_code == 400, response.text + body = response.json() + assert body["valid"] is False + assert body["file_name"] == "empty.xlsx" + assert isinstance(body["errors"], list) + assert body["errors"], "expected at least one validation error for an empty spreadsheet" diff --git a/setup.py b/setup.py index 2b9c530..31eb2b6 100644 --- a/setup.py +++ b/setup.py @@ -53,6 +53,16 @@ packages=['ait.commons.util', 'ait.commons.util.settings', 'ait.commons.util.command'], include_package_data=True, install_requires=ALL_REQS, + extras_require={ + # `pip install morphic-util[service]` brings in the FastAPI HTTP /validate + # service deps. Kept separate from the CLI install so the published PyPI + # package stays lean. + 'service': [ + 'fastapi', + 'uvicorn[standard]', + 'python-multipart', + ], + }, entry_points={ 'console_scripts': [ f'{NAME}=ait.commons.util.__main__:main',