diff --git a/src/datasure/checks/outliers.py b/src/datasure/checks/outliers.py deleted file mode 100644 index d14bcb39..00000000 --- a/src/datasure/checks/outliers.py +++ /dev/null @@ -1,2864 +0,0 @@ -"""Outliers detection module for survey data quality checks. - -This module provides comprehensive outlier detection functionality with: -- Multiple detection methods (IQR, Standard Deviation) -- Polars-based optimizations for performance -- Pydantic validation for data integrity -- Modular, testable architecture -""" - -import re -from enum import Enum, IntEnum, StrEnum -from typing import Any - -import numpy as np -import pandas as pd -import plotly.graph_objects as go # type: ignore -import polars as pl -import streamlit as st -from pydantic import ( - BaseModel, - Field, - ValidationError, - field_validator, - model_validator, -) - -from datasure.utils.dataframe_utils import ( - ColumnByType, - safe_to_numeric, - sanitize_df_for_join, -) -from datasure.utils.duckdb_utils import duckdb_get_table, duckdb_save_table -from datasure.utils.navigations_utils import demo_callout -from datasure.utils.onboarding_utils import demo_output_onboarding, is_demo_project -from datasure.utils.settings_utils import ( - load_check_settings, - save_check_settings, - trigger_save, -) - -TAB_NAME: str = "outliers" - - -# ============================================================================= -# Enums and Constants -# ============================================================================= - - -class OutlierMethod(StrEnum): - """Supported outlier detection methods.""" - - IQR = "Interquartile Range (IQR)" - SD = "Standard Deviation (SD)" - - -class SearchType(StrEnum): - """Column search pattern types.""" - - EXACT = "exact" - STARTSWITH = "startswith" - ENDSWITH = "endswith" - CONTAINS = "contains" - REGEX = "regex" - - -class OutlierThresholds(IntEnum): - """Integer thresholds""" - - IQR = 20 - SD = 30 - - -class OutlierMultipliers(float, Enum): - """Float multipliers""" - - IQR = 1.5 - SD = 3.0 - - -# ============================================================================= -# Pydantic Models for Data Validation -# ============================================================================= - - -class OutlierBounds(BaseModel): - """Statistical bounds for outlier detection.""" - - lower_bound: float - upper_bound: float - - -class OutlierOptionsConfig(BaseModel): - """Configuration for outlier options.""" - - outlier_method: OutlierMethod = Field( - ..., description="Outlier detection method to use." - ) - outlier_multiplier: float = Field( - ..., - gt=0, - le=10.0, - description="Multiplier for outlier detection method.", - ) - outlier_threshold: int = Field( - ..., - gt=0, - description="Minimum number of non-null values required to flag outliers.", - ) - - -class ConstraintBounds(BaseModel): - """User-defined constraint bounds for outlier detection. - - Bounds hierarchy: hard_min <= soft_min <= soft_max <= hard_max - Values can be positive, negative, or zero. Infinity values are not allowed. - """ - - hard_min: int | float | None = Field(None, description="Absolute Minimum bound") - soft_min: int | float | None = Field(None, description="Expected Minimum bound") - soft_max: int | float | None = Field(None, description="Expected Maximum bound") - hard_max: int | float | None = Field(None, description="Absolute Maximum bound") - - @model_validator(mode="after") - def validate_bounds_hierarchy(self): - """Validate the complete hierarchy of bounds.""" - bounds = [ - ("hard_min", self.hard_min), - ("soft_min", self.soft_min), - ("soft_max", self.soft_max), - ("hard_max", self.hard_max), - ] - - # Get only non-None values with their names - defined_bounds = [(name, val) for name, val in bounds if val is not None] - - # Check that all defined bounds are in ascending order - for i in range(len(defined_bounds) - 1): - curr_name, curr_val = defined_bounds[i] - next_name, next_val = defined_bounds[i + 1] - if curr_val > next_val: - raise ValueError( - f"{curr_name} ({curr_val}) must be <= {next_name} ({next_val}). " - f"Bounds must follow hierarchy: hard_min <= soft_min <= soft_max <= hard_max" - ) - - return self - - -class ConstraintMetrics(BaseModel): - """Computed metrics for constraint violations.""" - - columns_checked: int = Field(ge=0, description="Number of columns checked") - total_violations: int = Field( - ge=0, description="Total number of constraint violations" - ) - hard_min_violations: int = Field(ge=0, description="Count of values below hard_min") - soft_min_violations: int = Field(ge=0, description="Count of values below soft_min") - soft_max_violations: int = Field(ge=0, description="Count of values above soft_max") - hard_max_violations: int = Field(ge=0, description="Count of values above hard_max") - - -class OutlierMetrics(BaseModel): - """Computed Metrics for Outlier Checks""" - - columns_checked: int = Field(ge=0, description="Number of columns checked") - columns_with_outliers: int = Field( - ge=0, description="Total number of columns with outlier values" - ) - total_outliers: int = Field(ge=0, description="Total number of outliers flagged") - enumerators_with_outliers: int = Field( - ge=0, description="Total number of outliers flagged" - ) - - -class OutlierStatistics(BaseModel): - """Complete statistical summary for outlier detection.""" - - count: int = Field(ge=0, description="Number of non-null values") - min_value: float - max_value: float - mean: float - median: float - sd: float | None - iqr: float | None - lower_bound: float | None - upper_bound: float | None - - class Config: - """Pydantic config.""" - - populate_by_name = True - - -class OutlierColumnConfig(BaseModel): - """Configuration for a single outlier column check.""" - - search_type: SearchType - pattern: str | None = None - outlier_cols: list[str] = Field(min_length=1) - lock_cols: bool = False - grouped_cols: bool = False - outlier_method: OutlierMethod = OutlierMethod.IQR - outlier_multiplier: float = Field(gt=0, le=10.0) - soft_min: float | None = None - soft_max: float | None = None - - @field_validator("pattern") - @classmethod - def validate_pattern(cls, v: str | None, info) -> str | None: - """Validate pattern is required for non-exact search types.""" - if info.data.get("search_type") != SearchType.EXACT and not v: - raise ValueError("Pattern is required for non-exact search types") - return v - - @field_validator("soft_max") - @classmethod - def validate_soft_bounds(cls, v: float | None, info) -> float | None: - """Validate soft_max is greater than soft_min.""" - soft_min = info.data.get("soft_min") - if v is not None and soft_min is not None and v <= soft_min: - raise ValueError("soft_max must be greater than soft_min") - return v - - -class OutlierSettings(BaseModel): - """Main configuration for outlier report.""" - - survey_key: str = Field(..., description="Column name for survey key", min_length=1) - survey_id: str | None = Field( - None, description="Column name for survey ID", min_length=1 - ) - survey_date: str | None = Field( - None, description="Column name for survey date", min_length=1 - ) - enumerator: str | None = Field( - None, description="Column name for enumerator ID", min_length=1 - ) - team: str | None = Field(None, description="Column name for team", min_length=1) - - -# ============================================================================= -# Utility Functions -# ============================================================================= - - -def _ensure_list(value: Any) -> list: - """Ensure value is a list. - - Parameters - ---------- - value : Any - Value to convert. - - Returns - ------- - list - Value as a list. - """ - if isinstance(value, str): - return [value] - if isinstance(value, list): - return value - return list(value) - - -def _build_include_cols( - survey_key: str, - survey_id: str | None, - survey_date: str | None, - enumerator: str | None, - team: str | None, -) -> list[str]: - """Build list of columns to include in output. - - Parameters - ---------- - survey_key : str - Survey key column. - survey_id : str | None - Survey ID column. - survey_date : str | None - Survey date column. - enumerator : str | None - Enumerator column. - team : str | None - Team column. - - Returns - ------- - list[str] - Deduplicated list of columns to include. - """ - include_cols = [] - for col in [survey_key, survey_id, survey_date, enumerator, team]: - if col and col not in include_cols: - include_cols.append(col) - return include_cols - - -# ============================================================================= -# Settings and Configuration Functions -# ============================================================================= - - -def load_default_settings( - settings_file: str, config: OutlierSettings -) -> OutlierSettings: - """Load the default settings for the outliers report. - - Parameters - ---------- - settings_file : str - The settings file to load. - config : OutlierSettings - Default configuration. - - Returns - ------- - OutlierSettings - Merged settings. - """ - # Load saved settings - saved_settings = load_check_settings(settings_file, TAB_NAME) - - default_settings: dict = dict(config) - default_settings.update(saved_settings) - - # Merge with defaults - return OutlierSettings(**default_settings) - - -@st.cache_data -def expand_col_names( - col_names: list[str], pattern: str, search_type: str = "exact" -) -> list[str]: - """Expand column names based on a pattern and search type. - - Parameters - ---------- - col_names : list[str] - List of column names to search in. - pattern : str - Pattern to match against column names. - search_type : str, default="exact" - Type of search to perform. - - Returns - ------- - list[str] - List of column names that match the pattern. - - Raises - ------ - TypeError - If input types are invalid. - ValueError - If search_type is not supported. - """ - if not isinstance(col_names, list): - raise TypeError("col_names must be a list of column names.") - if not pattern: - raise TypeError("pattern must be provided.") - if not isinstance(pattern, str): - raise TypeError("pattern must be a string.") - - search_funcs = { - SearchType.EXACT.value: lambda col: col == pattern, - SearchType.STARTSWITH.value: lambda col: col.startswith(pattern), - SearchType.ENDSWITH.value: lambda col: col.endswith(pattern), - SearchType.CONTAINS.value: lambda col: pattern in col, - SearchType.REGEX.value: lambda col: re.match(pattern, col), - } - - if search_type not in search_funcs: - valid_types = ", ".join(search_funcs.keys()) - raise ValueError( - f"Invalid search_type '{search_type}'. Choose from: {valid_types}." - ) - - return [col for col in col_names if search_funcs[search_type](col)] - - -def _should_expand_row(row: dict) -> bool: - """Check if a configuration row should have its columns expanded. - - Parameters - ---------- - row : dict - Configuration row to check (from Polars iter_rows). - - Returns - ------- - bool - True if row should be expanded. - """ - return row["search_type"] != SearchType.EXACT.value and not row.get("locked", False) - - -def _update_unlocked_cols( - column_config: pl.DataFrame, - col_names: list[str], -) -> pl.DataFrame: - """Update column names for unlocked rows in column configuration. - - Parameters - ---------- - column_config : pl.DataFrame - Polars DataFrame containing outlier column configuration. - col_names : list[str] - List of available column names. - - Returns - ------- - pl.DataFrame - Updated column configuration with expanded column names. - - Raises - ------ - ValueError - If essential columns are missing or pattern is invalid. - """ - required_columns = {"search_type", "pattern", "column_name", "locked"} - missing_columns = required_columns - set(column_config.columns) - if missing_columns: - raise ValueError( - f"Missing required columns in column_config: {', '.join(missing_columns)}" - ) - - updated_rows = [] - for row in column_config.iter_rows(named=True): - if _should_expand_row(row): - expanded_cols = expand_col_names( - col_names=col_names, - pattern=row["pattern"], - search_type=row["search_type"], - ) - row["outlier_cols"] = expanded_cols - updated_rows.append(row) - - return pl.DataFrame(updated_rows) - - -def update_unlocked_cols( - outlier_settings: pl.DataFrame, col_names: list[str] -) -> pl.DataFrame: - """Update column names for unlocked rows in outlier settings. - - Public API wrapper for backward compatibility. - - Parameters - ---------- - outlier_settings : pl.DataFrame - Polars DataFrame containing outlier settings or column configuration. - col_names : list[str] - List of available column names. - - Returns - ------- - pl.DataFrame - Updated settings with expanded column names. - - Raises - ------ - ValueError - If essential columns are missing or pattern is invalid. - """ - return _update_unlocked_cols(outlier_settings, col_names) - - -# ============================================================================= -# Statistical Computation Functions (Polars-optimized) -# ============================================================================= - - -def _compute_iqr_bounds(series: pl.Series, multiplier: float) -> OutlierBounds: - """Compute IQR-based outlier bounds. - - Parameters - ---------- - series : pl.Series - Numeric series to compute bounds for. - multiplier : float - IQR multiplier (typically 1.5). - - Returns - ------- - OutlierBounds - Lower and upper bounds. - """ - q1 = series.quantile(0.25) - q3 = series.quantile(0.75) - iqr = q3 - q1 - lower_bound = q1 - (multiplier * iqr) - upper_bound = q3 + (multiplier * iqr) - return OutlierBounds(lower_bound=lower_bound, upper_bound=upper_bound) - - -def _compute_sd_bounds(series: pl.Series, multiplier: float) -> OutlierBounds: - """Compute standard deviation-based outlier bounds. - - Parameters - ---------- - series : pl.Series - Numeric series to compute bounds for. - multiplier : float - SD multiplier (typically 3.0). - - Returns - ------- - OutlierBounds - Lower and upper bounds. - """ - mean = series.mean() - std = series.std() - lower_bound = mean - (multiplier * std) - upper_bound = mean + (multiplier * std) - return OutlierBounds(lower_bound=lower_bound, upper_bound=upper_bound) - - -def compute_outlier_stats_polars( - series: pl.Series, - outlier_type: str | None, - multiplier: float | None, -) -> OutlierStatistics: - """Compute outlier statistics using Polars for better performance. - - Parameters - ---------- - series : pl.Series - The Series to compute statistics for. - outlier_type : str | None - The type of outlier detection method to use. - multiplier : float | None - The multiplier to use for outlier detection. - - Returns - ------- - OutlierStatistics - Pydantic model containing computed statistics. - - Raises - ------ - ValueError - If series is empty or parameters are invalid. - """ - if series.len() == 0: - raise ValueError("The Series is empty.") - - valid_types = [None, OutlierMethod.IQR.value, OutlierMethod.SD.value] - if outlier_type not in valid_types: - raise ValueError( - f"Invalid outlier type. Use 'IQR' or 'SD', got: {outlier_type}" - ) - - if multiplier is not None and multiplier <= 0: - raise ValueError("Multiplier must be a positive number.") - - # remove nulls for accurate stats - series = series.drop_nulls() - - series = safe_to_numeric(series) - - # return empty stats if no non-null values - if series.len() == 0: - return OutlierStatistics( - count=0, - min_value=float("nan"), - max_value=float("nan"), - mean=float("nan"), - median=float("nan"), - sd=float("nan"), - iqr=float("nan"), - lower_bound=float("nan"), - upper_bound=float("nan"), - ) - - # Compute basic statistics - count = series.len() - series.null_count() - min_value = series.min() - max_value = series.max() - mean = series.mean() - median = series.median() - sd = series.std() - q1 = series.quantile(0.25) - q3 = series.quantile(0.75) - iqr = q3 - q1 - - # Compute bounds based on method - if outlier_type == OutlierMethod.SD.value: - multiplier = multiplier or OutlierMultipliers.SD.value - bounds = _compute_sd_bounds(series, multiplier) - else: # Default to IQR - multiplier = multiplier or OutlierMultipliers.IQR.value - bounds = _compute_iqr_bounds(series, multiplier) - - return OutlierStatistics( - count=count, - min_value=min_value, - max_value=max_value, - mean=mean, - median=median, - sd=sd, - iqr=iqr, - lower_bound=bounds.lower_bound, - upper_bound=bounds.upper_bound, - ) - - -@st.cache_data(hash_funcs={pl.DataFrame: lambda df: str(df.schema)}) -def stack_outlier_columns(df: pl.DataFrame, col_names: list[str]) -> pl.Series: - """Stack specified columns of a DataFrame into a single Series. - - Parameters - ---------- - df : pl.DataFrame - The DataFrame containing the data. - col_names : list[str] - List of column names to stack. - - Returns - ------- - pl.Series - A Series containing the stacked values. - - Raises - ------ - ValueError - If DataFrame is empty or columns don't exist. - """ - if df.is_empty(): - raise ValueError("The DataFrame is empty.") - - for col in col_names: - if col not in df.columns: - raise ValueError(f"Column '{col}' does not exist in the DataFrame.") - - # Check and convert columns to numeric if needed - for col in col_names: - dtype = df[col].dtype - if dtype not in [ - pl.Int8, - pl.Int16, - pl.Int32, - pl.Int64, - pl.UInt8, - pl.UInt16, - pl.UInt32, - pl.UInt64, - pl.Float32, - pl.Float64, - ]: - try: - df = df.with_columns(pl.col(col).cast(pl.Float64)) - except Exception: - raise ValueError( - f"Column '{col}' cannot be converted to numeric type." - ) from None - - # Stack the columns - melt/unpivot in Polars - stacked_values = ( - df.select(col_names) - .unpivot() - .get_column("value") - .drop_nulls() # Remove null values like pandas stack() does - ) - - return stacked_values - - -def _build_outlier_expression( - col: str, - lower_bound: float, - upper_bound: float, -) -> pl.Expr: - """Build Polars expression for outlier flagging. - - Parameters - ---------- - col : str - Column name. - lower_bound : float - Statistical lower bound. - upper_bound : float - Statistical upper bound. - - Returns - ------- - pl.Expr - Polars expression for outlier detection. - """ - outlier_expr = ( - pl.when(pl.col(col) < lower_bound) - .then(pl.lit(f"Value is below lower bound {lower_bound:.2f}")) - .when(pl.col(col) > upper_bound) - .then(pl.lit(f"Value is above upper bound {upper_bound:.2f}")) - ) - - return outlier_expr.otherwise(pl.lit("no outlier")) - - -def _add_statistics_columns( - col_df: pl.DataFrame, - outlier_stats: OutlierStatistics, - outlier_method: str, - outlier_multiplier: float, - col_name: str, -) -> pl.DataFrame: - """Add statistics columns to the outlier dataframe. - - Parameters - ---------- - col_df : pl.DataFrame - DataFrame to add columns to. - outlier_stats : OutlierStatistics - Computed statistics. - outlier_method : str - Detection method used. - outlier_multiplier : float - Multiplier used. - col_name : str - Name of the column being analyzed. - - Returns - ------- - pl.DataFrame - DataFrame with added statistics columns. - """ - return col_df.with_columns( - [ - pl.lit(outlier_stats.min_value, dtype=pl.Float64).alias("min_value"), - pl.lit(outlier_stats.max_value, dtype=pl.Float64).alias("max_value"), - pl.lit(outlier_stats.mean, dtype=pl.Float64).alias("mean"), - pl.lit(outlier_stats.median, dtype=pl.Float64).alias("median"), - pl.lit(outlier_stats.sd, dtype=pl.Float64).alias("std"), - pl.lit(outlier_stats.iqr, dtype=pl.Float64).alias("iqr"), - pl.lit(outlier_stats.lower_bound, dtype=pl.Float64).alias("lower_bound"), - pl.lit(outlier_stats.upper_bound, dtype=pl.Float64).alias("upper_bound"), - pl.lit(outlier_method).alias("outlier_method"), - pl.lit(outlier_multiplier, dtype=pl.Float64).alias("outlier_multiplier"), - pl.lit(col_name).alias("column name"), - ] - ) - - -def _process_single_column_outliers( - df_polars: pl.DataFrame, - col: str, - survey_key: str, - outlier_stats: OutlierStatistics, - outlier_method: str, - outlier_multiplier: float, - min_threshold: int, - non_null_count: int, -) -> pl.DataFrame: - """Process outliers for a single column using Polars. - - Parameters - ---------- - df_polars : pl.DataFrame - Polars DataFrame containing the data. - col : str - Column name to process. - survey_key : str - Survey key column name. - outlier_stats : OutlierStatistics - Pre-computed statistics. - outlier_method : str - Outlier detection method. - outlier_multiplier : float - Multiplier for detection. - min_threshold : int - Minimum sample size threshold. - non_null_count : int - Number of non-null values. - - Returns - ------- - pl.DataFrame - DataFrame with outlier information for the column. - """ - # Select relevant columns - col_df = df_polars.select([survey_key, col]) - col_df = safe_to_numeric(col_df, col) - - # Add outlier reason - if non_null_count < min_threshold: - col_df = col_df.with_columns(pl.lit("no outlier").alias("outlier reason")) - else: - # Vectorized outlier flagging - outlier_expr = _build_outlier_expression( - col, - outlier_stats.lower_bound, - outlier_stats.upper_bound, - ) - col_df = col_df.with_columns(outlier_expr.alias("outlier reason")) - - # Add statistics columns - col_df = _add_statistics_columns( - col_df, - outlier_stats, - outlier_method, - outlier_multiplier, - col, - ) - - # Rename and reorder - col_df = col_df.rename({col: "column value"}) - col_df = col_df.select( - [ - survey_key, - "column name", - "column value", - "min_value", - "max_value", - "mean", - "median", - "std", - "iqr", - "lower_bound", - "upper_bound", - "outlier reason", - "outlier_method", - "outlier_multiplier", - ] - ) - - return col_df - - -# ============================================================================= -# Outlier Detection - Main Logic -# ============================================================================= - - -def _compute_column_stats( - df_polars: pl.DataFrame, - outlier_cols: list[str], - grouped_cols: bool, - outlier_method: str, - outlier_multiplier: float, -) -> tuple[OutlierStatistics, int]: - """Compute outlier statistics for grouped columns. - - Parameters - ---------- - df_polars : pl.DataFrame - DataFrame containing the data (with survey_key and outlier columns). - outlier_cols : list[str] - List of columns to analyze. - grouped_cols : bool - Whether columns should be analyzed together. - outlier_method : str - Outlier detection method (IQR or SD). - outlier_multiplier : float - Multiplier for bounds calculation. - - Returns - ------- - tuple[OutlierStatistics, int] - Computed statistics and non-null count. - """ - if len(outlier_cols) == 1 or grouped_cols: - if len(outlier_cols) == 1: - series = df_polars[outlier_cols[0]] - else: - series = pl.concat([df_polars[col] for col in outlier_cols]) - - non_null_count = series.len() - series.null_count() - stats = compute_outlier_stats_polars( - series, - outlier_type=outlier_method, - multiplier=outlier_multiplier, - ) - return stats, non_null_count - - # For non-grouped multiple columns, return None to signal per-column computation - return None, 0 - - -def _compute_single_column_stats( - df_polars: pl.DataFrame, - col: str, - outlier_method: str, - outlier_multiplier: float, -) -> tuple[OutlierStatistics, int]: - """Compute outlier statistics for a single column. - - Parameters - ---------- - df_polars : pl.DataFrame - DataFrame containing the data. - col : str - Column name to analyze. - outlier_method : str - Outlier detection method. - outlier_multiplier : float - Multiplier for bounds calculation. - - Returns - ------- - tuple[OutlierStatistics, int] - Computed statistics and non-null count. - """ - non_null_count = df_polars.height - df_polars[col].null_count() - stats = compute_outlier_stats_polars( - df_polars[col], - outlier_type=outlier_method, - multiplier=outlier_multiplier, - ) - return stats, non_null_count - - -def _merge_outlier_results( - outlier_results_list: list[pl.DataFrame], - admin_data_polars: pl.DataFrame, - survey_key: str, -) -> pl.DataFrame: - """Merge outlier results with admin data. - - Parameters - ---------- - outlier_results_list : list[pl.DataFrame] - List of outlier result DataFrames. - admin_data_polars : pl.DataFrame - Admin data DataFrame. - survey_key : str - Survey key column name. - - Returns - ------- - pl.DataFrame - Merged results or empty DataFrame if no results. - """ - if not outlier_results_list: - return pl.DataFrame() - - outlier_results_polars = pl.concat(outlier_results_list) - - if admin_data_polars.is_empty(): - return outlier_results_polars - - return admin_data_polars.join( - outlier_results_polars, - on=survey_key, - how="left", - ) - - -def _process_outlier_configs( - data: pl.DataFrame, - column_config: pl.DataFrame, - survey_key: str, -) -> list[pl.DataFrame]: - """Process all outlier configurations and return results. - - Parameters - ---------- - data : pl.DataFrame - DataFrame containing the survey data. - column_config : pl.DataFrame - DataFrame containing the outlier column configurations. - survey_key : str - Survey key column name. - - Returns - ------- - list[pl.DataFrame] - List of outlier result DataFrames. - """ - outlier_results_list = [] - - for row in column_config.iter_rows(named=True): - if not row.get("outlier_enabled", False): - continue - - results = _process_single_config(data, row, survey_key) - outlier_results_list.extend(results) - - return outlier_results_list - - -def _process_single_config( - data: pl.DataFrame, - row: dict, - survey_key: str, -) -> list[pl.DataFrame]: - """Process a single outlier configuration row. - - Parameters - ---------- - data : pl.DataFrame - DataFrame containing the survey data. - row : dict - Configuration row from column_config. - survey_key : str - Survey key column name. - - Returns - ------- - list[pl.DataFrame] - List of outlier results for this configuration. - """ - # Extract settings with defaults - outlier_cols = _ensure_list(row.get("column_name", [])) - grouped_cols = row.get("grouped_columns", False) - outlier_method = row.get("outlier_method", OutlierMethod.IQR.value) - threshold = row.get("outlier_threshold", OutlierThresholds.IQR.value) - outlier_multiplier = row.get("outlier_multiplier", OutlierMultipliers.IQR.value) - - # Create subset - outlier_df_polars = data.select([survey_key, *outlier_cols]) - - # Compute shared stats for single column or grouped columns - shared_stats, shared_count = _compute_column_stats( - outlier_df_polars, - outlier_cols, - grouped_cols, - outlier_method, - outlier_multiplier, - ) - - # Process each column - results = [] - for col in outlier_cols: - if shared_stats is not None: - outlier_stats, non_null_count = shared_stats, shared_count - else: - outlier_stats, non_null_count = _compute_single_column_stats( - outlier_df_polars, col, outlier_method, outlier_multiplier - ) - - col_result = _process_single_column_outliers( - df_polars=outlier_df_polars, - col=col, - survey_key=survey_key, - outlier_stats=outlier_stats, - outlier_method=outlier_method, - outlier_multiplier=outlier_multiplier, - min_threshold=threshold, - non_null_count=non_null_count, - ) - results.append(col_result) - - return results - - -def compute_outlier_output( - data: pl.DataFrame, - outlier_settings: dict, - column_config: pl.DataFrame, -) -> pl.DataFrame: - """Detect outliers in DataFrame based on settings (Polars-optimized). - - Parameters - ---------- - data : pl.DataFrame - DataFrame containing the survey data. - outlier_settings : dict - Outlier settings configuration. - column_config : pl.DataFrame - DataFrame containing the outlier column configurations. - - Returns - ------- - pl.DataFrame - DataFrame containing the outlier summary. - - Raises - ------ - ValueError - If DataFrame is empty. - """ - if data.is_empty(): - raise ValueError("The DataFrame is empty. Please provide a valid DataFrame.") - - # Build include columns list - survey_key = outlier_settings.survey_key - include_cols = _build_include_cols( - survey_key, - outlier_settings.survey_id, - outlier_settings.survey_date, - outlier_settings.enumerator, - outlier_settings.team, - ) - admin_data_polars = data.select(include_cols) - - # Process outlier settings - outlier_results_list = _process_outlier_configs(data, column_config, survey_key) - - return _merge_outlier_results(outlier_results_list, admin_data_polars, survey_key) - - -# ============================================================================= -# Constraint Violations - Main Logic -# ============================================================================= - - -def compute_constraint_violations( - data: pl.DataFrame, - settings: OutlierSettings, - column_config: pl.DataFrame, -) -> pl.DataFrame: - """Compute constraint violations for outlier detection. - - Parameters - ---------- - data : pl.DataFrame - DataFrame containing the survey data. - settings : OutlierSettings - Outlier settings configuration. - column_config : pl.DataFrame - DataFrame containing the outlier column configurations. - - Returns - ------- - pl.DataFrame - DataFrame containing constraint violation information. - """ - survey_key = settings.survey_key - - violation_results = pl.DataFrame() - - for row in column_config.iter_rows(named=True): - outlier_cols = _ensure_list(row.get("column_name", [])) - hard_min = row.get("hard_min", None) - soft_min = row.get("soft_min", None) - soft_max = row.get("soft_max", None) - hard_max = row.get("hard_max", None) - - # skip if no bounds are set - if all(bound is None for bound in [hard_min, soft_min, soft_max, hard_max]): - continue - - for col in outlier_cols: - col_df = data.select([survey_key, col]) - - violation_expr = ( - pl.when((hard_min is not None) & (pl.col(col) < hard_min)) - .then(pl.lit(f"Value is below hard minimum {hard_min}")) - .when((soft_min is not None) & (pl.col(col) < soft_min)) - .then(pl.lit(f"Value is below soft minimum {soft_min}")) - .when((soft_max is not None) & (pl.col(col) > soft_max)) - .then(pl.lit(f"Value is above soft maximum {soft_max}")) - .when((hard_max is not None) & (pl.col(col) > hard_max)) - .then(pl.lit(f"Value is above hard maximum {hard_max}")) - ) - - col_df = safe_to_numeric(col_df, col) - - col_df = col_df.with_columns( - violation_expr.otherwise(pl.lit("no violation")).alias( - "violation reason" - ) - ) - - # add hard and soft bounds columns - for bound_name, bound_value in [ - ("hard_min", hard_min), - ("soft_min", soft_min), - ("soft_max", soft_max), - ("hard_max", hard_max), - ]: - col_df = col_df.with_columns(pl.lit(bound_value).alias(bound_name)) - - col_df = col_df.rename({col: "column value"}) - col_df = col_df.with_columns(pl.lit(col).alias("column name")).select( - [ - survey_key, - "column name", - "column value", - "hard_min", - "soft_min", - "soft_max", - "hard_max", - "violation reason", - ] - ) - - violation_results = ( - violation_results.vstack(col_df) - if not violation_results.is_empty() - else col_df - ) - - return violation_results - - -# ============================================================================= -# Metrics Computation - Analytics -# ============================================================================= - - -def _compute_constraint_metrics(violation_data: pl.DataFrame) -> ConstraintMetrics: - """Compute metrics related to constraint violations. - - Parameters - ---------- - violation_data : pl.DataFrame - DataFrame containing constraint violation data. - - Returns - ------- - ConstraintMetrics - Pydantic model containing computed metrics. - """ - columns_checked = violation_data.select("column name").n_unique() - total_violations = violation_data.filter( - pl.col("violation reason") != "no violation" - ).height - - hard_min_violations = violation_data.filter( - pl.col("violation reason").str.contains("below hard minimum") - ).height - soft_min_violations = violation_data.filter( - pl.col("violation reason").str.contains("below soft minimum") - ).height - soft_max_violations = violation_data.filter( - pl.col("violation reason").str.contains("above soft maximum") - ).height - hard_max_violations = violation_data.filter( - pl.col("violation reason").str.contains("above hard maximum") - ).height - - return ConstraintMetrics( - columns_checked=columns_checked, - total_violations=total_violations, - hard_min_violations=hard_min_violations, - soft_min_violations=soft_min_violations, - soft_max_violations=soft_max_violations, - hard_max_violations=hard_max_violations, - ) - - -def _compute_outlier_metrics( - outliers_data: pl.DataFrame, - enumerator: str | None, -) -> OutlierMetrics: - """Compute outlier metrics. - - Parameters - ---------- - outliers_data : pl.DataFrame - DataFrame containing outlier data. - enumerator : str | None - Enumerator column name. - - Returns - ------- - OutlierMetrics - Pydantic model containing computed metrics. - """ - columns_checked = outliers_data.select("column name").n_unique() - columns_with_outliers = ( - outliers_data.filter(pl.col("outlier reason") != "no outlier") - .select("column name") - .n_unique() - ) - total_outliers = outliers_data.filter( - pl.col("outlier reason") != "no outlier" - ).height - if enumerator: - enumerators_with_outliers = ( - outliers_data.filter(pl.col("outlier reason") != "no outlier") - .select(enumerator) - .n_unique() - ) - else: - enumerators_with_outliers = 0 - - return OutlierMetrics( - columns_checked=columns_checked, - columns_with_outliers=columns_with_outliers, - total_outliers=total_outliers, - enumerators_with_outliers=enumerators_with_outliers, - ) - - -def compute_column_outlier_summary( - outlier_data: pl.DataFrame, survey_key: str -) -> pl.DataFrame: - """Compute a summary of outliers for each column using Polars. - - Parameters - ---------- - outlier_data : pl.DataFrame - Polars DataFrame containing outlier data. - survey_key : str - Survey key column name. - - Returns - ------- - pl.DataFrame - Summary DataFrame with outlier counts per column. - """ - if outlier_data.is_empty(): - return pl.DataFrame() - - # Remove duplicates - outlier_summary = outlier_data.unique(subset=["column name", survey_key]) - - # Count occurrences per column - col_counts = outlier_summary.group_by("column name").agg(pl.count().alias("count")) - - # Join counts back - outlier_summary = outlier_summary.join(col_counts, on="column name", how="left") - - # Flag outliers - outlier_summary = outlier_summary.with_columns( - pl.when(pl.col("outlier reason") != "no outlier") - .then(pl.lit(1)) - .otherwise(pl.lit(0)) - .alias("flagged as outlier") - ) - - # Count outliers per column - outlier_counts = outlier_summary.group_by("column name").agg( - pl.col("flagged as outlier").sum().alias("outlier count") - ) - - # Merge outlier counts - outlier_summary = outlier_summary.join(outlier_counts, on="column name", how="left") - - # Select and order columns - outlier_summary = outlier_summary.select( - [ - "column name", - "count", - "outlier count", - "min_value", - "max_value", - "mean", - "median", - "std", - "iqr", - "lower_bound", - "upper_bound", - ] - ) - - return outlier_summary.unique(subset=["column name"]) - - -def get_outlier_cols(outlier_settings: pd.DataFrame) -> list[str]: - """Get list of outlier columns from settings DataFrame. - - Parameters - ---------- - outlier_settings : pd.DataFrame - DataFrame containing outlier settings. - - Returns - ------- - list[str] - List of column names to check for outliers. - """ - cols = [] - for i in range(len(outlier_settings)): - col = outlier_settings.iloc[i]["outlier_cols"] - if isinstance(col, np.ndarray): - cols.append(col[0]) - elif isinstance(col, list): - cols.extend(col) - - return cols - - -# ============================================================================= -# Visualization Functions -# ============================================================================= - - -@st.cache_data -def _create_box_plot(data: pd.Series, title: str) -> go.Figure: - """Create a box plot using plotly. - - Parameters - ---------- - data : pd.Series - Data series to plot. - title : str - Title for the plot. - - Returns - ------- - go.Figure - Plotly figure object. - """ - return go.Figure( - data=go.Box( - y=data, - boxpoints="outliers", - marker_color="darkblue", - line_color="black", - fillcolor="lightblue", - opacity=0.6, - x0=title, - ) - ) - - -@st.cache_data -def _create_descriptive_stats(column_data: pl.DataFrame) -> pl.DataFrame: - """Create descriptive statistics table. - - Parameters - ---------- - column_data : pl.DataFrame - Column data to analyze. - - Returns - ------- - pl.DataFrame - Descriptive statistics table. - """ - table = column_data.describe() - table.columns = ["statistic", "value"] - # rename statistics - stat_rename = { - "count": "Number of Values", - "null_count": "Number of Missing Values", - "mean": "Mean", - "std": "Standard Deviation", - "min": "Minimum Value", - "25%": "25th Percentile (Q1)", - "50%": "Median (Q2)", - "75%": "75th Percentile (Q3)", - "max": "Maximum Value", - } - - table = table.with_columns( - pl.col("statistic").replace(stat_rename).alias("statistic") - ) - - return table - - -# ============================================================================= -# Streamlit UI - Metrics Display -# ============================================================================= - - -def _render_constraint_metrics( - violation_data: pl.DataFrame, -) -> None: - """Render constraint violation metrics using Streamlit. - - Parameters - ---------- - violation_data : pl.DataFrame - DataFrame containing constraint violation data. - """ - metrics: ConstraintMetrics = _compute_constraint_metrics(violation_data) - - _, _, uc3, uc4 = st.columns(4) - with uc3, st.container(border=True): - st.metric( - label="Number of columns checked", - value=f"{metrics.columns_checked:,}", - help="Number of columns checked for constraint violations", - ) - with uc4, st.container(border=True): - st.metric( - label="Total Violations", - value=f"{metrics.total_violations:,}", - help="Total number of constraint violations detected", - ) - - lc1, lc2, lc3, lc4 = st.columns(4, border=True) - lc1.metric( - label="Hard Min Violations", - value=f"{metrics.hard_min_violations:,}", - help="Number of violations below hard minimum", - ) - lc2.metric( - label="Soft Min Violations", - value=f"{metrics.soft_min_violations:,}", - help="Number of violations below soft minimum", - ) - lc3.metric( - label="Soft Max Violations", - value=f"{metrics.soft_max_violations:,}", - help="Number of violations above soft maximum", - ) - lc4.metric( - label="Hard Max Violations", - value=f"{metrics.hard_max_violations:,}", - help="Number of violations above hard maximum", - ) - - -def _render_outlier_metrics( - outliers_data: pl.DataFrame, - settings: OutlierSettings, -) -> None: - """Render outlier metrics using Streamlit. - - Parameters - ---------- - outliers_data : pl.DataFrame - DataFrame containing outlier data. - settings : OutlierSettings - Outlier settings configuration. - """ - metrics: OutlierMetrics = _compute_outlier_metrics( - outliers_data, settings.enumerator - ) - - uc1, uc2, uc3, uc4 = st.columns(4, border=True) - uc1.metric( - label="Number of columns checked", - value=f"{metrics.columns_checked:,}", - help="Number of columns checked for outliers", - ) - uc2.metric( - label="Columns with Outliers", - value=f"{metrics.columns_with_outliers:,}", - help="Number of columns that have outliers detected", - ) - uc3.metric( - label="Total Outliers", - value=f"{metrics.total_outliers:,}", - help="Total number of outliers detected", - ) - if settings.enumerator: - uc4.metric( - label="Enumerators with Outliers", - value=f"{metrics.enumerators_with_outliers:,}", - help="Number of unique enumerators with outliers detected", - ) - - -# ============================================================================= -# Streamlit UI - Table Display -# ============================================================================= - - -def _render_constraint_violations_table( - data: pl.DataFrame, - violation_data: pl.DataFrame, - settings: OutlierSettings, - setting_file: str, -) -> None: - """Render constraint violations table using Streamlit. - - Parameters - ---------- - data : pl.DataFrame - Original survey data. - violation_data : pl.DataFrame - DataFrame containing constraint violation data. - settings : OutlierSettings - Outlier settings configuration. - setting_file : str - Path to settings file. - """ - if violation_data.is_empty(): - st.info("No constraint violations detected.") - return - - all_columns = data.columns - - include_cols = _build_include_cols( - survey_key=settings.survey_key, - survey_id=settings.survey_id, - survey_date=settings.survey_date, - enumerator=settings.enumerator, - team=settings.team, - ) - - display_options = [col for col in all_columns if col not in include_cols] - - with st.expander(":material/clarify: Show more columns in report", expanded=False): - st.info( - "Select additional columns to include in the constraint violations report." - ) - - # get saved settings - saved_settings = load_check_settings(setting_file, TAB_NAME) - cols = saved_settings.get("constraint_display_cols", []) - default_constraint_display_cols = [ - col for col in cols if col in display_options - ] - - constraint_display_cols = st.multiselect( - label="Select columns to display", - options=display_options, - default=default_constraint_display_cols, - key="constraint_violation_display_cols", - on_change=trigger_save, - kwargs={"state_name": TAB_NAME + "_constraint_display_cols"}, - ) - save_check_settings( - setting_file, TAB_NAME, {"constraint_display_cols": constraint_display_cols} - ) - - if constraint_display_cols: - include_cols.extend(constraint_display_cols) - - # select columns to display from data - display_df = data.select(include_cols) - # sanitize violation_data to avoid column name conflicts - violation_df = sanitize_df_for_join( - main_df=display_df, - join_df=violation_data, - join_key=settings.survey_key, - ) - - display_df = display_df.join( - violation_df, - on=settings.survey_key, - how="inner", - ) - - # show only rows with violations - violations_df = display_df.filter(pl.col("violation reason") != "no violation") - - # add violation type column ie. "Soft Min", "Soft Max", "Hard Min", "Hard Max" - violation_type_expr = ( - pl.when(pl.col("violation reason").str.contains("below hard minimum")) - .then(pl.lit("Hard Min")) - .when(pl.col("violation reason").str.contains("below soft minimum")) - .then(pl.lit("Soft Min")) - .when(pl.col("violation reason").str.contains("above soft maximum")) - .then(pl.lit("Soft Max")) - .when(pl.col("violation reason").str.contains("above hard maximum")) - .then(pl.lit("Hard Max")) - .otherwise(pl.lit("Unknown")) - ) - - violations_df = violations_df.with_columns( - violation_type_expr.alias("violation type") - ) - - st.dataframe(violations_df) - - -def _render_outlier_table( - data: pl.DataFrame, - outliers_data: pl.DataFrame, - settings: OutlierSettings, - setting_file: str, -) -> None: - """Render outlier data table using Streamlit. - - Parameters - ---------- - data : pl.DataFrame - Original survey data. - outliers_data : pl.DataFrame - DataFrame containing outlier data. - settings : OutlierSettings - Outlier settings configuration. - setting_file : str - Path to settings file. - """ - if outliers_data.is_empty(): - st.info("No outliers detected in the selected columns.") - return - - all_columns = data.columns - - include_cols = _build_include_cols( - survey_key=settings.survey_key, - survey_id=settings.survey_id, - survey_date=settings.survey_date, - enumerator=settings.enumerator, - team=settings.team, - ) - - display_options = [col for col in all_columns if col not in include_cols] - - # get saved settings - saved_settings = load_check_settings(setting_file, TAB_NAME) - cols = saved_settings.get("outlier_display_cols", []) - default_outlier_display_cols = [col for col in cols if col in display_options] - - with st.expander(":material/clarify: Show more columns in report", expanded=False): - st.info("Select additional columns to include in the outlier report.") - outlier_display_cols = st.multiselect( - label="Select columns to display", - default=default_outlier_display_cols, - options=display_options, - key="outlier_display_cols", - on_change=trigger_save, - kwargs={"state_name": TAB_NAME + "_outlier_display_cols"}, - ) - save_check_settings( - setting_file, TAB_NAME, {"outlier_display_cols": outlier_display_cols} - ) - - if outlier_display_cols: - include_cols.extend(outlier_display_cols) - - # select columns to display from data - display_df = data.select(include_cols) - outliers_df = sanitize_df_for_join(display_df, outliers_data, settings.survey_key) - display_df = display_df.join( - outliers_df, - on=settings.survey_key, - how="inner", - ) - - # show only rows with outliers - outlier_show_df = display_df.filter(pl.col("outlier reason") != "no outlier") - - st.dataframe(outlier_show_df) - - -def _render_outlier_column_inspection( - data: pl.DataFrame, - outliers_data: pl.DataFrame, - settings: OutlierSettings, - setting_file: str, -) -> None: - """Inspect outlier columns in the DataFrame. - - Parameters - ---------- - data : pl.DataFrame - DataFrame containing the survey data. - outliers_data : pl.DataFrame - DataFrame containing outlier detection results. - settings : OutlierSettings - Outlier settings configuration. - setting_file : str - Path to settings file. - """ - if outliers_data.is_empty(): - st.info( - "No outlier columns selected. Please select outlier columns to inspect." - ) - return - - all_columns = data.columns - - include_cols = _build_include_cols( - survey_key=settings.survey_key, - survey_id=settings.survey_id, - survey_date=settings.survey_date, - enumerator=settings.enumerator, - team=settings.team, - ) - - # list of outlier columns checked - columns_checked_list = ( - outliers_data.select("column name").unique().to_series().to_list() - ) - - ic1, _ = st.columns([0.2, 0.8]) - - with ic1: - # get saved settings - saved_settings = load_check_settings(setting_file, TAB_NAME) - default_selected_col = saved_settings.get("selected_col", None) - default_selected_col_index = ( - columns_checked_list.index(default_selected_col) - if default_selected_col and default_selected_col in columns_checked_list - else None - ) - selected_col = st.selectbox( - label="Select outlier columns to inspect", - options=columns_checked_list, - index=default_selected_col_index, - key="outlier_inspect_col", - help="Select the outlier columns to inspect. " - "You can only select one column at a time.", - on_change=trigger_save, - kwargs={"state_name": TAB_NAME + "_selected_col"}, - ) - save_check_settings(setting_file, TAB_NAME, {"selected_col": selected_col}) - - if not selected_col: - st.info("Select an outlier column to inspect.") - return - - if selected_col not in data.columns: - raise ValueError( - f"Selected column '{selected_col}' is not present in the data. " - "Please select a valid column." - ) - else: - include_cols.append(selected_col) - - # create a subset of the data - column_data = data.select([selected_col]) - - st.subheader(f"Details/Distribution for {selected_col} values") - dc1, _, dc3 = st.columns([0.3, 0.1, 0.6]) - with dc1: - desc_stats = _create_descriptive_stats(column_data) - st.dataframe(desc_stats) - - with dc3: - box_plot = _create_box_plot( - data=column_data[selected_col].to_pandas(), - title=selected_col, - ) - st.plotly_chart(box_plot, width="stretch") - - with st.expander(":material/clarify: Show more columns in report", expanded=False): - st.info( - "Select additional columns to include in the outlier inspection report." - ) - display_options = [ - col - for col in all_columns - if col not in include_cols and col != selected_col - ] - inspect_display_cols = st.multiselect( - label="Select columns to display", - options=display_options, - default=None, - help="Select the columns to display in the inspection table.", - disabled=not selected_col, - ) - - if inspect_display_cols: - include_cols.extend(inspect_display_cols) - - # select columns to display from data - display_df = data.select(include_cols) - outliers_df = sanitize_df_for_join(display_df, outliers_data, settings.survey_key) - display_df = display_df.join( - outliers_df, - on=settings.survey_key, - how="inner", - ) - - st.dataframe( - display_df, - width="stretch", - hide_index=False, - ) - - -# ============================================================================= -# Streamlit UI - Settings Configuration -# ============================================================================= - - -def _create_search_type_info(search_type_param: str) -> None: - """Display info based on the selected search type. - - Parameters - ---------- - search_type_param : str - The search type to display info for. - """ - info_messages = { - SearchType.EXACT.value: "Select columns that match the exact name. " - "You may select multiple columns.", - SearchType.STARTSWITH.value: "Select columns that start with the specified pattern. " - "You will have to enter the pattern in the input box below.", - SearchType.ENDSWITH.value: "Select columns that end with the specified pattern. " - "You will have to enter the pattern in the input box below.", - SearchType.CONTAINS.value: "Select columns that contain the specified pattern. " - "You will have to enter the pattern in the input box below.", - SearchType.REGEX.value: "Select columns that match the specified regex pattern. " - "You will have to enter the pattern in the input box below.", - } - - st.info(info_messages.get(search_type_param, "Unknown search type.")) - - -@demo_output_onboarding(TAB_NAME) -def outliers_report_settings( - settings_file: str, - config: OutlierSettings, - categorical_columns: list[str], - datetime_columns: list[str], -) -> OutlierSettings: - """Create a settings UI for outliers report configuration. - - This function creates the comprehensive Streamlit UI for configuring - outlier detection settings. Due to its complexity (UI rendering), - it maintains a higher cognitive complexity but is well-structured. - - Parameters - ---------- - settings_file : str - Path to settings file. - config : OutlierSettings - Default configuration. - categorical_columns : list[str] - List of categorical columns. - datetime_columns : list[str] - List of datetime columns. - - Returns - ------- - OutlierSettings - User-configured settings. - """ - with st.expander("settings", icon=":material/settings:"): - st.markdown("## Configure settings for outliers report") - st.write("---") - - # Load default settings - default_settings = load_default_settings(settings_file, config) - - # Survey Identifiers - with st.container(border=True): - st.markdown("#### Survey Identifiers") - si1, si2, _ = st.columns(3) - - with si1: - default_survey_key = default_settings.survey_key - default_survey_key_index = ( - categorical_columns.index(default_survey_key) - if default_survey_key and default_survey_key in categorical_columns - else None - ) - survey_key = st.selectbox( - "Survey Key", - options=categorical_columns, - key="survey_key_outliers", - help="Select the column that contains the survey key", - index=default_survey_key_index, - on_change=trigger_save, - kwargs={"state_name": TAB_NAME + "_survey_key"}, - ) - save_check_settings(settings_file, TAB_NAME, {"survey_key": survey_key}) - - with si2: - default_survey_id = default_settings.survey_id - default_survey_id_index = ( - categorical_columns.index(default_survey_id) - if default_survey_id and default_survey_id in categorical_columns - else None - ) - survey_id = st.selectbox( - "Survey ID", - options=categorical_columns, - help="Select the column that contains the survey ID", - key="survey_id_outliers", - index=default_survey_id_index, - on_change=trigger_save, - kwargs={"state_name": TAB_NAME + "_survey_id"}, - ) - save_check_settings(settings_file, TAB_NAME, {"survey_id": survey_id}) - - with st.container(border=True): - st.markdown("#### Survey Date") - - sd1, _, _ = st.columns(3) - - with sd1: - default_survey_date = default_settings.survey_date - default_survey_date_index = ( - datetime_columns.index(default_survey_date) - if default_survey_date and default_survey_date in datetime_columns - else None - ) - - survey_date = st.selectbox( - "Survey Date", - options=datetime_columns, - help="Select the column that contains the survey date", - key="survey_date_outliers", - index=default_survey_date_index, - on_change=trigger_save, - kwargs={"state_name": TAB_NAME + "_survey_date"}, - ) - save_check_settings( - settings_file, TAB_NAME, {"survey_date": survey_date} - ) - - with st.container(border=True): - st.markdown("#### Enumerator & Team") - ec1, ec2, _ = st.columns(3) - with ec1: - default_enumerator = default_settings.enumerator - default_enumerator_index = ( - categorical_columns.index(default_enumerator) - if default_enumerator and default_enumerator in categorical_columns - else None - ) - enumerator = st.selectbox( - "Enumerator ID", - options=categorical_columns, - key="enumerator_outliers", - help="Select the column that contains the enumerator ID", - index=default_enumerator_index, - on_change=trigger_save, - kwargs={"state_name": TAB_NAME + "_enumerator"}, - ) - save_check_settings(settings_file, TAB_NAME, {"enumerator": enumerator}) - - with ec2: - default_team = default_settings.team - default_team_index = ( - categorical_columns.index(default_team) - if default_team and default_team in categorical_columns - else None - ) - team = st.selectbox( - "Team ID", - options=categorical_columns, - key="team_outliers", - help="Select the column that contains the team ID", - index=default_team_index, - on_change=trigger_save, - kwargs={"state_name": TAB_NAME + "_team"}, - ) - save_check_settings(settings_file, TAB_NAME, {"team": team}) - - return OutlierSettings( - survey_key=survey_key, - survey_id=survey_id, - survey_date=survey_date, - enumerator=enumerator, - team=team, - ) - - -# ============================================================================= -# Streamlit UI - Column Configuration -# ============================================================================= - - -def _render_search_type_selection( - numeric_columns: list[str], -) -> tuple[str, str | None, list[str], bool]: - """Render search type selection UI. - - Parameters - ---------- - numeric_columns : list[str] - List of numeric columns. - - Returns - ------- - tuple[str, str | None, list[str], bool] - Search type, pattern, selected columns, and lock_cols flag. - """ - search_type_options = [e.value for e in SearchType] - search_type = st.selectbox( - label="Search type", - options=search_type_options, - index=0, - help="Select the type of search to perform on the column names.", - ) - - _create_search_type_info(search_type) - - if search_type == SearchType.EXACT.value: - outlier_cols_sel = st.multiselect( - label="Select columns to check", - options=numeric_columns, - default=None, - help="Select column or group of columns to check for outliers.", - ) - pattern, lock_cols = None, None - return search_type, pattern, outlier_cols_sel, lock_cols - else: - pattern = st.text_input( - label="Enter pattern to match column names", - placeholder="Enter pattern to match column names", - help="Enter the pattern to match column names based on the " - "selected search type.", - ) - if pattern: - outlier_cols_patt = expand_col_names( - numeric_columns, pattern, search_type=search_type - ) - else: - outlier_cols_patt = [] - - st.write( - "**Columns Selected:** ", - ", ".join(outlier_cols_patt) if outlier_cols_patt else "None", - ) - return search_type, pattern, outlier_cols_patt, None - - -def _render_column_grouping_options( - outlier_cols: list[str], search_type: str -) -> tuple[bool, bool]: - """Render column grouping and locking options. - - Parameters - ---------- - outlier_cols : list[str] - Selected outlier columns. - search_type : str - Search type used. - - Returns - ------- - tuple[bool, bool] - Group columns flag and lock columns flag. - """ - gc1, gc2 = st.columns([0.5, 0.5]) - with gc1: - group_cols = st.toggle( - label="Group columns", - key="group_outlier_cols", - help="Group selected columns together for outlier detection.", - disabled=not outlier_cols or len(outlier_cols) < 2, - ) - with gc2: - lock_cols = st.toggle( - label="Lock column selection", - key="outlier_cols_lock", - help="Lock the selected columns to prevent changes.", - disabled=not outlier_cols - or len(outlier_cols) < 2 - or search_type == SearchType.EXACT.value, - ) - return group_cols, lock_cols - - -def _render_outlier_options() -> tuple[bool, dict | None, bool]: - """Render outlier detection options UI. - - Returns - ------- - tuple[bool, dict | None, bool] - Enable outliers flag, outlier settings dict, and validation status. - """ - with st.container(border=True): - st.write("**Outlier Options:**") - enable_outliers = st.toggle( - "Enable Outlier Checks", key="enable_coutlier", value=True - ) - if enable_outliers: - oc1, oc2 = st.columns([0.5, 0.5]) - with oc1: - outlier_method = st.selectbox( - label="Select outlier detection method", - options=[e.value for e in OutlierMethod], - index=0, - help="Select the method to use for outlier detection.", - key="outlier_method", - ) - with oc2: - default_multiplier = ( - OutlierMultipliers.IQR.value - if outlier_method == OutlierMethod.IQR.value - else OutlierMultipliers.SD.value - ) - outlier_multiplier = st.number_input( - label="Select multiplier for outlier detection", - min_value=0.1, - max_value=10.0, - value=default_multiplier, - step=0.1, - help="Select the multiplier to use for outlier detection.", - key="outlier_multiplier", - ) - - outlier_threshold_default = ( - OutlierThresholds.SD.value - if outlier_method == OutlierMethod.SD.value - else OutlierThresholds.IQR.value - ) - outlier_threshold = st.number_input( - label="Outlier threshold (%)", - min_value=1, - value=outlier_threshold_default, - help="Set the minimum number values required to flag outliers in the column.", - key="outlier_threshold", - ) - - outlier_settings, valid_outlier = _validate_outlier_settings( - { - "outlier_method": outlier_method, - "outlier_multiplier": outlier_multiplier, - "outlier_threshold": outlier_threshold, - } - ) - return enable_outliers, outlier_settings, valid_outlier - else: - return False, None, True - - -def _render_constraint_options() -> tuple[dict, bool]: - """Render constraint bounds options UI. - - Returns - ------- - tuple[dict, bool] - Constraint settings dict and validation status. - """ - with st.container(border=True): - st.write("**Constraint Options:**") - - hc1, hc2 = st.columns(2) - with hc1: - hard_min = st.number_input( - label="(OPTIONAL) Hard minimum", - help="(OPTIONAL) Hard minimum value for outlier detection.", - value=None, - ) - with hc2: - hard_max = st.number_input( - label="(OPTIONAL) Hard maximum", - help="(OPTIONAL) Hard maximum value for outlier detection.", - value=None, - ) - - sc1, sc2 = st.columns(2) - with sc1: - soft_min = st.number_input( - label="(OPTIONAL) Soft minimum", - help="(OPTIONAL) Soft minimum value for outlier detection.", - value=None, - ) - with sc2: - soft_max = st.number_input( - label="(OPTIONAL) Soft maximum", - help="(OPTIONAL) Soft maximum value for outlier detection.", - value=None, - ) - - return _validate_constraint_settings( - { - "hard_min": hard_min, - "soft_min": soft_min, - "soft_max": soft_max, - "hard_max": hard_max, - } - ) - - -def _render_outlier_column_actions( - project_id: str, page_name_id: str, numeric_columns: list[str] -) -> None: - """Render the outlier column configuration UI. - - Parameters - ---------- - project_id : str - Project identifier. - page_name_id : str - Page name identifier. - numeric_columns : list[str] - List of numeric columns. - """ - outlier_settings = duckdb_get_table( - project_id, - f"outliers_{page_name_id}", - "logs", - ) - - os1, os2, _ = st.columns([0.4, 0.3, 0.3]) - with os1: - st.button( - "Add Outlier/Constraint Column", - key="add_outlier_column", - help="Add a new outlier column configuration.", - width="stretch", - type="primary", - on_click=_add_outlier_column, - args=( - project_id, - page_name_id, - numeric_columns, - ), - ) - with os2: - _delete_outlier_column(project_id, page_name_id, outlier_settings) - - if outlier_settings.is_empty(): - st.info( - "Use the :material/add: button to add columns to check for outliers and the " - ":material/delete: button to remove columns." - ) - else: - _render_outlier_settings_table(outlier_settings) - - -@st.dialog("Add Outlier & Constraint Column(s)", width="medium") -def _add_outlier_column( - project_id: str, page_name_id: str, numeric_columns: list[str] -) -> None: - """Dialog to add a new outlier column configuration. - - Parameters - ---------- - project_id : str - Project identifier. - page_name_id : str - Page name identifier. - numeric_columns : list[str] - List of numeric columns. - """ - # Render search type selection - search_type, pattern, outlier_cols, lock_cols_initial = ( - _render_search_type_selection(numeric_columns) - ) - - if outlier_cols: - # Render grouping options - group_cols, lock_cols = _render_column_grouping_options( - outlier_cols, search_type - ) - if lock_cols_initial is not None: - lock_cols = lock_cols_initial - - # Render outlier options - enable_outliers, outlier_settings, valid_outlier = _render_outlier_options() - - # Render constraint options - constraint_settings, valid_constraint = _render_constraint_options() - - button_disabled = ( - not outlier_cols - or (enable_outliers and not valid_outlier) - or not valid_constraint - ) - if st.button( - "Add Outlier & Constraint Configuration", - key="confirm_add_outlier_column", - type="primary", - width="stretch", - disabled=button_disabled, - ): - _update_outlier_column_config( - project_id, - page_name_id, - search_type, - pattern, - outlier_cols, - group_cols, - lock_cols, - enable_outliers, - outlier_settings, - constraint_settings, - ) - - st.success("Outlier & Constraint configuration added successfully.") - st.rerun() - - -def _validate_constraint_settings( - constraint_settings: dict, -) -> tuple[ConstraintBounds | None, bool]: - """Validate constraint settings using Pydantic model. - - Parameters - ---------- - constraint_settings : dict[str, Any] - Dictionary containing constraint settings. - - Returns - ------- - tuple[ConstraintBounds | None, bool] - Validated constraint settings and validation status. - """ - try: - return ConstraintBounds(**constraint_settings), True - except ValidationError as e: - user_message = _format_constraint_validation_error(e) - st.error(user_message) - return None, False - - -def _validate_outlier_settings( - outlier_settings: dict, -) -> tuple[OutlierOptionsConfig | None, bool]: - """Validate outlier settings using Pydantic model. - - Parameters - ---------- - outlier_settings : dict[str, Any] - Dictionary containing outlier settings. - - Returns - ------- - tuple[OutlierOptionsConfig | None, bool] - Validated outlier settings and validation status. - """ - try: - return OutlierOptionsConfig(**outlier_settings), True - except ValidationError as e: - user_message = _format_outlier_validation_error(e) - st.error(user_message) - return None, False - - -def _format_constraint_validation_error(e: ValidationError) -> str: - """Convert Pydantic ValidationError to user-friendly message. - - Parameters - ---------- - e : ValidationError - Pydantic validation error. - - Returns - ------- - str - User-friendly error message. - """ - errors = [] - for error in e.errors(): - field = " -> ".join(str(loc) for loc in error["loc"]) - msg = error["msg"] - - # Customize messages based on error type - if error["type"] == "float_not_finite": - errors.append( - f"• {field}: Value must be a finite number (not NaN or infinity)" - ) - elif error["type"] == "value_error": - errors.append(f"• {msg}") # Your custom validation messages - else: - errors.append(f"• {field}: {msg}") - - return "Invalid constraint configuration:\n" + "\n".join(errors) - - -def _format_outlier_validation_error(e: ValidationError) -> str: - """Convert Pydantic ValidationError to user-friendly message. - - Parameters - ---------- - e : ValidationError - Pydantic validation error. - - Returns - ------- - str - User-friendly error message. - """ - errors = [] - for error in e.errors(): - field = " -> ".join(str(loc) for loc in error["loc"]) - msg = error["msg"] - - # Customize messages based on error type - if error["type"] == "value_error.number.not_ge": - errors.append( - f"• {field}: Value must be greater than or equal to the minimum allowed." - ) - elif error["type"] == "value_error.number.not_le": - errors.append( - f"• {field}: Value must be less than or equal to the maximum allowed." - ) - else: - errors.append(f"• {field}: {msg}") - - return "Invalid outlier configuration:\n" + "\n".join(errors) - - -def _update_outlier_column_config( - project_id: str, - page_name_id: str, - search_type: str, - pattern: str | None, - outlier_cols: list[str], - group_cols: bool, - lock_cols: bool, - outlier_enabled: bool, - outlier_settings: OutlierOptionsConfig | None, - constraint_settings: ConstraintBounds | None, -) -> None: - """Update the outlier column configuration in the database. - - Parameters - ---------- - project_id : str - Project identifier. - page_name_id : str - Page name identifier. - search_type : str - Search type used. - pattern : str | None - Pattern for column matching. - outlier_cols : list[str] - Selected columns. - group_cols : bool - Whether to group columns. - lock_cols : bool - Whether to lock column selection. - outlier_enabled : bool - Whether outlier detection is enabled. - outlier_settings : OutlierOptionsConfig | None - Outlier detection settings. - constraint_settings : ConstraintBounds | None - Constraint bounds settings. - """ - # get existing config - existing_config = duckdb_get_table( - project_id, - f"outliers_{page_name_id}", - db_name="logs", - ) - - # Prepare new configurations - new_config = { - "search_type": search_type, - "pattern": pattern, - "column_name": [outlier_cols], - "grouped_columns": group_cols, - "locked": lock_cols, - "outlier_enabled": outlier_enabled, - "outlier_method": outlier_settings.outlier_method if outlier_settings else None, - "outlier_multiplier": outlier_settings.outlier_multiplier - if outlier_settings - else None, - "outlier_threshold": outlier_settings.outlier_threshold - if outlier_settings - else None, - "hard_min": constraint_settings.hard_min if constraint_settings else None, - "soft_min": constraint_settings.soft_min if constraint_settings else None, - "soft_max": constraint_settings.soft_max if constraint_settings else None, - "hard_max": constraint_settings.hard_max if constraint_settings else None, - } - - schema = { - "search_type": pl.Utf8, - "pattern": pl.Utf8, - "column_name": pl.List(pl.Utf8), - "grouped_columns": pl.Boolean, - "locked": pl.Boolean, - "outlier_enabled": pl.Boolean, - "outlier_method": pl.Utf8, - "outlier_multiplier": pl.Float64, - "outlier_threshold": pl.Int64, - "hard_min": pl.Float64, - "soft_min": pl.Float64, - "soft_max": pl.Float64, - "hard_max": pl.Float64, - } - - # Append new configurations to existing polars DataFrame - new_config_df = pl.DataFrame(new_config, schema=schema) - if not existing_config.is_empty(): - formatted_existing_config = _ensure_column_formats(existing_config) - updated_config = pl.concat( - [formatted_existing_config, new_config_df], how="vertical" - ) - else: - updated_config = new_config_df - - # Save updated configurations back to the database - duckdb_save_table( - project_id, - updated_config, - f"outliers_{page_name_id}", - db_name="logs", - ) - - -def _ensure_column_formats( - outlier_settings: pl.DataFrame, -) -> pl.DataFrame: - """Ensure correct data types for outlier settings DataFrame. - - Parameters - ---------- - outlier_settings : pl.DataFrame - Outlier settings configuration. - - Returns - ------- - pl.DataFrame - DataFrame with ensured data types. - """ - return outlier_settings.with_columns( - [ - pl.col("search_type").cast(pl.Utf8), - pl.col("pattern").cast(pl.Utf8), - pl.col("column_name").cast(pl.List(pl.Utf8)), - pl.col("grouped_columns").cast(pl.Boolean), - pl.col("locked").cast(pl.Boolean), - pl.col("outlier_enabled").cast(pl.Boolean), - pl.col("outlier_method").cast(pl.Utf8), - pl.col("outlier_multiplier").cast(pl.Float64), - pl.col("outlier_threshold").cast(pl.Int64), - pl.col("hard_min").cast(pl.Float64), - pl.col("soft_min").cast(pl.Float64), - pl.col("soft_max").cast(pl.Float64), - pl.col("hard_max").cast(pl.Float64), - ] - ) - - -def _render_outlier_settings_table(outlier_settings: pl.DataFrame) -> None: - """Render the outlier settings table in Streamlit. - - Parameters - ---------- - outlier_settings : pl.DataFrame - Outlier settings configuration. - """ - with st.expander("Outlier & Constraint Column Settings", expanded=False): - st.dataframe( - outlier_settings, - width="stretch", - hide_index=True, - column_config={ - "search_type": st.column_config.Column("Search Type"), - "pattern": st.column_config.Column("Pattern"), - "column_name": st.column_config.Column("Column Name(s)"), - "grouped_columns": st.column_config.CheckboxColumn("Grouped Columns"), - "locked": st.column_config.CheckboxColumn("Locked"), - "outlier_enabled": st.column_config.CheckboxColumn("Outlier Enabled"), - "outlier_method": st.column_config.Column("Outlier Method"), - "outlier_multiplier": st.column_config.NumberColumn( - "Outlier Multiplier" - ), - "outlier_threshold": st.column_config.NumberColumn("Outlier Threshold"), - "hard_min": st.column_config.NumberColumn("Hard Min"), - "soft_min": st.column_config.NumberColumn("Soft Min"), - "soft_max": st.column_config.NumberColumn("Soft Max"), - "hard_max": st.column_config.NumberColumn("Hard Max"), - }, - ) - - -def _delete_outlier_column( - project_id: str, page_name_id: str, outliers_settings: pl.DataFrame -) -> None: - """Render delete outlier column button and handle deletion. - - Parameters - ---------- - project_id : str - Project identifier. - page_name_id : str - Page name identifier. - outliers_settings : pl.DataFrame - Current outlier settings. - """ - with ( - st.popover( - label=":material/delete: Delete outlier column", - width="stretch", - ), - ): - st.markdown("#### Remove outlier columns") - - if outliers_settings.is_empty(): - st.info("No outlier columns have been added yet. ") - else: - outliers_settings = outliers_settings.with_row_index().with_columns( - ( - pl.col("index").cast(pl.Utf8) - + " - " - + pl.col("search_type") - + " - " - + pl.col("pattern").fill_null("") - ).alias("composite_index") - ) - - unique_index = ( - outliers_settings["composite_index"] - .unique(maintain_order=True) - .to_list() - ) - - selected_index = st.selectbox( - label="Select outlier column to remove", - options=unique_index, - help="Select the outlier column to remove from the list.", - ) - - if st.button( - label="Confirm deletion", - type="primary", - width="stretch", - help="Click to confirm deletion of the selected outlier column.", - key="confirm_delete_outlier_column", - disabled=not selected_index, - ): - updated_settings = outliers_settings.filter( - pl.col("composite_index") != selected_index - ).drop("composite_index") - - duckdb_save_table( - project_id, - updated_settings, - f"outliers_{page_name_id}", - "logs", - ) - - st.rerun() - - -# ============================================================================= -# Main Report Function -# ============================================================================= - - -def outliers_report( - project_id: str, - page_name_id: str, - data: pl.DataFrame, - setting_file: str, - config: dict, - survey_columns: ColumnByType, -) -> None: - """Create a comprehensive outliers report. - - Parameters - ---------- - project_id : str - The project identifier. - page_name_id : str - Page name identifier. - data : pd.DataFrame - DataFrame containing the survey data. - setting_file : str - Path to settings file. - config : dict - Configuration dictionary. - """ - # get column info - categorical_columns = survey_columns.categorical_columns - datetime_columns = survey_columns.datetime_columns - numeric_columns = survey_columns.numeric_columns - - st.title("Outliers and Constraints Report") - - if is_demo_project(): - demo_callout( - "This tab checks your survey data against two types of rules:\n\n" - "- **Constraint Violations**: Records that breach hard or soft numeric bounds " - "you define (e.g., age below 0 or above 120).\n" - "- **Outliers**: Records flagged by a statistical method (IQR or Standard " - "Deviation) as unusually high or low.\n\n" - "Start by reviewing the :material/settings: **settings** panel — your columns " - "are pre-filled. Then use **Add Outlier/Constraint Column** to configure " - "which columns to check." - ) - - # Load settings - config_settings = OutlierSettings(**config) - outliers_settings = outliers_report_settings( - setting_file, config_settings, categorical_columns, datetime_columns - ) - - # Outlier columns configuration - st.subheader("Outlier/Constraint Columns Configuration") - - if is_demo_project(): - demo_callout( - "Click **Add Outlier/Constraint Column** to select which numeric columns to " - "analyse. In the dialog that opens:\n\n" - "1. Leave **Search type** as **exact** and select **age** and " - "**household_count** from the column list.\n" - "2. Under **Outlier Options**, keep the default IQR method (multiplier 1.5, " - "threshold 20).\n" - "3. Under **Constraint Options**, optionally enter bounds — for example, " - "for **age** set **Hard Min = 0**, **Soft Min = 15**, **Soft Max = 60**, " - "**Hard Max = 100**. Hard bounds flag impossible values; soft bounds flag " - "values that are unusual but may be legitimate.\n" - "4. Click **Add Outlier & Constraint Configuration** to save.\n\n" - "Repeat the steps to add **land_acre** as a second column." - ) - - _render_outlier_column_actions(project_id, page_name_id, numeric_columns) - - # get outlier column config - outliers_column_config = duckdb_get_table( - project_id, - f"outliers_{page_name_id}", - "logs", - ) - - if outliers_column_config.is_empty(): - return - - # update lock columns if needed - outliers_column_config = _update_unlocked_cols( - outliers_column_config, - categorical_columns, - ) - - # save updated config - duckdb_save_table( - project_id, - outliers_column_config, - f"outliers_{page_name_id}", - db_name="logs", - ) - - # Show constraint violations - st.write("---") - st.title("Constraint Violations") - - if is_demo_project(): - demo_callout( - "This section shows records that breach the **hard or soft bounds** you set " - "when configuring columns above.\n\n" - "- **Hard Min / Hard Max**: Absolute limits — any value outside these is an " - "unambiguous error (e.g., age < 0 or age > 120).\n" - "- **Soft Min / Soft Max**: Advisory range — values outside these are " - "unexpected but may be legitimate (e.g., a very large land holding).\n\n" - "Six metrics show how many records breach each bound type. " - "Use **:material/clarify: Show more columns in report** to add context " - "columns such as **enum_name** or **state** to the violations table." - ) - - # compute constraint violations - constraint_violations = compute_constraint_violations( - data, - outliers_settings, - outliers_column_config, - ) - - if constraint_violations.is_empty(): - st.info("No constraint violations detected.") - - else: - # show constraint metrics - _render_constraint_metrics(constraint_violations) - - # show constraint violations table - st.subheader("Constraint Violations Details") - _render_constraint_violations_table( - data, - constraint_violations, - outliers_settings, - setting_file, - ) - - # show outliers metrics - st.write("---") - st.title("Outliers") - - if is_demo_project(): - demo_callout( - "This section flags records that fall outside the **statistical bounds** " - "computed by the method you chose (IQR or Standard Deviation).\n\n" - "Four metrics summarise the findings: **Columns Checked**, " - "**Columns with Outliers**, **Total Outliers**, and " - "**Enumerators with Outliers**.\n\n" - "Under **Inspect Columns**, select a column from the dropdown to see " - "its descriptive statistics and a **box plot** showing where flagged values " - "sit relative to the distribution. Expand " - "**:material/clarify: Show more columns in report** to add context columns " - "to the record table below the chart." - ) - - # Compute outliers - outlier_data = compute_outlier_output( - data, - outliers_settings, - outliers_column_config, - ) - - if outlier_data.is_empty(): - st.info("No outliers detected.") - - else: - # show outlier metrics - _render_outlier_metrics(outlier_data, outliers_settings) - - # show outlier column inspection - st.subheader("Inspect Columns") - - _render_outlier_column_inspection( - data, - outlier_data, - outliers_settings, - setting_file, - ) - - demo_callout( - "**Next**: :material/arrow_upward: Scroll up and select the **GPS Checks** tab." - ) diff --git a/src/datasure/checks/outliers/__init__.py b/src/datasure/checks/outliers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/datasure/checks/outliers/compute.py b/src/datasure/checks/outliers/compute.py new file mode 100644 index 00000000..d80339ba --- /dev/null +++ b/src/datasure/checks/outliers/compute.py @@ -0,0 +1,1220 @@ +"""Pure data-computation functions for the outliers module.""" + +import re +from typing import Any + +import numpy as np +import pandas as pd +import plotly.graph_objects as go # type: ignore +import polars as pl +import streamlit as st + +from datasure.checks.outliers.models import ( + TAB_NAME, + ConstraintMetrics, + OutlierBounds, + OutlierMethod, + OutlierMetrics, + OutlierMultipliers, + OutlierSettings, + OutlierStatistics, + OutlierThresholds, + SearchType, +) +from datasure.utils.dataframe_utils import safe_to_numeric +from datasure.utils.settings_utils import load_check_settings + +# ============================================================================= +# Utility Functions +# ============================================================================= + + +def _ensure_list(value: Any) -> list: + """Ensure value is a list. + + Parameters + ---------- + value : Any + Value to convert. + + Returns + ------- + list + Value as a list. + """ + if isinstance(value, str): + return [value] + if isinstance(value, list): + return value + return list(value) + + +def _build_include_cols( + survey_key: str, + survey_id: str | None, + survey_date: str | None, + enumerator: str | None, + team: str | None, +) -> list[str]: + """Build list of columns to include in output. + + Parameters + ---------- + survey_key : str + Survey key column. + survey_id : str | None + Survey ID column. + survey_date : str | None + Survey date column. + enumerator : str | None + Enumerator column. + team : str | None + Team column. + + Returns + ------- + list[str] + Deduplicated list of columns to include. + """ + include_cols = [] + for col in [survey_key, survey_id, survey_date, enumerator, team]: + if col and col not in include_cols: + include_cols.append(col) + return include_cols + + +# ============================================================================= +# Settings and Configuration Functions +# ============================================================================= + + +def load_default_settings( + settings_file: str, config: OutlierSettings +) -> OutlierSettings: + """Load the default settings for the outliers report. + + Parameters + ---------- + settings_file : str + The settings file to load. + config : OutlierSettings + Default configuration. + + Returns + ------- + OutlierSettings + Merged settings. + """ + # Load saved settings + saved_settings = load_check_settings(settings_file, TAB_NAME) + + default_settings: dict = dict(config) + default_settings.update(saved_settings) + + # Merge with defaults + return OutlierSettings(**default_settings) + + +@st.cache_data +def expand_col_names( + col_names: list[str], pattern: str, search_type: str = "exact" +) -> list[str]: + """Expand column names based on a pattern and search type. + + Parameters + ---------- + col_names : list[str] + List of column names to search in. + pattern : str + Pattern to match against column names. + search_type : str, default="exact" + Type of search to perform. + + Returns + ------- + list[str] + List of column names that match the pattern. + + Raises + ------ + TypeError + If input types are invalid. + ValueError + If search_type is not supported. + """ + if not isinstance(col_names, list): + raise TypeError("col_names must be a list of column names.") + if not pattern: + raise TypeError("pattern must be provided.") + if not isinstance(pattern, str): + raise TypeError("pattern must be a string.") + + search_funcs = { + SearchType.EXACT.value: lambda col: col == pattern, + SearchType.STARTSWITH.value: lambda col: col.startswith(pattern), + SearchType.ENDSWITH.value: lambda col: col.endswith(pattern), + SearchType.CONTAINS.value: lambda col: pattern in col, + SearchType.REGEX.value: lambda col: re.match(pattern, col), + } + + if search_type not in search_funcs: + valid_types = ", ".join(search_funcs.keys()) + raise ValueError( + f"Invalid search_type '{search_type}'. Choose from: {valid_types}." + ) + + return [col for col in col_names if search_funcs[search_type](col)] + + +def _should_expand_row(row: dict) -> bool: + """Check if a configuration row should have its columns expanded. + + Parameters + ---------- + row : dict + Configuration row to check (from Polars iter_rows). + + Returns + ------- + bool + True if row should be expanded. + """ + return row["search_type"] != SearchType.EXACT.value and not row.get("locked", False) + + +def _update_unlocked_cols( + column_config: pl.DataFrame, + col_names: list[str], +) -> pl.DataFrame: + """Update column names for unlocked rows in column configuration. + + Parameters + ---------- + column_config : pl.DataFrame + Polars DataFrame containing outlier column configuration. + col_names : list[str] + List of available column names. + + Returns + ------- + pl.DataFrame + Updated column configuration with expanded column names. + + Raises + ------ + ValueError + If essential columns are missing or pattern is invalid. + """ + required_columns = {"search_type", "pattern", "column_name", "locked"} + missing_columns = required_columns - set(column_config.columns) + if missing_columns: + raise ValueError( + f"Missing required columns in column_config: {', '.join(missing_columns)}" + ) + + updated_rows = [] + for row in column_config.iter_rows(named=True): + if _should_expand_row(row): + expanded_cols = expand_col_names( + col_names=col_names, + pattern=row["pattern"], + search_type=row["search_type"], + ) + row["outlier_cols"] = expanded_cols + updated_rows.append(row) + + return pl.DataFrame(updated_rows) + + +def update_unlocked_cols( + outlier_settings: pl.DataFrame, col_names: list[str] +) -> pl.DataFrame: + """Update column names for unlocked rows in outlier settings. + + Public API wrapper for backward compatibility. + + Parameters + ---------- + outlier_settings : pl.DataFrame + Polars DataFrame containing outlier settings or column configuration. + col_names : list[str] + List of available column names. + + Returns + ------- + pl.DataFrame + Updated settings with expanded column names. + + Raises + ------ + ValueError + If essential columns are missing or pattern is invalid. + """ + return _update_unlocked_cols(outlier_settings, col_names) + + +# ============================================================================= +# Statistical Computation Functions (Polars-optimized) +# ============================================================================= + + +def _compute_iqr_bounds(series: pl.Series, multiplier: float) -> OutlierBounds: + """Compute IQR-based outlier bounds. + + Parameters + ---------- + series : pl.Series + Numeric series to compute bounds for. + multiplier : float + IQR multiplier (typically 1.5). + + Returns + ------- + OutlierBounds + Lower and upper bounds. + """ + q1 = series.quantile(0.25) + q3 = series.quantile(0.75) + iqr = q3 - q1 + lower_bound = q1 - (multiplier * iqr) + upper_bound = q3 + (multiplier * iqr) + return OutlierBounds(lower_bound=lower_bound, upper_bound=upper_bound) + + +def _compute_sd_bounds(series: pl.Series, multiplier: float) -> OutlierBounds: + """Compute standard deviation-based outlier bounds. + + Parameters + ---------- + series : pl.Series + Numeric series to compute bounds for. + multiplier : float + SD multiplier (typically 3.0). + + Returns + ------- + OutlierBounds + Lower and upper bounds. + """ + mean = series.mean() + std = series.std() + lower_bound = mean - (multiplier * std) + upper_bound = mean + (multiplier * std) + return OutlierBounds(lower_bound=lower_bound, upper_bound=upper_bound) + + +def compute_outlier_stats_polars( + series: pl.Series, + outlier_type: str | None, + multiplier: float | None, +) -> OutlierStatistics: + """Compute outlier statistics using Polars for better performance. + + Parameters + ---------- + series : pl.Series + The Series to compute statistics for. + outlier_type : str | None + The type of outlier detection method to use. + multiplier : float | None + The multiplier to use for outlier detection. + + Returns + ------- + OutlierStatistics + Pydantic model containing computed statistics. + + Raises + ------ + ValueError + If series is empty or parameters are invalid. + """ + if series.len() == 0: + raise ValueError("The Series is empty.") + + valid_types = [None, OutlierMethod.IQR.value, OutlierMethod.SD.value] + if outlier_type not in valid_types: + raise ValueError( + f"Invalid outlier type. Use 'IQR' or 'SD', got: {outlier_type}" + ) + + if multiplier is not None and multiplier <= 0: + raise ValueError("Multiplier must be a positive number.") + + # remove nulls for accurate stats + series = series.drop_nulls() + + series = safe_to_numeric(series) + + # return empty stats if no non-null values + if series.len() == 0: + return OutlierStatistics( + count=0, + min_value=float("nan"), + max_value=float("nan"), + mean=float("nan"), + median=float("nan"), + sd=float("nan"), + iqr=float("nan"), + lower_bound=float("nan"), + upper_bound=float("nan"), + ) + + # Compute basic statistics + count = series.len() - series.null_count() + min_value = series.min() + max_value = series.max() + mean = series.mean() + median = series.median() + sd = series.std() + q1 = series.quantile(0.25) + q3 = series.quantile(0.75) + iqr = q3 - q1 + + # Compute bounds based on method + if outlier_type == OutlierMethod.SD.value: + multiplier = multiplier or OutlierMultipliers.SD.value + bounds = _compute_sd_bounds(series, multiplier) + else: # Default to IQR + multiplier = multiplier or OutlierMultipliers.IQR.value + bounds = _compute_iqr_bounds(series, multiplier) + + return OutlierStatistics( + count=count, + min_value=min_value, + max_value=max_value, + mean=mean, + median=median, + sd=sd, + iqr=iqr, + lower_bound=bounds.lower_bound, + upper_bound=bounds.upper_bound, + ) + + +@st.cache_data(hash_funcs={pl.DataFrame: lambda df: str(df.schema)}) +def stack_outlier_columns(df: pl.DataFrame, col_names: list[str]) -> pl.Series: + """Stack specified columns of a DataFrame into a single Series. + + Parameters + ---------- + df : pl.DataFrame + The DataFrame containing the data. + col_names : list[str] + List of column names to stack. + + Returns + ------- + pl.Series + A Series containing the stacked values. + + Raises + ------ + ValueError + If DataFrame is empty or columns don't exist. + """ + if df.is_empty(): + raise ValueError("The DataFrame is empty.") + + for col in col_names: + if col not in df.columns: + raise ValueError(f"Column '{col}' does not exist in the DataFrame.") + + # Check and convert columns to numeric if needed + for col in col_names: + dtype = df[col].dtype + if dtype not in [ + pl.Int8, + pl.Int16, + pl.Int32, + pl.Int64, + pl.UInt8, + pl.UInt16, + pl.UInt32, + pl.UInt64, + pl.Float32, + pl.Float64, + ]: + try: + df = df.with_columns(pl.col(col).cast(pl.Float64)) + except Exception: + raise ValueError( + f"Column '{col}' cannot be converted to numeric type." + ) from None + + # Stack the columns - melt/unpivot in Polars + stacked_values = ( + df.select(col_names) + .unpivot() + .get_column("value") + .drop_nulls() # Remove null values like pandas stack() does + ) + + return stacked_values + + +def _build_outlier_expression( + col: str, + lower_bound: float, + upper_bound: float, +) -> pl.Expr: + """Build Polars expression for outlier flagging. + + Parameters + ---------- + col : str + Column name. + lower_bound : float + Statistical lower bound. + upper_bound : float + Statistical upper bound. + + Returns + ------- + pl.Expr + Polars expression for outlier detection. + """ + outlier_expr = ( + pl.when(pl.col(col) < lower_bound) + .then(pl.lit(f"Value is below lower bound {lower_bound:.2f}")) + .when(pl.col(col) > upper_bound) + .then(pl.lit(f"Value is above upper bound {upper_bound:.2f}")) + ) + + return outlier_expr.otherwise(pl.lit("no outlier")) + + +def _add_statistics_columns( + col_df: pl.DataFrame, + outlier_stats: OutlierStatistics, + outlier_method: str, + outlier_multiplier: float, + col_name: str, +) -> pl.DataFrame: + """Add statistics columns to the outlier dataframe. + + Parameters + ---------- + col_df : pl.DataFrame + DataFrame to add columns to. + outlier_stats : OutlierStatistics + Computed statistics. + outlier_method : str + Detection method used. + outlier_multiplier : float + Multiplier used. + col_name : str + Name of the column being analyzed. + + Returns + ------- + pl.DataFrame + DataFrame with added statistics columns. + """ + return col_df.with_columns( + [ + pl.lit(outlier_stats.min_value, dtype=pl.Float64).alias("min_value"), + pl.lit(outlier_stats.max_value, dtype=pl.Float64).alias("max_value"), + pl.lit(outlier_stats.mean, dtype=pl.Float64).alias("mean"), + pl.lit(outlier_stats.median, dtype=pl.Float64).alias("median"), + pl.lit(outlier_stats.sd, dtype=pl.Float64).alias("std"), + pl.lit(outlier_stats.iqr, dtype=pl.Float64).alias("iqr"), + pl.lit(outlier_stats.lower_bound, dtype=pl.Float64).alias("lower_bound"), + pl.lit(outlier_stats.upper_bound, dtype=pl.Float64).alias("upper_bound"), + pl.lit(outlier_method).alias("outlier_method"), + pl.lit(outlier_multiplier, dtype=pl.Float64).alias("outlier_multiplier"), + pl.lit(col_name).alias("column name"), + ] + ) + + +def _process_single_column_outliers( + df_polars: pl.DataFrame, + col: str, + survey_key: str, + outlier_stats: OutlierStatistics, + outlier_method: str, + outlier_multiplier: float, + min_threshold: int, + non_null_count: int, +) -> pl.DataFrame: + """Process outliers for a single column using Polars. + + Parameters + ---------- + df_polars : pl.DataFrame + Polars DataFrame containing the data. + col : str + Column name to process. + survey_key : str + Survey key column name. + outlier_stats : OutlierStatistics + Pre-computed statistics. + outlier_method : str + Outlier detection method. + outlier_multiplier : float + Multiplier for detection. + min_threshold : int + Minimum sample size threshold. + non_null_count : int + Number of non-null values. + + Returns + ------- + pl.DataFrame + DataFrame with outlier information for the column. + """ + # Select relevant columns + col_df = df_polars.select([survey_key, col]) + col_df = safe_to_numeric(col_df, col) + + # Add outlier reason + if non_null_count < min_threshold: + col_df = col_df.with_columns(pl.lit("no outlier").alias("outlier reason")) + else: + # Vectorized outlier flagging + outlier_expr = _build_outlier_expression( + col, + outlier_stats.lower_bound, + outlier_stats.upper_bound, + ) + col_df = col_df.with_columns(outlier_expr.alias("outlier reason")) + + # Add statistics columns + col_df = _add_statistics_columns( + col_df, + outlier_stats, + outlier_method, + outlier_multiplier, + col, + ) + + # Rename and reorder + col_df = col_df.rename({col: "column value"}) + col_df = col_df.select( + [ + survey_key, + "column name", + "column value", + "min_value", + "max_value", + "mean", + "median", + "std", + "iqr", + "lower_bound", + "upper_bound", + "outlier reason", + "outlier_method", + "outlier_multiplier", + ] + ) + + return col_df + + +# ============================================================================= +# Outlier Detection - Main Logic +# ============================================================================= + + +def _compute_column_stats( + df_polars: pl.DataFrame, + outlier_cols: list[str], + grouped_cols: bool, + outlier_method: str, + outlier_multiplier: float, +) -> tuple[OutlierStatistics, int]: + """Compute outlier statistics for grouped columns. + + Parameters + ---------- + df_polars : pl.DataFrame + DataFrame containing the data (with survey_key and outlier columns). + outlier_cols : list[str] + List of columns to analyze. + grouped_cols : bool + Whether columns should be analyzed together. + outlier_method : str + Outlier detection method (IQR or SD). + outlier_multiplier : float + Multiplier for bounds calculation. + + Returns + ------- + tuple[OutlierStatistics, int] + Computed statistics and non-null count. + """ + if len(outlier_cols) == 1 or grouped_cols: + if len(outlier_cols) == 1: + series = df_polars[outlier_cols[0]] + else: + series = pl.concat([df_polars[col] for col in outlier_cols]) + + non_null_count = series.len() - series.null_count() + stats = compute_outlier_stats_polars( + series, + outlier_type=outlier_method, + multiplier=outlier_multiplier, + ) + return stats, non_null_count + + # For non-grouped multiple columns, return None to signal per-column computation + return None, 0 + + +def _compute_single_column_stats( + df_polars: pl.DataFrame, + col: str, + outlier_method: str, + outlier_multiplier: float, +) -> tuple[OutlierStatistics, int]: + """Compute outlier statistics for a single column. + + Parameters + ---------- + df_polars : pl.DataFrame + DataFrame containing the data. + col : str + Column name to analyze. + outlier_method : str + Outlier detection method. + outlier_multiplier : float + Multiplier for bounds calculation. + + Returns + ------- + tuple[OutlierStatistics, int] + Computed statistics and non-null count. + """ + non_null_count = df_polars.height - df_polars[col].null_count() + stats = compute_outlier_stats_polars( + df_polars[col], + outlier_type=outlier_method, + multiplier=outlier_multiplier, + ) + return stats, non_null_count + + +def _merge_outlier_results( + outlier_results_list: list[pl.DataFrame], + admin_data_polars: pl.DataFrame, + survey_key: str, +) -> pl.DataFrame: + """Merge outlier results with admin data. + + Parameters + ---------- + outlier_results_list : list[pl.DataFrame] + List of outlier result DataFrames. + admin_data_polars : pl.DataFrame + Admin data DataFrame. + survey_key : str + Survey key column name. + + Returns + ------- + pl.DataFrame + Merged results or empty DataFrame if no results. + """ + if not outlier_results_list: + return pl.DataFrame() + + outlier_results_polars = pl.concat(outlier_results_list) + + if admin_data_polars.is_empty(): + return outlier_results_polars + + return admin_data_polars.join( + outlier_results_polars, + on=survey_key, + how="left", + ) + + +def _process_outlier_configs( + data: pl.DataFrame, + column_config: pl.DataFrame, + survey_key: str, +) -> list[pl.DataFrame]: + """Process all outlier configurations and return results. + + Parameters + ---------- + data : pl.DataFrame + DataFrame containing the survey data. + column_config : pl.DataFrame + DataFrame containing the outlier column configurations. + survey_key : str + Survey key column name. + + Returns + ------- + list[pl.DataFrame] + List of outlier result DataFrames. + """ + outlier_results_list = [] + + for row in column_config.iter_rows(named=True): + if not row.get("outlier_enabled", False): + continue + + results = _process_single_config(data, row, survey_key) + outlier_results_list.extend(results) + + return outlier_results_list + + +def _process_single_config( + data: pl.DataFrame, + row: dict, + survey_key: str, +) -> list[pl.DataFrame]: + """Process a single outlier configuration row. + + Parameters + ---------- + data : pl.DataFrame + DataFrame containing the survey data. + row : dict + Configuration row from column_config. + survey_key : str + Survey key column name. + + Returns + ------- + list[pl.DataFrame] + List of outlier results for this configuration. + """ + # Extract settings with defaults + outlier_cols = _ensure_list(row.get("column_name", [])) + grouped_cols = row.get("grouped_columns", False) + outlier_method = row.get("outlier_method", OutlierMethod.IQR.value) + threshold = row.get("outlier_threshold", OutlierThresholds.IQR.value) + outlier_multiplier = row.get("outlier_multiplier", OutlierMultipliers.IQR.value) + + # Create subset + outlier_df_polars = data.select([survey_key, *outlier_cols]) + + # Compute shared stats for single column or grouped columns + shared_stats, shared_count = _compute_column_stats( + outlier_df_polars, + outlier_cols, + grouped_cols, + outlier_method, + outlier_multiplier, + ) + + # Process each column + results = [] + for col in outlier_cols: + if shared_stats is not None: + outlier_stats, non_null_count = shared_stats, shared_count + else: + outlier_stats, non_null_count = _compute_single_column_stats( + outlier_df_polars, col, outlier_method, outlier_multiplier + ) + + col_result = _process_single_column_outliers( + df_polars=outlier_df_polars, + col=col, + survey_key=survey_key, + outlier_stats=outlier_stats, + outlier_method=outlier_method, + outlier_multiplier=outlier_multiplier, + min_threshold=threshold, + non_null_count=non_null_count, + ) + results.append(col_result) + + return results + + +def compute_outlier_output( + data: pl.DataFrame, + outlier_settings: dict, + column_config: pl.DataFrame, +) -> pl.DataFrame: + """Detect outliers in DataFrame based on settings (Polars-optimized). + + Parameters + ---------- + data : pl.DataFrame + DataFrame containing the survey data. + outlier_settings : dict + Outlier settings configuration. + column_config : pl.DataFrame + DataFrame containing the outlier column configurations. + + Returns + ------- + pl.DataFrame + DataFrame containing the outlier summary. + + Raises + ------ + ValueError + If DataFrame is empty. + """ + if data.is_empty(): + raise ValueError("The DataFrame is empty. Please provide a valid DataFrame.") + + # Build include columns list + survey_key = outlier_settings.survey_key + include_cols = _build_include_cols( + survey_key, + outlier_settings.survey_id, + outlier_settings.survey_date, + outlier_settings.enumerator, + outlier_settings.team, + ) + admin_data_polars = data.select(include_cols) + + # Process outlier settings + outlier_results_list = _process_outlier_configs(data, column_config, survey_key) + + return _merge_outlier_results(outlier_results_list, admin_data_polars, survey_key) + + +# ============================================================================= +# Constraint Violations - Main Logic +# ============================================================================= + + +def compute_constraint_violations( + data: pl.DataFrame, + settings: OutlierSettings, + column_config: pl.DataFrame, +) -> pl.DataFrame: + """Compute constraint violations for outlier detection. + + Parameters + ---------- + data : pl.DataFrame + DataFrame containing the survey data. + settings : OutlierSettings + Outlier settings configuration. + column_config : pl.DataFrame + DataFrame containing the outlier column configurations. + + Returns + ------- + pl.DataFrame + DataFrame containing constraint violation information. + """ + survey_key = settings.survey_key + + violation_results = pl.DataFrame() + + for row in column_config.iter_rows(named=True): + outlier_cols = _ensure_list(row.get("column_name", [])) + hard_min = row.get("hard_min", None) + soft_min = row.get("soft_min", None) + soft_max = row.get("soft_max", None) + hard_max = row.get("hard_max", None) + + # skip if no bounds are set + if all(bound is None for bound in [hard_min, soft_min, soft_max, hard_max]): + continue + + for col in outlier_cols: + col_df = data.select([survey_key, col]) + + violation_expr = ( + pl.when((hard_min is not None) & (pl.col(col) < hard_min)) + .then(pl.lit(f"Value is below hard minimum {hard_min}")) + .when((soft_min is not None) & (pl.col(col) < soft_min)) + .then(pl.lit(f"Value is below soft minimum {soft_min}")) + .when((soft_max is not None) & (pl.col(col) > soft_max)) + .then(pl.lit(f"Value is above soft maximum {soft_max}")) + .when((hard_max is not None) & (pl.col(col) > hard_max)) + .then(pl.lit(f"Value is above hard maximum {hard_max}")) + ) + + col_df = safe_to_numeric(col_df, col) + + col_df = col_df.with_columns( + violation_expr.otherwise(pl.lit("no violation")).alias( + "violation reason" + ) + ) + + # add hard and soft bounds columns + for bound_name, bound_value in [ + ("hard_min", hard_min), + ("soft_min", soft_min), + ("soft_max", soft_max), + ("hard_max", hard_max), + ]: + col_df = col_df.with_columns(pl.lit(bound_value).alias(bound_name)) + + col_df = col_df.rename({col: "column value"}) + col_df = col_df.with_columns(pl.lit(col).alias("column name")).select( + [ + survey_key, + "column name", + "column value", + "hard_min", + "soft_min", + "soft_max", + "hard_max", + "violation reason", + ] + ) + + violation_results = ( + violation_results.vstack(col_df) + if not violation_results.is_empty() + else col_df + ) + + return violation_results + + +# ============================================================================= +# Metrics Computation - Analytics +# ============================================================================= + + +def _compute_constraint_metrics(violation_data: pl.DataFrame) -> ConstraintMetrics: + """Compute metrics related to constraint violations. + + Parameters + ---------- + violation_data : pl.DataFrame + DataFrame containing constraint violation data. + + Returns + ------- + ConstraintMetrics + Pydantic model containing computed metrics. + """ + columns_checked = violation_data.select("column name").n_unique() + total_violations = violation_data.filter( + pl.col("violation reason") != "no violation" + ).height + + hard_min_violations = violation_data.filter( + pl.col("violation reason").str.contains("below hard minimum") + ).height + soft_min_violations = violation_data.filter( + pl.col("violation reason").str.contains("below soft minimum") + ).height + soft_max_violations = violation_data.filter( + pl.col("violation reason").str.contains("above soft maximum") + ).height + hard_max_violations = violation_data.filter( + pl.col("violation reason").str.contains("above hard maximum") + ).height + + return ConstraintMetrics( + columns_checked=columns_checked, + total_violations=total_violations, + hard_min_violations=hard_min_violations, + soft_min_violations=soft_min_violations, + soft_max_violations=soft_max_violations, + hard_max_violations=hard_max_violations, + ) + + +def _compute_outlier_metrics( + outliers_data: pl.DataFrame, + enumerator: str | None, +) -> OutlierMetrics: + """Compute outlier metrics. + + Parameters + ---------- + outliers_data : pl.DataFrame + DataFrame containing outlier data. + enumerator : str | None + Enumerator column name. + + Returns + ------- + OutlierMetrics + Pydantic model containing computed metrics. + """ + columns_checked = outliers_data.select("column name").n_unique() + columns_with_outliers = ( + outliers_data.filter(pl.col("outlier reason") != "no outlier") + .select("column name") + .n_unique() + ) + total_outliers = outliers_data.filter( + pl.col("outlier reason") != "no outlier" + ).height + if enumerator: + enumerators_with_outliers = ( + outliers_data.filter(pl.col("outlier reason") != "no outlier") + .select(enumerator) + .n_unique() + ) + else: + enumerators_with_outliers = 0 + + return OutlierMetrics( + columns_checked=columns_checked, + columns_with_outliers=columns_with_outliers, + total_outliers=total_outliers, + enumerators_with_outliers=enumerators_with_outliers, + ) + + +def compute_column_outlier_summary( + outlier_data: pl.DataFrame, survey_key: str +) -> pl.DataFrame: + """Compute a summary of outliers for each column using Polars. + + Parameters + ---------- + outlier_data : pl.DataFrame + Polars DataFrame containing outlier data. + survey_key : str + Survey key column name. + + Returns + ------- + pl.DataFrame + Summary DataFrame with outlier counts per column. + """ + if outlier_data.is_empty(): + return pl.DataFrame() + + # Remove duplicates + outlier_summary = outlier_data.unique(subset=["column name", survey_key]) + + # Count occurrences per column + col_counts = outlier_summary.group_by("column name").agg(pl.count().alias("count")) + + # Join counts back + outlier_summary = outlier_summary.join(col_counts, on="column name", how="left") + + # Flag outliers + outlier_summary = outlier_summary.with_columns( + pl.when(pl.col("outlier reason") != "no outlier") + .then(pl.lit(1)) + .otherwise(pl.lit(0)) + .alias("flagged as outlier") + ) + + # Count outliers per column + outlier_counts = outlier_summary.group_by("column name").agg( + pl.col("flagged as outlier").sum().alias("outlier count") + ) + + # Merge outlier counts + outlier_summary = outlier_summary.join(outlier_counts, on="column name", how="left") + + # Select and order columns + outlier_summary = outlier_summary.select( + [ + "column name", + "count", + "outlier count", + "min_value", + "max_value", + "mean", + "median", + "std", + "iqr", + "lower_bound", + "upper_bound", + ] + ) + + return outlier_summary.unique(subset=["column name"]) + + +def get_outlier_cols(outlier_settings: pd.DataFrame) -> list[str]: + """Get list of outlier columns from settings DataFrame. + + Parameters + ---------- + outlier_settings : pd.DataFrame + DataFrame containing outlier settings. + + Returns + ------- + list[str] + List of column names to check for outliers. + """ + cols = [] + for i in range(len(outlier_settings)): + col = outlier_settings.iloc[i]["outlier_cols"] + if isinstance(col, np.ndarray): + cols.append(col[0]) + elif isinstance(col, list): + cols.extend(col) + + return cols + + +# ============================================================================= +# Visualization Functions +# ============================================================================= + + +@st.cache_data +def _create_box_plot(data: pd.Series, title: str) -> go.Figure: + """Create a box plot using plotly. + + Parameters + ---------- + data : pd.Series + Data series to plot. + title : str + Title for the plot. + + Returns + ------- + go.Figure + Plotly figure object. + """ + return go.Figure( + data=go.Box( + y=data, + boxpoints="outliers", + marker_color="darkblue", + line_color="black", + fillcolor="lightblue", + opacity=0.6, + x0=title, + ) + ) + + +@st.cache_data +def _create_descriptive_stats(column_data: pl.DataFrame) -> pl.DataFrame: + """Create descriptive statistics table. + + Parameters + ---------- + column_data : pl.DataFrame + Column data to analyze. + + Returns + ------- + pl.DataFrame + Descriptive statistics table. + """ + table = column_data.describe() + table.columns = ["statistic", "value"] + # rename statistics + stat_rename = { + "count": "Number of Values", + "null_count": "Number of Missing Values", + "mean": "Mean", + "std": "Standard Deviation", + "min": "Minimum Value", + "25%": "25th Percentile (Q1)", + "50%": "Median (Q2)", + "75%": "75th Percentile (Q3)", + "max": "Maximum Value", + } + + table = table.with_columns( + pl.col("statistic").replace(stat_rename).alias("statistic") + ) + + return table diff --git a/src/datasure/checks/outliers/models.py b/src/datasure/checks/outliers/models.py new file mode 100644 index 00000000..0438de8a --- /dev/null +++ b/src/datasure/checks/outliers/models.py @@ -0,0 +1,209 @@ +"""Pydantic models, enums, and constants for the outliers module.""" + +from enum import Enum, IntEnum, StrEnum + +from pydantic import ( + BaseModel, + Field, + field_validator, + model_validator, +) + +TAB_NAME: str = "outliers" + + +# ============================================================================= +# Enums and Constants +# ============================================================================= + + +class OutlierMethod(StrEnum): + """Supported outlier detection methods.""" + + IQR = "Interquartile Range (IQR)" + SD = "Standard Deviation (SD)" + + +class SearchType(StrEnum): + """Column search pattern types.""" + + EXACT = "exact" + STARTSWITH = "startswith" + ENDSWITH = "endswith" + CONTAINS = "contains" + REGEX = "regex" + + +class OutlierThresholds(IntEnum): + """Integer thresholds""" + + IQR = 20 + SD = 30 + + +class OutlierMultipliers(float, Enum): + """Float multipliers""" + + IQR = 1.5 + SD = 3.0 + + +# ============================================================================= +# Pydantic Models for Data Validation +# ============================================================================= + + +class OutlierBounds(BaseModel): + """Statistical bounds for outlier detection.""" + + lower_bound: float + upper_bound: float + + +class OutlierOptionsConfig(BaseModel): + """Configuration for outlier options.""" + + outlier_method: OutlierMethod = Field( + ..., description="Outlier detection method to use." + ) + outlier_multiplier: float = Field( + ..., + gt=0, + le=10.0, + description="Multiplier for outlier detection method.", + ) + outlier_threshold: int = Field( + ..., + gt=0, + description="Minimum number of non-null values required to flag outliers.", + ) + + +class ConstraintBounds(BaseModel): + """User-defined constraint bounds for outlier detection. + + Bounds hierarchy: hard_min <= soft_min <= soft_max <= hard_max + Values can be positive, negative, or zero. Infinity values are not allowed. + """ + + hard_min: int | float | None = Field(None, description="Absolute Minimum bound") + soft_min: int | float | None = Field(None, description="Expected Minimum bound") + soft_max: int | float | None = Field(None, description="Expected Maximum bound") + hard_max: int | float | None = Field(None, description="Absolute Maximum bound") + + @model_validator(mode="after") + def validate_bounds_hierarchy(self): + """Validate the complete hierarchy of bounds.""" + bounds = [ + ("hard_min", self.hard_min), + ("soft_min", self.soft_min), + ("soft_max", self.soft_max), + ("hard_max", self.hard_max), + ] + + # Get only non-None values with their names + defined_bounds = [(name, val) for name, val in bounds if val is not None] + + # Check that all defined bounds are in ascending order + for i in range(len(defined_bounds) - 1): + curr_name, curr_val = defined_bounds[i] + next_name, next_val = defined_bounds[i + 1] + if curr_val > next_val: + raise ValueError( + f"{curr_name} ({curr_val}) must be <= {next_name} ({next_val}). " + f"Bounds must follow hierarchy: hard_min <= soft_min <= soft_max <= hard_max" + ) + + return self + + +class ConstraintMetrics(BaseModel): + """Computed metrics for constraint violations.""" + + columns_checked: int = Field(ge=0, description="Number of columns checked") + total_violations: int = Field( + ge=0, description="Total number of constraint violations" + ) + hard_min_violations: int = Field(ge=0, description="Count of values below hard_min") + soft_min_violations: int = Field(ge=0, description="Count of values below soft_min") + soft_max_violations: int = Field(ge=0, description="Count of values above soft_max") + hard_max_violations: int = Field(ge=0, description="Count of values above hard_max") + + +class OutlierMetrics(BaseModel): + """Computed Metrics for Outlier Checks""" + + columns_checked: int = Field(ge=0, description="Number of columns checked") + columns_with_outliers: int = Field( + ge=0, description="Total number of columns with outlier values" + ) + total_outliers: int = Field(ge=0, description="Total number of outliers flagged") + enumerators_with_outliers: int = Field( + ge=0, description="Total number of outliers flagged" + ) + + +class OutlierStatistics(BaseModel): + """Complete statistical summary for outlier detection.""" + + count: int = Field(ge=0, description="Number of non-null values") + min_value: float + max_value: float + mean: float + median: float + sd: float | None + iqr: float | None + lower_bound: float | None + upper_bound: float | None + + class Config: + """Pydantic config.""" + + populate_by_name = True + + +class OutlierColumnConfig(BaseModel): + """Configuration for a single outlier column check.""" + + search_type: SearchType + pattern: str | None = None + outlier_cols: list[str] = Field(min_length=1) + lock_cols: bool = False + grouped_cols: bool = False + outlier_method: OutlierMethod = OutlierMethod.IQR + outlier_multiplier: float = Field(gt=0, le=10.0) + soft_min: float | None = None + soft_max: float | None = None + + @field_validator("pattern") + @classmethod + def validate_pattern(cls, v: str | None, info) -> str | None: + """Validate pattern is required for non-exact search types.""" + if info.data.get("search_type") != SearchType.EXACT and not v: + raise ValueError("Pattern is required for non-exact search types") + return v + + @field_validator("soft_max") + @classmethod + def validate_soft_bounds(cls, v: float | None, info) -> float | None: + """Validate soft_max is greater than soft_min.""" + soft_min = info.data.get("soft_min") + if v is not None and soft_min is not None and v <= soft_min: + raise ValueError("soft_max must be greater than soft_min") + return v + + +class OutlierSettings(BaseModel): + """Main configuration for outlier report.""" + + survey_key: str = Field(..., description="Column name for survey key", min_length=1) + survey_id: str | None = Field( + None, description="Column name for survey ID", min_length=1 + ) + survey_date: str | None = Field( + None, description="Column name for survey date", min_length=1 + ) + enumerator: str | None = Field( + None, description="Column name for enumerator ID", min_length=1 + ) + team: str | None = Field(None, description="Column name for team", min_length=1) diff --git a/src/datasure/checks/outliers/report_ui.py b/src/datasure/checks/outliers/report_ui.py new file mode 100644 index 00000000..aa6023af --- /dev/null +++ b/src/datasure/checks/outliers/report_ui.py @@ -0,0 +1,1373 @@ +"""Report-rendering UI for the outliers report.""" + +from collections.abc import Callable + +import polars as pl +import streamlit as st +from pydantic import BaseModel, ValidationError + +from datasure.checks.outliers.compute import ( + _build_include_cols, + _compute_constraint_metrics, + _compute_outlier_metrics, + _create_box_plot, + _create_descriptive_stats, + _update_unlocked_cols, + compute_constraint_violations, + compute_outlier_output, + expand_col_names, +) +from datasure.checks.outliers.models import ( + TAB_NAME, + ConstraintBounds, + ConstraintMetrics, + OutlierMethod, + OutlierMetrics, + OutlierMultipliers, + OutlierOptionsConfig, + OutlierSettings, + OutlierThresholds, + SearchType, +) +from datasure.checks.outliers.settings_ui import outliers_report_settings +from datasure.utils.dataframe_utils import ColumnByType, sanitize_df_for_join +from datasure.utils.duckdb_utils import duckdb_get_table, duckdb_save_table +from datasure.utils.navigations_utils import demo_callout +from datasure.utils.onboarding_utils import is_demo_project +from datasure.utils.settings_utils import ( + load_check_settings, + save_check_settings, + trigger_save, +) + +# ============================================================================= +# Streamlit UI - Metrics Display +# ============================================================================= + + +def _render_constraint_metrics( + violation_data: pl.DataFrame, +) -> None: + """Render constraint violation metrics using Streamlit. + + Parameters + ---------- + violation_data : pl.DataFrame + DataFrame containing constraint violation data. + """ + metrics: ConstraintMetrics = _compute_constraint_metrics(violation_data) + + _, _, uc3, uc4 = st.columns(4) + with uc3, st.container(border=True): + st.metric( + label="Number of columns checked", + value=f"{metrics.columns_checked:,}", + help="Number of columns checked for constraint violations", + ) + with uc4, st.container(border=True): + st.metric( + label="Total Violations", + value=f"{metrics.total_violations:,}", + help="Total number of constraint violations detected", + ) + + lc1, lc2, lc3, lc4 = st.columns(4, border=True) + lc1.metric( + label="Hard Min Violations", + value=f"{metrics.hard_min_violations:,}", + help="Number of violations below hard minimum", + ) + lc2.metric( + label="Soft Min Violations", + value=f"{metrics.soft_min_violations:,}", + help="Number of violations below soft minimum", + ) + lc3.metric( + label="Soft Max Violations", + value=f"{metrics.soft_max_violations:,}", + help="Number of violations above soft maximum", + ) + lc4.metric( + label="Hard Max Violations", + value=f"{metrics.hard_max_violations:,}", + help="Number of violations above hard maximum", + ) + + +def _render_outlier_metrics( + outliers_data: pl.DataFrame, + settings: OutlierSettings, +) -> None: + """Render outlier metrics using Streamlit. + + Parameters + ---------- + outliers_data : pl.DataFrame + DataFrame containing outlier data. + settings : OutlierSettings + Outlier settings configuration. + """ + metrics: OutlierMetrics = _compute_outlier_metrics( + outliers_data, settings.enumerator + ) + + uc1, uc2, uc3, uc4 = st.columns(4, border=True) + uc1.metric( + label="Number of columns checked", + value=f"{metrics.columns_checked:,}", + help="Number of columns checked for outliers", + ) + uc2.metric( + label="Columns with Outliers", + value=f"{metrics.columns_with_outliers:,}", + help="Number of columns that have outliers detected", + ) + uc3.metric( + label="Total Outliers", + value=f"{metrics.total_outliers:,}", + help="Total number of outliers detected", + ) + if settings.enumerator: + uc4.metric( + label="Enumerators with Outliers", + value=f"{metrics.enumerators_with_outliers:,}", + help="Number of unique enumerators with outliers detected", + ) + + +# ============================================================================= +# Streamlit UI - Table Display +# ============================================================================= + + +def _render_display_columns_expander( + setting_file: str, + settings_key: str, + widget_key: str, + display_options: list[str], + info_message: str, +) -> list[str]: + """Render the "Show more columns" expander and return the selected columns. + + Shared by ``_render_constraint_violations_table`` and ``_render_outlier_table``, + which both let users add extra context columns to a results table, persisting + the selection to the settings file under ``settings_key``. + + Parameters + ---------- + setting_file : str + Path to settings file. + settings_key : str + Key under which the selected columns are persisted in settings, and + used as the suffix for the ``trigger_save`` state name. + widget_key : str + Streamlit widget key for the multiselect. + display_options : list[str] + Columns available for selection. + info_message : str + Help text shown above the multiselect. + + Returns + ------- + list[str] + Columns selected by the user. + """ + saved_settings = load_check_settings(setting_file, TAB_NAME) + cols = saved_settings.get(settings_key, []) + default_display_cols = [col for col in cols if col in display_options] + + with st.expander(":material/clarify: Show more columns in report", expanded=False): + st.info(info_message) + display_cols = st.multiselect( + label="Select columns to display", + options=display_options, + default=default_display_cols, + key=widget_key, + on_change=trigger_save, + kwargs={"state_name": TAB_NAME + "_" + settings_key}, + ) + save_check_settings(setting_file, TAB_NAME, {settings_key: display_cols}) + + return display_cols + + +def _render_constraint_violations_table( + data: pl.DataFrame, + violation_data: pl.DataFrame, + settings: OutlierSettings, + setting_file: str, +) -> None: + """Render constraint violations table using Streamlit. + + Parameters + ---------- + data : pl.DataFrame + Original survey data. + violation_data : pl.DataFrame + DataFrame containing constraint violation data. + settings : OutlierSettings + Outlier settings configuration. + setting_file : str + Path to settings file. + """ + if violation_data.is_empty(): + st.info("No constraint violations detected.") + return + + all_columns = data.columns + + include_cols = _build_include_cols( + survey_key=settings.survey_key, + survey_id=settings.survey_id, + survey_date=settings.survey_date, + enumerator=settings.enumerator, + team=settings.team, + ) + + display_options = [col for col in all_columns if col not in include_cols] + + constraint_display_cols = _render_display_columns_expander( + setting_file, + "constraint_display_cols", + "constraint_violation_display_cols", + display_options, + "Select additional columns to include in the constraint violations report.", + ) + + if constraint_display_cols: + include_cols.extend(constraint_display_cols) + + # select columns to display from data + display_df = data.select(include_cols) + # sanitize violation_data to avoid column name conflicts + violation_df = sanitize_df_for_join( + main_df=display_df, + join_df=violation_data, + join_key=settings.survey_key, + ) + + display_df = display_df.join( + violation_df, + on=settings.survey_key, + how="inner", + ) + + # show only rows with violations + violations_df = display_df.filter(pl.col("violation reason") != "no violation") + + # add violation type column ie. "Soft Min", "Soft Max", "Hard Min", "Hard Max" + violation_type_expr = ( + pl.when(pl.col("violation reason").str.contains("below hard minimum")) + .then(pl.lit("Hard Min")) + .when(pl.col("violation reason").str.contains("below soft minimum")) + .then(pl.lit("Soft Min")) + .when(pl.col("violation reason").str.contains("above soft maximum")) + .then(pl.lit("Soft Max")) + .when(pl.col("violation reason").str.contains("above hard maximum")) + .then(pl.lit("Hard Max")) + .otherwise(pl.lit("Unknown")) + ) + + violations_df = violations_df.with_columns( + violation_type_expr.alias("violation type") + ) + + st.dataframe(violations_df) + + +def _render_outlier_table( + data: pl.DataFrame, + outliers_data: pl.DataFrame, + settings: OutlierSettings, + setting_file: str, +) -> None: + """Render outlier data table using Streamlit. + + Parameters + ---------- + data : pl.DataFrame + Original survey data. + outliers_data : pl.DataFrame + DataFrame containing outlier data. + settings : OutlierSettings + Outlier settings configuration. + setting_file : str + Path to settings file. + """ + if outliers_data.is_empty(): + st.info("No outliers detected in the selected columns.") + return + + all_columns = data.columns + + include_cols = _build_include_cols( + survey_key=settings.survey_key, + survey_id=settings.survey_id, + survey_date=settings.survey_date, + enumerator=settings.enumerator, + team=settings.team, + ) + + display_options = [col for col in all_columns if col not in include_cols] + + outlier_display_cols = _render_display_columns_expander( + setting_file, + "outlier_display_cols", + "outlier_display_cols", + display_options, + "Select additional columns to include in the outlier report.", + ) + + if outlier_display_cols: + include_cols.extend(outlier_display_cols) + + # select columns to display from data + display_df = data.select(include_cols) + outliers_df = sanitize_df_for_join(display_df, outliers_data, settings.survey_key) + display_df = display_df.join( + outliers_df, + on=settings.survey_key, + how="inner", + ) + + # show only rows with outliers + outlier_show_df = display_df.filter(pl.col("outlier reason") != "no outlier") + + st.dataframe(outlier_show_df) + + +def _render_outlier_column_inspection( + data: pl.DataFrame, + outliers_data: pl.DataFrame, + settings: OutlierSettings, + setting_file: str, +) -> None: + """Inspect outlier columns in the DataFrame. + + Parameters + ---------- + data : pl.DataFrame + DataFrame containing the survey data. + outliers_data : pl.DataFrame + DataFrame containing outlier detection results. + settings : OutlierSettings + Outlier settings configuration. + setting_file : str + Path to settings file. + """ + if outliers_data.is_empty(): + st.info( + "No outlier columns selected. Please select outlier columns to inspect." + ) + return + + all_columns = data.columns + + include_cols = _build_include_cols( + survey_key=settings.survey_key, + survey_id=settings.survey_id, + survey_date=settings.survey_date, + enumerator=settings.enumerator, + team=settings.team, + ) + + # list of outlier columns checked + columns_checked_list = ( + outliers_data.select("column name").unique().to_series().to_list() + ) + + ic1, _ = st.columns([0.2, 0.8]) + + with ic1: + # get saved settings + saved_settings = load_check_settings(setting_file, TAB_NAME) + default_selected_col = saved_settings.get("selected_col", None) + default_selected_col_index = ( + columns_checked_list.index(default_selected_col) + if default_selected_col and default_selected_col in columns_checked_list + else None + ) + selected_col = st.selectbox( + label="Select outlier columns to inspect", + options=columns_checked_list, + index=default_selected_col_index, + key="outlier_inspect_col", + help="Select the outlier columns to inspect. " + "You can only select one column at a time.", + on_change=trigger_save, + kwargs={"state_name": TAB_NAME + "_selected_col"}, + ) + save_check_settings(setting_file, TAB_NAME, {"selected_col": selected_col}) + + if not selected_col: + st.info("Select an outlier column to inspect.") + return + + if selected_col not in data.columns: + raise ValueError( + f"Selected column '{selected_col}' is not present in the data. " + "Please select a valid column." + ) + else: + include_cols.append(selected_col) + + # create a subset of the data + column_data = data.select([selected_col]) + + st.subheader(f"Details/Distribution for {selected_col} values") + dc1, _, dc3 = st.columns([0.3, 0.1, 0.6]) + with dc1: + desc_stats = _create_descriptive_stats(column_data) + st.dataframe(desc_stats) + + with dc3: + box_plot = _create_box_plot( + data=column_data[selected_col].to_pandas(), + title=selected_col, + ) + st.plotly_chart(box_plot, width="stretch") + + with st.expander(":material/clarify: Show more columns in report", expanded=False): + st.info( + "Select additional columns to include in the outlier inspection report." + ) + display_options = [ + col + for col in all_columns + if col not in include_cols and col != selected_col + ] + inspect_display_cols = st.multiselect( + label="Select columns to display", + options=display_options, + default=None, + help="Select the columns to display in the inspection table.", + disabled=not selected_col, + ) + + if inspect_display_cols: + include_cols.extend(inspect_display_cols) + + # select columns to display from data + display_df = data.select(include_cols) + outliers_df = sanitize_df_for_join(display_df, outliers_data, settings.survey_key) + display_df = display_df.join( + outliers_df, + on=settings.survey_key, + how="inner", + ) + + st.dataframe( + display_df, + width="stretch", + hide_index=False, + ) + + +# ============================================================================= +# Streamlit UI - Column Search/Configuration Widgets +# ============================================================================= + + +def _create_search_type_info(search_type_param: str) -> None: + """Display info based on the selected search type. + + Parameters + ---------- + search_type_param : str + The search type to display info for. + """ + info_messages = { + SearchType.EXACT.value: "Select columns that match the exact name. " + "You may select multiple columns.", + SearchType.STARTSWITH.value: "Select columns that start with the specified pattern. " + "You will have to enter the pattern in the input box below.", + SearchType.ENDSWITH.value: "Select columns that end with the specified pattern. " + "You will have to enter the pattern in the input box below.", + SearchType.CONTAINS.value: "Select columns that contain the specified pattern. " + "You will have to enter the pattern in the input box below.", + SearchType.REGEX.value: "Select columns that match the specified regex pattern. " + "You will have to enter the pattern in the input box below.", + } + + st.info(info_messages.get(search_type_param, "Unknown search type.")) + + +def _render_search_type_selection( + numeric_columns: list[str], +) -> tuple[str, str | None, list[str], bool]: + """Render search type selection UI. + + Parameters + ---------- + numeric_columns : list[str] + List of numeric columns. + + Returns + ------- + tuple[str, str | None, list[str], bool] + Search type, pattern, selected columns, and lock_cols flag. + """ + search_type_options = [e.value for e in SearchType] + search_type = st.selectbox( + label="Search type", + options=search_type_options, + index=0, + help="Select the type of search to perform on the column names.", + ) + + _create_search_type_info(search_type) + + if search_type == SearchType.EXACT.value: + outlier_cols_sel = st.multiselect( + label="Select columns to check", + options=numeric_columns, + default=None, + help="Select column or group of columns to check for outliers.", + ) + pattern, lock_cols = None, None + return search_type, pattern, outlier_cols_sel, lock_cols + else: + pattern = st.text_input( + label="Enter pattern to match column names", + placeholder="Enter pattern to match column names", + help="Enter the pattern to match column names based on the " + "selected search type.", + ) + if pattern: + outlier_cols_patt = expand_col_names( + numeric_columns, pattern, search_type=search_type + ) + else: + outlier_cols_patt = [] + + st.write( + "**Columns Selected:** ", + ", ".join(outlier_cols_patt) if outlier_cols_patt else "None", + ) + return search_type, pattern, outlier_cols_patt, None + + +def _render_column_grouping_options( + outlier_cols: list[str], search_type: str +) -> tuple[bool, bool]: + """Render column grouping and locking options. + + Parameters + ---------- + outlier_cols : list[str] + Selected outlier columns. + search_type : str + Search type used. + + Returns + ------- + tuple[bool, bool] + Group columns flag and lock columns flag. + """ + gc1, gc2 = st.columns([0.5, 0.5]) + with gc1: + group_cols = st.toggle( + label="Group columns", + key="group_outlier_cols", + help="Group selected columns together for outlier detection.", + disabled=not outlier_cols or len(outlier_cols) < 2, + ) + with gc2: + lock_cols = st.toggle( + label="Lock column selection", + key="outlier_cols_lock", + help="Lock the selected columns to prevent changes.", + disabled=not outlier_cols + or len(outlier_cols) < 2 + or search_type == SearchType.EXACT.value, + ) + return group_cols, lock_cols + + +def _render_outlier_options() -> tuple[bool, dict | None, bool]: + """Render outlier detection options UI. + + Returns + ------- + tuple[bool, dict | None, bool] + Enable outliers flag, outlier settings dict, and validation status. + """ + with st.container(border=True): + st.write("**Outlier Options:**") + enable_outliers = st.toggle( + "Enable Outlier Checks", key="enable_coutlier", value=True + ) + if enable_outliers: + oc1, oc2 = st.columns([0.5, 0.5]) + with oc1: + outlier_method = st.selectbox( + label="Select outlier detection method", + options=[e.value for e in OutlierMethod], + index=0, + help="Select the method to use for outlier detection.", + key="outlier_method", + ) + with oc2: + default_multiplier = ( + OutlierMultipliers.IQR.value + if outlier_method == OutlierMethod.IQR.value + else OutlierMultipliers.SD.value + ) + outlier_multiplier = st.number_input( + label="Select multiplier for outlier detection", + min_value=0.1, + max_value=10.0, + value=default_multiplier, + step=0.1, + help="Select the multiplier to use for outlier detection.", + key="outlier_multiplier", + ) + + outlier_threshold_default = ( + OutlierThresholds.SD.value + if outlier_method == OutlierMethod.SD.value + else OutlierThresholds.IQR.value + ) + outlier_threshold = st.number_input( + label="Outlier threshold (%)", + min_value=1, + value=outlier_threshold_default, + help="Set the minimum number values required to flag outliers in the column.", + key="outlier_threshold", + ) + + outlier_settings, valid_outlier = _validate_outlier_settings( + { + "outlier_method": outlier_method, + "outlier_multiplier": outlier_multiplier, + "outlier_threshold": outlier_threshold, + } + ) + return enable_outliers, outlier_settings, valid_outlier + else: + return False, None, True + + +def _render_constraint_options() -> tuple[dict, bool]: + """Render constraint bounds options UI. + + Returns + ------- + tuple[dict, bool] + Constraint settings dict and validation status. + """ + with st.container(border=True): + st.write("**Constraint Options:**") + + hc1, hc2 = st.columns(2) + with hc1: + hard_min = st.number_input( + label="(OPTIONAL) Hard minimum", + help="(OPTIONAL) Hard minimum value for outlier detection.", + value=None, + ) + with hc2: + hard_max = st.number_input( + label="(OPTIONAL) Hard maximum", + help="(OPTIONAL) Hard maximum value for outlier detection.", + value=None, + ) + + sc1, sc2 = st.columns(2) + with sc1: + soft_min = st.number_input( + label="(OPTIONAL) Soft minimum", + help="(OPTIONAL) Soft minimum value for outlier detection.", + value=None, + ) + with sc2: + soft_max = st.number_input( + label="(OPTIONAL) Soft maximum", + help="(OPTIONAL) Soft maximum value for outlier detection.", + value=None, + ) + + return _validate_constraint_settings( + { + "hard_min": hard_min, + "soft_min": soft_min, + "soft_max": soft_max, + "hard_max": hard_max, + } + ) + + +# ============================================================================= +# Streamlit UI - Column Configuration (CRUD) +# ============================================================================= +# +# NOTE: although these functions read as "settings" (they configure outlier +# columns), they are only ever invoked from `outliers_report` via +# `_render_outlier_column_actions` -- never from `outliers_report_settings` -- +# so they live here in report_ui rather than in settings_ui. + + +def _render_outlier_column_actions( + project_id: str, page_name_id: str, numeric_columns: list[str] +) -> None: + """Render the outlier column configuration UI. + + Parameters + ---------- + project_id : str + Project identifier. + page_name_id : str + Page name identifier. + numeric_columns : list[str] + List of numeric columns. + """ + outlier_settings = duckdb_get_table( + project_id, + f"outliers_{page_name_id}", + "logs", + ) + + os1, os2, _ = st.columns([0.4, 0.3, 0.3]) + with os1: + st.button( + "Add Outlier/Constraint Column", + key="add_outlier_column", + help="Add a new outlier column configuration.", + width="stretch", + type="primary", + on_click=_add_outlier_column, + args=( + project_id, + page_name_id, + numeric_columns, + ), + ) + with os2: + _delete_outlier_column(project_id, page_name_id, outlier_settings) + + if outlier_settings.is_empty(): + st.info( + "Use the :material/add: button to add columns to check for outliers and the " + ":material/delete: button to remove columns." + ) + else: + _render_outlier_settings_table(outlier_settings) + + +@st.dialog("Add Outlier & Constraint Column(s)", width="medium") +def _add_outlier_column( + project_id: str, page_name_id: str, numeric_columns: list[str] +) -> None: + """Dialog to add a new outlier column configuration. + + Parameters + ---------- + project_id : str + Project identifier. + page_name_id : str + Page name identifier. + numeric_columns : list[str] + List of numeric columns. + """ + # Render search type selection + search_type, pattern, outlier_cols, lock_cols_initial = ( + _render_search_type_selection(numeric_columns) + ) + + if outlier_cols: + # Render grouping options + group_cols, lock_cols = _render_column_grouping_options( + outlier_cols, search_type + ) + if lock_cols_initial is not None: + lock_cols = lock_cols_initial + + # Render outlier options + enable_outliers, outlier_settings, valid_outlier = _render_outlier_options() + + # Render constraint options + constraint_settings, valid_constraint = _render_constraint_options() + + button_disabled = ( + not outlier_cols + or (enable_outliers and not valid_outlier) + or not valid_constraint + ) + if st.button( + "Add Outlier & Constraint Configuration", + key="confirm_add_outlier_column", + type="primary", + width="stretch", + disabled=button_disabled, + ): + _update_outlier_column_config( + project_id, + page_name_id, + search_type, + pattern, + outlier_cols, + group_cols, + lock_cols, + enable_outliers, + outlier_settings, + constraint_settings, + ) + + st.success("Outlier & Constraint configuration added successfully.") + st.rerun() + + +def _validate_settings( + settings: dict, + model_cls: type[BaseModel], + format_error: Callable[[ValidationError], str], +) -> tuple[BaseModel | None, bool]: + """Validate a settings dict against a Pydantic model, showing errors via st.error. + + Shared by ``_validate_constraint_settings`` and ``_validate_outlier_settings``, + which differ only in which model and error formatter they use. + + Parameters + ---------- + settings : dict + Dictionary of settings to validate. + model_cls : type[BaseModel] + Pydantic model class to validate against. + format_error : Callable[[ValidationError], str] + Function that converts a ValidationError into a user-friendly message. + + Returns + ------- + tuple[BaseModel | None, bool] + Validated model instance and validation status. + """ + try: + return model_cls(**settings), True + except ValidationError as e: + st.error(format_error(e)) + return None, False + + +def _validate_constraint_settings( + constraint_settings: dict, +) -> tuple[ConstraintBounds | None, bool]: + """Validate constraint settings using Pydantic model. + + Parameters + ---------- + constraint_settings : dict[str, Any] + Dictionary containing constraint settings. + + Returns + ------- + tuple[ConstraintBounds | None, bool] + Validated constraint settings and validation status. + """ + return _validate_settings( + constraint_settings, ConstraintBounds, _format_constraint_validation_error + ) + + +def _validate_outlier_settings( + outlier_settings: dict, +) -> tuple[OutlierOptionsConfig | None, bool]: + """Validate outlier settings using Pydantic model. + + Parameters + ---------- + outlier_settings : dict[str, Any] + Dictionary containing outlier settings. + + Returns + ------- + tuple[OutlierOptionsConfig | None, bool] + Validated outlier settings and validation status. + """ + return _validate_settings( + outlier_settings, OutlierOptionsConfig, _format_outlier_validation_error + ) + + +def _format_constraint_validation_error(e: ValidationError) -> str: + """Convert Pydantic ValidationError to user-friendly message. + + Parameters + ---------- + e : ValidationError + Pydantic validation error. + + Returns + ------- + str + User-friendly error message. + """ + errors = [] + for error in e.errors(): + field = " -> ".join(str(loc) for loc in error["loc"]) + msg = error["msg"] + + # Customize messages based on error type + if error["type"] == "float_not_finite": + errors.append( + f"• {field}: Value must be a finite number (not NaN or infinity)" + ) + elif error["type"] == "value_error": + errors.append(f"• {msg}") # Your custom validation messages + else: + errors.append(f"• {field}: {msg}") + + return "Invalid constraint configuration:\n" + "\n".join(errors) + + +def _format_outlier_validation_error(e: ValidationError) -> str: + """Convert Pydantic ValidationError to user-friendly message. + + Parameters + ---------- + e : ValidationError + Pydantic validation error. + + Returns + ------- + str + User-friendly error message. + """ + errors = [] + for error in e.errors(): + field = " -> ".join(str(loc) for loc in error["loc"]) + msg = error["msg"] + + # Customize messages based on error type + if error["type"] == "value_error.number.not_ge": + errors.append( + f"• {field}: Value must be greater than or equal to the minimum allowed." + ) + elif error["type"] == "value_error.number.not_le": + errors.append( + f"• {field}: Value must be less than or equal to the maximum allowed." + ) + else: + errors.append(f"• {field}: {msg}") + + return "Invalid outlier configuration:\n" + "\n".join(errors) + + +def _update_outlier_column_config( + project_id: str, + page_name_id: str, + search_type: str, + pattern: str | None, + outlier_cols: list[str], + group_cols: bool, + lock_cols: bool, + outlier_enabled: bool, + outlier_settings: OutlierOptionsConfig | None, + constraint_settings: ConstraintBounds | None, +) -> None: + """Update the outlier column configuration in the database. + + Parameters + ---------- + project_id : str + Project identifier. + page_name_id : str + Page name identifier. + search_type : str + Search type used. + pattern : str | None + Pattern for column matching. + outlier_cols : list[str] + Selected columns. + group_cols : bool + Whether to group columns. + lock_cols : bool + Whether to lock column selection. + outlier_enabled : bool + Whether outlier detection is enabled. + outlier_settings : OutlierOptionsConfig | None + Outlier detection settings. + constraint_settings : ConstraintBounds | None + Constraint bounds settings. + """ + # get existing config + existing_config = duckdb_get_table( + project_id, + f"outliers_{page_name_id}", + db_name="logs", + ) + + # Prepare new configurations + new_config = { + "search_type": search_type, + "pattern": pattern, + "column_name": [outlier_cols], + "grouped_columns": group_cols, + "locked": lock_cols, + "outlier_enabled": outlier_enabled, + "outlier_method": outlier_settings.outlier_method if outlier_settings else None, + "outlier_multiplier": outlier_settings.outlier_multiplier + if outlier_settings + else None, + "outlier_threshold": outlier_settings.outlier_threshold + if outlier_settings + else None, + "hard_min": constraint_settings.hard_min if constraint_settings else None, + "soft_min": constraint_settings.soft_min if constraint_settings else None, + "soft_max": constraint_settings.soft_max if constraint_settings else None, + "hard_max": constraint_settings.hard_max if constraint_settings else None, + } + + schema = { + "search_type": pl.Utf8, + "pattern": pl.Utf8, + "column_name": pl.List(pl.Utf8), + "grouped_columns": pl.Boolean, + "locked": pl.Boolean, + "outlier_enabled": pl.Boolean, + "outlier_method": pl.Utf8, + "outlier_multiplier": pl.Float64, + "outlier_threshold": pl.Int64, + "hard_min": pl.Float64, + "soft_min": pl.Float64, + "soft_max": pl.Float64, + "hard_max": pl.Float64, + } + + # Append new configurations to existing polars DataFrame + new_config_df = pl.DataFrame(new_config, schema=schema) + if not existing_config.is_empty(): + formatted_existing_config = _ensure_column_formats(existing_config) + updated_config = pl.concat( + [formatted_existing_config, new_config_df], how="vertical" + ) + else: + updated_config = new_config_df + + # Save updated configurations back to the database + duckdb_save_table( + project_id, + updated_config, + f"outliers_{page_name_id}", + db_name="logs", + ) + + +def _ensure_column_formats( + outlier_settings: pl.DataFrame, +) -> pl.DataFrame: + """Ensure correct data types for outlier settings DataFrame. + + Parameters + ---------- + outlier_settings : pl.DataFrame + Outlier settings configuration. + + Returns + ------- + pl.DataFrame + DataFrame with ensured data types. + """ + return outlier_settings.with_columns( + [ + pl.col("search_type").cast(pl.Utf8), + pl.col("pattern").cast(pl.Utf8), + pl.col("column_name").cast(pl.List(pl.Utf8)), + pl.col("grouped_columns").cast(pl.Boolean), + pl.col("locked").cast(pl.Boolean), + pl.col("outlier_enabled").cast(pl.Boolean), + pl.col("outlier_method").cast(pl.Utf8), + pl.col("outlier_multiplier").cast(pl.Float64), + pl.col("outlier_threshold").cast(pl.Int64), + pl.col("hard_min").cast(pl.Float64), + pl.col("soft_min").cast(pl.Float64), + pl.col("soft_max").cast(pl.Float64), + pl.col("hard_max").cast(pl.Float64), + ] + ) + + +def _render_outlier_settings_table(outlier_settings: pl.DataFrame) -> None: + """Render the outlier settings table in Streamlit. + + Parameters + ---------- + outlier_settings : pl.DataFrame + Outlier settings configuration. + """ + with st.expander("Outlier & Constraint Column Settings", expanded=False): + st.dataframe( + outlier_settings, + width="stretch", + hide_index=True, + column_config={ + "search_type": st.column_config.Column("Search Type"), + "pattern": st.column_config.Column("Pattern"), + "column_name": st.column_config.Column("Column Name(s)"), + "grouped_columns": st.column_config.CheckboxColumn("Grouped Columns"), + "locked": st.column_config.CheckboxColumn("Locked"), + "outlier_enabled": st.column_config.CheckboxColumn("Outlier Enabled"), + "outlier_method": st.column_config.Column("Outlier Method"), + "outlier_multiplier": st.column_config.NumberColumn( + "Outlier Multiplier" + ), + "outlier_threshold": st.column_config.NumberColumn("Outlier Threshold"), + "hard_min": st.column_config.NumberColumn("Hard Min"), + "soft_min": st.column_config.NumberColumn("Soft Min"), + "soft_max": st.column_config.NumberColumn("Soft Max"), + "hard_max": st.column_config.NumberColumn("Hard Max"), + }, + ) + + +def _delete_outlier_column( + project_id: str, page_name_id: str, outliers_settings: pl.DataFrame +) -> None: + """Render delete outlier column button and handle deletion. + + Parameters + ---------- + project_id : str + Project identifier. + page_name_id : str + Page name identifier. + outliers_settings : pl.DataFrame + Current outlier settings. + """ + with ( + st.popover( + label=":material/delete: Delete outlier column", + width="stretch", + ), + ): + st.markdown("#### Remove outlier columns") + + if outliers_settings.is_empty(): + st.info("No outlier columns have been added yet. ") + else: + outliers_settings = outliers_settings.with_row_index().with_columns( + ( + pl.col("index").cast(pl.Utf8) + + " - " + + pl.col("search_type") + + " - " + + pl.col("pattern").fill_null("") + ).alias("composite_index") + ) + + unique_index = ( + outliers_settings["composite_index"] + .unique(maintain_order=True) + .to_list() + ) + + selected_index = st.selectbox( + label="Select outlier column to remove", + options=unique_index, + help="Select the outlier column to remove from the list.", + ) + + if st.button( + label="Confirm deletion", + type="primary", + width="stretch", + help="Click to confirm deletion of the selected outlier column.", + key="confirm_delete_outlier_column", + disabled=not selected_index, + ): + updated_settings = outliers_settings.filter( + pl.col("composite_index") != selected_index + ).drop("composite_index") + + duckdb_save_table( + project_id, + updated_settings, + f"outliers_{page_name_id}", + "logs", + ) + + st.rerun() + + +# ============================================================================= +# Main Report Function +# ============================================================================= + + +def outliers_report( + project_id: str, + page_name_id: str, + data: pl.DataFrame, + setting_file: str, + config: dict, + survey_columns: ColumnByType, +) -> None: + """Create a comprehensive outliers report. + + Parameters + ---------- + project_id : str + The project identifier. + page_name_id : str + Page name identifier. + data : pd.DataFrame + DataFrame containing the survey data. + setting_file : str + Path to settings file. + config : dict + Configuration dictionary. + """ + # get column info + categorical_columns = survey_columns.categorical_columns + datetime_columns = survey_columns.datetime_columns + numeric_columns = survey_columns.numeric_columns + + st.title("Outliers and Constraints Report") + + if is_demo_project(): + demo_callout( + "This tab checks your survey data against two types of rules:\n\n" + "- **Constraint Violations**: Records that breach hard or soft numeric bounds " + "you define (e.g., age below 0 or above 120).\n" + "- **Outliers**: Records flagged by a statistical method (IQR or Standard " + "Deviation) as unusually high or low.\n\n" + "Start by reviewing the :material/settings: **settings** panel — your columns " + "are pre-filled. Then use **Add Outlier/Constraint Column** to configure " + "which columns to check." + ) + + # Load settings + config_settings = OutlierSettings(**config) + outliers_settings = outliers_report_settings( + setting_file, config_settings, categorical_columns, datetime_columns + ) + + # Outlier columns configuration + st.subheader("Outlier/Constraint Columns Configuration") + + if is_demo_project(): + demo_callout( + "Click **Add Outlier/Constraint Column** to select which numeric columns to " + "analyse. In the dialog that opens:\n\n" + "1. Leave **Search type** as **exact** and select **age** and " + "**household_count** from the column list.\n" + "2. Under **Outlier Options**, keep the default IQR method (multiplier 1.5, " + "threshold 20).\n" + "3. Under **Constraint Options**, optionally enter bounds — for example, " + "for **age** set **Hard Min = 0**, **Soft Min = 15**, **Soft Max = 60**, " + "**Hard Max = 100**. Hard bounds flag impossible values; soft bounds flag " + "values that are unusual but may be legitimate.\n" + "4. Click **Add Outlier & Constraint Configuration** to save.\n\n" + "Repeat the steps to add **land_acre** as a second column." + ) + + _render_outlier_column_actions(project_id, page_name_id, numeric_columns) + + # get outlier column config + outliers_column_config = duckdb_get_table( + project_id, + f"outliers_{page_name_id}", + "logs", + ) + + if outliers_column_config.is_empty(): + return + + # update lock columns if needed + outliers_column_config = _update_unlocked_cols( + outliers_column_config, + categorical_columns, + ) + + # save updated config + duckdb_save_table( + project_id, + outliers_column_config, + f"outliers_{page_name_id}", + db_name="logs", + ) + + # Show constraint violations + st.write("---") + st.title("Constraint Violations") + + if is_demo_project(): + demo_callout( + "This section shows records that breach the **hard or soft bounds** you set " + "when configuring columns above.\n\n" + "- **Hard Min / Hard Max**: Absolute limits — any value outside these is an " + "unambiguous error (e.g., age < 0 or age > 120).\n" + "- **Soft Min / Soft Max**: Advisory range — values outside these are " + "unexpected but may be legitimate (e.g., a very large land holding).\n\n" + "Six metrics show how many records breach each bound type. " + "Use **:material/clarify: Show more columns in report** to add context " + "columns such as **enum_name** or **state** to the violations table." + ) + + # compute constraint violations + constraint_violations = compute_constraint_violations( + data, + outliers_settings, + outliers_column_config, + ) + + if constraint_violations.is_empty(): + st.info("No constraint violations detected.") + + else: + # show constraint metrics + _render_constraint_metrics(constraint_violations) + + # show constraint violations table + st.subheader("Constraint Violations Details") + _render_constraint_violations_table( + data, + constraint_violations, + outliers_settings, + setting_file, + ) + + # show outliers metrics + st.write("---") + st.title("Outliers") + + if is_demo_project(): + demo_callout( + "This section flags records that fall outside the **statistical bounds** " + "computed by the method you chose (IQR or Standard Deviation).\n\n" + "Four metrics summarise the findings: **Columns Checked**, " + "**Columns with Outliers**, **Total Outliers**, and " + "**Enumerators with Outliers**.\n\n" + "Under **Inspect Columns**, select a column from the dropdown to see " + "its descriptive statistics and a **box plot** showing where flagged values " + "sit relative to the distribution. Expand " + "**:material/clarify: Show more columns in report** to add context columns " + "to the record table below the chart." + ) + + # Compute outliers + outlier_data = compute_outlier_output( + data, + outliers_settings, + outliers_column_config, + ) + + if outlier_data.is_empty(): + st.info("No outliers detected.") + + else: + # show outlier metrics + _render_outlier_metrics(outlier_data, outliers_settings) + + # show outlier column inspection + st.subheader("Inspect Columns") + + _render_outlier_column_inspection( + data, + outlier_data, + outliers_settings, + setting_file, + ) + + demo_callout( + "**Next**: :material/arrow_upward: Scroll up and select the **GPS Checks** tab." + ) diff --git a/src/datasure/checks/outliers/settings_ui.py b/src/datasure/checks/outliers/settings_ui.py new file mode 100644 index 00000000..591fca80 --- /dev/null +++ b/src/datasure/checks/outliers/settings_ui.py @@ -0,0 +1,159 @@ +"""Settings UI for the outliers report.""" + +import streamlit as st + +from datasure.checks.outliers.compute import load_default_settings +from datasure.checks.outliers.models import TAB_NAME, OutlierSettings +from datasure.utils.onboarding_utils import demo_output_onboarding +from datasure.utils.settings_utils import save_check_settings, trigger_save + + +@demo_output_onboarding(TAB_NAME) +def outliers_report_settings( + settings_file: str, + config: OutlierSettings, + categorical_columns: list[str], + datetime_columns: list[str], +) -> OutlierSettings: + """Create a settings UI for outliers report configuration. + + This function creates the comprehensive Streamlit UI for configuring + outlier detection settings. Due to its complexity (UI rendering), + it maintains a higher cognitive complexity but is well-structured. + + Parameters + ---------- + settings_file : str + Path to settings file. + config : OutlierSettings + Default configuration. + categorical_columns : list[str] + List of categorical columns. + datetime_columns : list[str] + List of datetime columns. + + Returns + ------- + OutlierSettings + User-configured settings. + """ + with st.expander("settings", icon=":material/settings:"): + st.markdown("## Configure settings for outliers report") + st.write("---") + + # Load default settings + default_settings = load_default_settings(settings_file, config) + + # Survey Identifiers + with st.container(border=True): + st.markdown("#### Survey Identifiers") + si1, si2, _ = st.columns(3) + + with si1: + default_survey_key = default_settings.survey_key + default_survey_key_index = ( + categorical_columns.index(default_survey_key) + if default_survey_key and default_survey_key in categorical_columns + else None + ) + survey_key = st.selectbox( + "Survey Key", + options=categorical_columns, + key="survey_key_outliers", + help="Select the column that contains the survey key", + index=default_survey_key_index, + on_change=trigger_save, + kwargs={"state_name": TAB_NAME + "_survey_key"}, + ) + save_check_settings(settings_file, TAB_NAME, {"survey_key": survey_key}) + + with si2: + default_survey_id = default_settings.survey_id + default_survey_id_index = ( + categorical_columns.index(default_survey_id) + if default_survey_id and default_survey_id in categorical_columns + else None + ) + survey_id = st.selectbox( + "Survey ID", + options=categorical_columns, + help="Select the column that contains the survey ID", + key="survey_id_outliers", + index=default_survey_id_index, + on_change=trigger_save, + kwargs={"state_name": TAB_NAME + "_survey_id"}, + ) + save_check_settings(settings_file, TAB_NAME, {"survey_id": survey_id}) + + with st.container(border=True): + st.markdown("#### Survey Date") + + sd1, _, _ = st.columns(3) + + with sd1: + default_survey_date = default_settings.survey_date + default_survey_date_index = ( + datetime_columns.index(default_survey_date) + if default_survey_date and default_survey_date in datetime_columns + else None + ) + + survey_date = st.selectbox( + "Survey Date", + options=datetime_columns, + help="Select the column that contains the survey date", + key="survey_date_outliers", + index=default_survey_date_index, + on_change=trigger_save, + kwargs={"state_name": TAB_NAME + "_survey_date"}, + ) + save_check_settings( + settings_file, TAB_NAME, {"survey_date": survey_date} + ) + + with st.container(border=True): + st.markdown("#### Enumerator & Team") + ec1, ec2, _ = st.columns(3) + with ec1: + default_enumerator = default_settings.enumerator + default_enumerator_index = ( + categorical_columns.index(default_enumerator) + if default_enumerator and default_enumerator in categorical_columns + else None + ) + enumerator = st.selectbox( + "Enumerator ID", + options=categorical_columns, + key="enumerator_outliers", + help="Select the column that contains the enumerator ID", + index=default_enumerator_index, + on_change=trigger_save, + kwargs={"state_name": TAB_NAME + "_enumerator"}, + ) + save_check_settings(settings_file, TAB_NAME, {"enumerator": enumerator}) + + with ec2: + default_team = default_settings.team + default_team_index = ( + categorical_columns.index(default_team) + if default_team and default_team in categorical_columns + else None + ) + team = st.selectbox( + "Team ID", + options=categorical_columns, + key="team_outliers", + help="Select the column that contains the team ID", + index=default_team_index, + on_change=trigger_save, + kwargs={"state_name": TAB_NAME + "_team"}, + ) + save_check_settings(settings_file, TAB_NAME, {"team": team}) + + return OutlierSettings( + survey_key=survey_key, + survey_id=survey_id, + survey_date=survey_date, + enumerator=enumerator, + team=team, + ) diff --git a/src/datasure/views/output_view_template.py b/src/datasure/views/output_view_template.py index 5ad17e36..8f041223 100644 --- a/src/datasure/views/output_view_template.py +++ b/src/datasure/views/output_view_template.py @@ -20,7 +20,7 @@ from datasure.checks.enumerator.report_ui import enumerator_report from datasure.checks.gpschecks.report_ui import gpschecks_report from datasure.checks.missing import missing_report -from datasure.checks.outliers import outliers_report +from datasure.checks.outliers.report_ui import outliers_report from datasure.checks.progress import progress_report from datasure.checks.summary import summary_report from datasure.utils.cache_utils import get_cache_path diff --git a/tests/checks/outliers/__init__.py b/tests/checks/outliers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/checks/outliers/conftest.py b/tests/checks/outliers/conftest.py new file mode 100644 index 00000000..9b0e9736 --- /dev/null +++ b/tests/checks/outliers/conftest.py @@ -0,0 +1,123 @@ +"""Shared fixtures for the split outliers module test suite.""" + +from unittest.mock import MagicMock + +import pandas as pd +import polars as pl +import pytest + +from datasure.checks.outliers.models import OutlierSettings + +# ============================================================================ +# MOCK STREAMLIT HELPERS +# ============================================================================ + + +def _columns_side_effect(*args, **kwargs): + """Return a list of MagicMocks whose length matches the columns argument.""" + n = args[0] if args else 1 + count = len(n) if isinstance(n, list | tuple) else int(n) + return [MagicMock() for _ in range(count)] + + +def _make_st_mock(): + """Create a MagicMock for streamlit with sensible defaults. + + Configures ``columns`` (matches the requested column count), + ``cache_data`` (identity passthrough, with or without kwargs), and + ``dialog`` (identity passthrough) so modules decorated with + ``@st.cache_data``/``@st.dialog`` can be reloaded against this mock + without their decorated behavior changing. + """ + st_mock = MagicMock() + st_mock.columns.side_effect = _columns_side_effect + + def _mock_cache_data(func=None, **kwargs): + if callable(func): + return func + return lambda f: f + + st_mock.cache_data = _mock_cache_data + st_mock.dialog = lambda *args, **kwargs: lambda f: f + return st_mock + + +# ============================================================================ +# FIXTURES +# ============================================================================ + + +@pytest.fixture(autouse=True) +def mock_database_functions(monkeypatch): + """Override the autouse fixture from conftest. + + Disables database mocking for these tests. + """ + pass + + +@pytest.fixture +def sample_polars_df(): + """Create a sample Polars DataFrame for testing.""" + from datetime import date + + return pl.DataFrame( + { + "survey_key": ["K001", "K002", "K003", "K004", "K005"], + "survey_id": ["S001", "S002", "S003", "S004", "S005"], + "enumerator": ["E001", "E002", "E001", "E003", "E002"], + "team": ["T1", "T2", "T1", "T3", "T2"], + "survey_date": [date(2024, 1, i) for i in range(1, 6)], + "numeric_col1": [1.0, 2.0, 3.0, 100.0, 5.0], # outlier: 100.0 + "numeric_col2": [10.0, 20.0, 30.0, 40.0, 500.0], # outlier: 500.0 + "string_col": ["A", "B", "C", "D", "E"], + } + ) + + +@pytest.fixture +def sample_pandas_df(): + """Create a sample pandas DataFrame for testing.""" + return pd.DataFrame( + { + "survey_key": ["K001", "K002", "K003", "K004", "K005"], + "survey_id": ["S001", "S002", "S003", "S004", "S005"], + "enumerator": ["E001", "E002", "E001", "E003", "E002"], + "numeric_col1": [1.0, 2.0, 3.0, 100.0, 5.0], + "numeric_col2": [10.0, 20.0, 30.0, 40.0, 500.0], + } + ) + + +@pytest.fixture +def outlier_column_config(): + """Create sample outlier column configuration.""" + return pl.DataFrame( + { + "search_type": ["exact"], + "pattern": [None], + "column_name": [["numeric_col1"]], + "grouped_columns": [False], + "locked": [False], + "outlier_enabled": [True], + "outlier_method": ["Interquartile Range (IQR)"], + "outlier_multiplier": [1.5], + "outlier_threshold": [3], + "hard_min": [None], + "soft_min": [0.0], + "soft_max": [50.0], + "hard_max": [None], + } + ) + + +@pytest.fixture +def outlier_settings(): + """Create sample outlier settings.""" + return OutlierSettings( + survey_key="survey_key", + survey_id="survey_id", + survey_date="survey_date", + enumerator="enumerator", + team="team", + ) diff --git a/tests/checks/test_outliers.py b/tests/checks/outliers/test_compute.py similarity index 53% rename from tests/checks/test_outliers.py rename to tests/checks/outliers/test_compute.py index 9b6e5242..2c96c0a9 100644 --- a/tests/checks/test_outliers.py +++ b/tests/checks/outliers/test_compute.py @@ -1,410 +1,77 @@ -"""Comprehensive tests for the outliers module with 100% code coverage.""" +"""Tests for datasure.checks.outliers.compute.""" -# Standard library imports import importlib import sys -from unittest.mock import MagicMock, patch +from unittest.mock import patch -# Third-party imports import pandas as pd import polars as pl import pytest -from pydantic import ValidationError -# Local application imports -from datasure.checks.outliers import ( - # Pydantic Models - ConstraintBounds, - ConstraintMetrics, - OutlierBounds, - OutlierColumnConfig, - # Enums - OutlierMethod, - OutlierMetrics, - OutlierOptionsConfig, - OutlierSettings, - OutlierStatistics, - SearchType, - # Refactored helper functions +from datasure.checks.outliers.compute import ( _add_statistics_columns, _build_include_cols, _build_outlier_expression, _compute_column_stats, + _compute_constraint_metrics, _compute_iqr_bounds, + _compute_outlier_metrics, _compute_sd_bounds, _compute_single_column_stats, - # Rendering/UI helpers - _create_search_type_info, - _delete_outlier_column, - _ensure_column_formats, _ensure_list, - _format_constraint_validation_error, - _format_outlier_validation_error, _merge_outlier_results, _process_outlier_configs, _process_single_column_outliers, _process_single_config, - _render_column_grouping_options, - _render_constraint_metrics, - _render_constraint_options, - _render_constraint_violations_table, - _render_outlier_column_actions, - _render_outlier_column_inspection, - _render_outlier_metrics, - _render_outlier_options, - _render_outlier_settings_table, - _render_outlier_table, - _render_search_type_selection, _should_expand_row, - _update_outlier_column_config, _update_unlocked_cols, - _validate_constraint_settings, - _validate_outlier_settings, - # Statistical functions compute_column_outlier_summary, compute_constraint_violations, compute_outlier_output, compute_outlier_stats_polars, - # Utility / top-level functions expand_col_names, get_outlier_cols, - # Settings functions load_default_settings, - outliers_report, - safe_to_numeric, stack_outlier_columns, update_unlocked_cols, ) +from datasure.checks.outliers.models import ( + OutlierBounds, + OutlierMethod, + OutlierSettings, + OutlierStatistics, +) from datasure.utils.dataframe_utils import ( convert_dataframe_column_to_numeric, convert_series_to_numeric, + safe_to_numeric, sanitize_df_for_join, ) +from tests.checks.outliers.conftest import _make_st_mock # ============================================================================ -# FIXTURES +# COMPUTE_MOD FIXTURE (reload compute with mocked streamlit for decorator tests) # ============================================================================ -@pytest.fixture(autouse=True) -def mock_database_functions(monkeypatch): - """Override the autouse fixture from conftest. - - Disables database mocking for these tests. - """ - pass - - @pytest.fixture -def sample_polars_df(): - """Create a sample Polars DataFrame for testing.""" - from datetime import date - - return pl.DataFrame( - { - "survey_key": ["K001", "K002", "K003", "K004", "K005"], - "survey_id": ["S001", "S002", "S003", "S004", "S005"], - "enumerator": ["E001", "E002", "E001", "E003", "E002"], - "team": ["T1", "T2", "T1", "T3", "T2"], - "survey_date": [date(2024, 1, i) for i in range(1, 6)], - "numeric_col1": [1.0, 2.0, 3.0, 100.0, 5.0], # outlier: 100.0 - "numeric_col2": [10.0, 20.0, 30.0, 40.0, 500.0], # outlier: 500.0 - "string_col": ["A", "B", "C", "D", "E"], - } - ) - - -@pytest.fixture -def sample_pandas_df(): - """Create a sample pandas DataFrame for testing.""" - return pd.DataFrame( - { - "survey_key": ["K001", "K002", "K003", "K004", "K005"], - "survey_id": ["S001", "S002", "S003", "S004", "S005"], - "enumerator": ["E001", "E002", "E001", "E003", "E002"], - "numeric_col1": [1.0, 2.0, 3.0, 100.0, 5.0], - "numeric_col2": [10.0, 20.0, 30.0, 40.0, 500.0], - } - ) - - -@pytest.fixture -def outlier_column_config(): - """Create sample outlier column configuration.""" - return pl.DataFrame( - { - "search_type": ["exact"], - "pattern": [None], - "column_name": [["numeric_col1"]], - "grouped_columns": [False], - "locked": [False], - "outlier_enabled": [True], - "outlier_method": ["Interquartile Range (IQR)"], - "outlier_multiplier": [1.5], - "outlier_threshold": [3], - "hard_min": [None], - "soft_min": [0.0], - "soft_max": [50.0], - "hard_max": [None], - } - ) - - -@pytest.fixture -def outlier_settings(): - """Create sample outlier settings.""" - return OutlierSettings( - survey_key="survey_key", - survey_id="survey_id", - survey_date="survey_date", - enumerator="enumerator", - team="team", - ) - - -# ============================================================================ -# PYDANTIC MODEL TESTS -# ============================================================================ - - -class TestOutlierBounds: - """Test OutlierBounds Pydantic model.""" - - def test_valid_bounds(self): - """Test creating valid outlier bounds.""" - bounds = OutlierBounds(lower_bound=0.0, upper_bound=100.0) - assert bounds.lower_bound == 0.0 - assert bounds.upper_bound == 100.0 +def compute_mod(): + """Reload compute with mocked Streamlit to strip @st.cache_data decorators.""" + mock_st = _make_st_mock() + original_st = sys.modules.get("streamlit") + sys.modules["streamlit"] = mock_st - def test_negative_bounds(self): - """Test bounds with negative values.""" - bounds = OutlierBounds(lower_bound=-50.0, upper_bound=50.0) - assert bounds.lower_bound == -50.0 - assert bounds.upper_bound == 50.0 + import datasure.checks.outliers.compute as compute_module - -class TestOutlierOptionsConfig: - """Test OutlierOptionsConfig Pydantic model.""" - - def test_valid_config(self): - """Test creating valid outlier options config.""" - config = OutlierOptionsConfig( - outlier_method=OutlierMethod.IQR, - outlier_multiplier=1.5, - outlier_threshold=20, - ) - assert config.outlier_method == OutlierMethod.IQR - assert config.outlier_multiplier == 1.5 - assert config.outlier_threshold == 20 - - def test_invalid_multiplier_zero(self): - """Test that zero multiplier raises validation error.""" - with pytest.raises(ValidationError): - OutlierOptionsConfig( - outlier_method=OutlierMethod.IQR, - outlier_multiplier=0.0, - outlier_threshold=20, - ) - - def test_invalid_multiplier_negative(self): - """Test that negative multiplier raises validation error.""" - with pytest.raises(ValidationError): - OutlierOptionsConfig( - outlier_method=OutlierMethod.IQR, - outlier_multiplier=-1.5, - outlier_threshold=20, - ) - - def test_invalid_threshold_zero(self): - """Test that zero threshold raises validation error.""" - with pytest.raises(ValidationError): - OutlierOptionsConfig( - outlier_method=OutlierMethod.IQR, - outlier_multiplier=1.5, - outlier_threshold=0, - ) - - -class TestConstraintBounds: - """Test ConstraintBounds Pydantic model.""" - - def test_valid_bounds_all_fields(self): - """Test creating valid constraint bounds with all fields.""" - bounds = ConstraintBounds( - hard_min=0.0, soft_min=10.0, soft_max=90.0, hard_max=100.0 - ) - assert bounds.hard_min == 0.0 - assert bounds.soft_min == 10.0 - assert bounds.soft_max == 90.0 - assert bounds.hard_max == 100.0 - - def test_valid_bounds_partial(self): - """Test creating valid constraint bounds with partial fields.""" - bounds = ConstraintBounds(soft_min=10.0, soft_max=90.0) - assert bounds.hard_min is None - assert bounds.soft_min == 10.0 - assert bounds.soft_max == 90.0 - assert bounds.hard_max is None - - def test_invalid_bounds_hierarchy(self): - """Test that invalid hierarchy raises validation error.""" - with pytest.raises(ValidationError, match="Bounds must follow hierarchy"): - ConstraintBounds( - hard_min=50.0, - soft_min=10.0, # hard_min > soft_min - ) - - def test_invalid_soft_bounds(self): - """Test that soft_min > soft_max raises validation error.""" - with pytest.raises(ValidationError): - ConstraintBounds(soft_min=90.0, soft_max=10.0) - - def test_negative_bounds(self): - """Test constraint bounds with negative values.""" - bounds = ConstraintBounds( - hard_min=-100.0, soft_min=-50.0, soft_max=50.0, hard_max=100.0 - ) - assert bounds.hard_min == -100.0 - - -class TestConstraintMetrics: - """Test ConstraintMetrics Pydantic model.""" - - def test_valid_metrics(self): - """Test creating valid constraint metrics.""" - metrics = ConstraintMetrics( - columns_checked=5, - total_violations=10, - hard_min_violations=2, - soft_min_violations=3, - soft_max_violations=3, - hard_max_violations=2, - ) - assert metrics.total_violations == 10 - - def test_negative_values_invalid(self): - """Test that negative values raise validation error.""" - with pytest.raises(ValidationError): - ConstraintMetrics( - columns_checked=-1, - total_violations=0, - hard_min_violations=0, - soft_min_violations=0, - soft_max_violations=0, - hard_max_violations=0, - ) - - -class TestOutlierMetrics: - """Test OutlierMetrics Pydantic model.""" - - def test_valid_metrics(self): - """Test creating valid outlier metrics.""" - metrics = OutlierMetrics( - columns_checked=5, - columns_with_outliers=3, - total_outliers=10, - enumerators_with_outliers=2, - ) - assert metrics.columns_checked == 5 - assert metrics.total_outliers == 10 - - -class TestOutlierStatistics: - """Test OutlierStatistics Pydantic model.""" - - def test_valid_statistics(self): - """Test creating valid outlier statistics.""" - stats = OutlierStatistics( - count=100, - min_value=0.0, - max_value=100.0, - mean=50.0, - median=48.0, - sd=15.0, - iqr=25.0, - lower_bound=10.0, - upper_bound=90.0, - ) - assert stats.count == 100 - assert stats.mean == 50.0 - assert stats.sd == 15.0 - - def test_alias_std(self): - """Test that 'sd' alias works for std field.""" - stats = OutlierStatistics( - count=100, - min_value=0.0, - max_value=100.0, - mean=50.0, - median=48.0, - sd=15.0, # Using alias - iqr=25.0, - lower_bound=10.0, - upper_bound=90.0, - ) - assert stats.sd == 15.0 - - -class TestOutlierColumnConfig: - """Test OutlierColumnConfig Pydantic model.""" - - def test_valid_config_exact(self): - """Test creating valid config with exact search.""" - config = OutlierColumnConfig( - search_type=SearchType.EXACT, - pattern=None, - outlier_cols=["col1", "col2"], - lock_cols=False, - grouped_cols=False, - outlier_method=OutlierMethod.IQR, - outlier_multiplier=1.5, - ) - assert config.search_type == SearchType.EXACT - - def test_invalid_pattern_required(self): - """Test that pattern is required for non-exact search types.""" - with pytest.raises(ValidationError): - OutlierColumnConfig( - search_type=SearchType.STARTSWITH, - pattern=None, # Should be required - outlier_cols=["col1"], - outlier_method=OutlierMethod.IQR, - outlier_multiplier=1.5, - ) - - def test_invalid_soft_bounds(self): - """Test that soft_max must be greater than soft_min.""" - with pytest.raises(ValidationError): - OutlierColumnConfig( - search_type=SearchType.EXACT, - outlier_cols=["col1"], - outlier_method=OutlierMethod.IQR, - outlier_multiplier=1.5, - soft_min=50.0, - soft_max=10.0, # Less than soft_min - ) - - -class TestOutlierSettings: - """Test OutlierSettings Pydantic model.""" - - def test_valid_settings(self): - """Test creating valid outlier settings.""" - settings = OutlierSettings( - survey_key="key", - survey_id="id", - survey_date="date", - enumerator="enum", - team="team", - ) - assert settings.survey_key == "key" - - def test_minimal_settings(self): - """Test creating minimal valid settings.""" - settings = OutlierSettings(survey_key="key") - assert settings.survey_key == "key" - assert settings.survey_id is None + try: + importlib.reload(compute_module) + yield compute_module + finally: + if original_st is not None: + sys.modules["streamlit"] = original_st + else: + sys.modules.pop("streamlit", None) + importlib.reload(compute_module) # ============================================================================ @@ -565,9 +232,6 @@ def test_stack_single_column(self, sample_polars_df): result = stack_outlier_columns(sample_polars_df, ["numeric_col1"]) assert result.len() == 5 - @pytest.mark.skip( - reason="Empty DataFrame causes Rust panic in Polars/Streamlit caching" - ) def test_stack_empty_dataframe(self): """Test with empty DataFrame.""" df = pl.DataFrame() @@ -705,7 +369,7 @@ def test_empty_dataframe(self): class TestLoadDefaultSettings: """Test load_default_settings function.""" - @patch("datasure.checks.outliers.load_check_settings") + @patch("datasure.checks.outliers.compute.load_check_settings") def test_load_with_saved_settings(self, mock_load): """Test loading with saved settings.""" mock_load.return_value = {"survey_id": "test_id"} @@ -714,7 +378,7 @@ def test_load_with_saved_settings(self, mock_load): assert result.survey_id == "test_id" assert result.survey_key == "key" - @patch("datasure.checks.outliers.load_check_settings") + @patch("datasure.checks.outliers.compute.load_check_settings") def test_load_with_empty_settings(self, mock_load): """Test loading with empty saved settings.""" mock_load.return_value = {} @@ -773,9 +437,6 @@ class TestGetOutlierCols: def test_with_numpy_array(self): """Test with numpy array in outlier_cols.""" import numpy as np - import pandas as pd - - from datasure.checks.outliers import get_outlier_cols df = pd.DataFrame( { @@ -790,10 +451,6 @@ def test_with_numpy_array(self): def test_with_list(self): """Test with list in outlier_cols.""" - import pandas as pd - - from datasure.checks.outliers import get_outlier_cols - df = pd.DataFrame( { "outlier_cols": [ @@ -808,9 +465,6 @@ def test_with_list(self): def test_mixed_types(self): """Test with mixed numpy array and list.""" import numpy as np - import pandas as pd - - from datasure.checks.outliers import get_outlier_cols df = pd.DataFrame( { @@ -829,8 +483,6 @@ class TestComputeMetrics: def test_compute_constraint_metrics(self): """Test _compute_constraint_metrics function.""" - from datasure.checks.outliers import _compute_constraint_metrics - violation_data = pl.DataFrame( { "column name": ["col1", "col1", "col2", "col3"], @@ -854,8 +506,6 @@ def test_compute_constraint_metrics(self): def test_compute_outlier_metrics_with_enumerator(self): """Test _compute_outlier_metrics with enumerator column.""" - from datasure.checks.outliers import _compute_outlier_metrics - outliers_data = pl.DataFrame( { "column name": ["col1", "col1", "col2"], @@ -877,8 +527,6 @@ def test_compute_outlier_metrics_with_enumerator(self): def test_compute_outlier_metrics_without_enumerator(self): """Test _compute_outlier_metrics without enumerator column.""" - from datasure.checks.outliers import _compute_outlier_metrics - outliers_data = pl.DataFrame( { "column name": ["col1", "col2"], @@ -990,76 +638,6 @@ def test_with_ungrouped_multiple_columns(self): assert len(result["column name"].unique()) == 2 -class TestConstraintValidation: - """Test constraint bounds validation.""" - - def test_constraint_bounds_all_none(self): - """Test ConstraintBounds with all None values.""" - bounds = ConstraintBounds() - assert bounds.hard_min is None - assert bounds.soft_min is None - assert bounds.soft_max is None - assert bounds.hard_max is None - - def test_constraint_bounds_partial(self): - """Test ConstraintBounds with partial values.""" - bounds = ConstraintBounds(soft_min=10, soft_max=100) - assert bounds.soft_min == 10 - assert bounds.soft_max == 100 - assert bounds.hard_min is None - assert bounds.hard_max is None - - def test_constraint_bounds_invalid_order(self): - """Test ConstraintBounds with invalid hierarchy.""" - with pytest.raises(ValidationError, match="must be <="): - ConstraintBounds(hard_min=100, soft_min=50) - - def test_constraint_bounds_negative_values(self): - """Test ConstraintBounds with negative values.""" - bounds = ConstraintBounds( - hard_min=-100, soft_min=-50, soft_max=50, hard_max=100 - ) - assert bounds.hard_min == -100 - assert bounds.soft_min == -50 - - -class TestOutlierColumnConfigValidation: - """Test OutlierColumnConfig validation edge cases.""" - - def test_pattern_required_for_non_exact(self): - """Test that pattern is required for non-exact search types.""" - with pytest.raises(ValidationError, match="Pattern is required"): - OutlierColumnConfig( - search_type=SearchType.STARTSWITH, - pattern=None, - outlier_cols=["col1"], - outlier_multiplier=1.5, - ) - - def test_soft_max_validation(self): - """Test soft_max must be greater than soft_min.""" - with pytest.raises(ValidationError, match="soft_max must be greater"): - OutlierColumnConfig( - search_type=SearchType.EXACT, - outlier_cols=["col1"], - outlier_multiplier=1.5, - soft_min=100, - soft_max=50, - ) - - def test_valid_config_with_constraints(self): - """Test valid configuration with all constraints.""" - config = OutlierColumnConfig( - search_type=SearchType.EXACT, - outlier_cols=["col1"], - outlier_multiplier=1.5, - soft_min=10, - soft_max=100, - ) - assert config.soft_min == 10 - assert config.soft_max == 100 - - class TestSafeToNumericEdgeCases: """Test safe_to_numeric edge cases.""" @@ -2307,68 +1885,9 @@ def test_sd_method_metadata(self): assert result["column name"][0] == "col" -# ============================================================================= -# HELPERS FOR ST-MOCKED TESTS -# ============================================================================= - - -def _columns_side_effect(*args, **kwargs): - """Return a list of MagicMocks whose length matches the columns argument.""" - n = args[0] if args else 1 - count = len(n) if isinstance(n, list | tuple) else int(n) - return [MagicMock() for _ in range(count)] - - -def _make_st_mock(): - """Create a MagicMock for streamlit with sensible defaults.""" - st_mock = MagicMock() - st_mock.columns.side_effect = _columns_side_effect - return st_mock - - -# ============================================================================= -# OUTLIERS_MOD FIXTURE (reimport with mocked streamlit) -# ============================================================================= - - -@pytest.fixture -def outliers_mod(): - """Reimport the outliers module with mocked streamlit for decorator tests.""" - orig = sys.modules.pop("datasure.checks.outliers", None) - orig_st = sys.modules.get("streamlit") - - st_mock = _make_st_mock() - - def mock_cache_data(func=None, **kwargs): - if callable(func): - return func - return lambda f: f - - st_mock.cache_data = mock_cache_data - st_mock.dialog = lambda *args, **kwargs: lambda f: f - - sys.modules["streamlit"] = st_mock - try: - with patch( - "datasure.utils.onboarding_utils.demo_output_onboarding", - lambda tab: lambda f: f, - ): - mod = importlib.import_module("datasure.checks.outliers") - sys.modules.pop("datasure.checks.outliers", None) - finally: - if orig_st is not None: - sys.modules["streamlit"] = orig_st - else: - sys.modules.pop("streamlit", None) - if orig is not None: - sys.modules["datasure.checks.outliers"] = orig - - return mod - - -# ============================================================================= +# ============================================================================ # TESTS: get_outlier_cols (list branch) -# ============================================================================= +# ============================================================================ class TestGetOutlierColsList: @@ -2391,50 +1910,50 @@ def test_empty_settings(self): assert get_outlier_cols(df) == [] -# ============================================================================= +# ============================================================================ # TESTS: stack_outlier_columns (empty-df branch via reimport) -# ============================================================================= +# ============================================================================ class TestStackOutlierColumnsEmpty: """Test stack_outlier_columns edge cases with empty or invalid data.""" - def test_raises_on_empty_df(self, outliers_mod): + def test_raises_on_empty_df(self, compute_mod): df = pl.DataFrame({"col": []}) with pytest.raises(ValueError, match="empty"): - outliers_mod.stack_outlier_columns(df, ["col"]) + compute_mod.stack_outlier_columns(df, ["col"]) - def test_raises_on_missing_column(self, outliers_mod): + def test_raises_on_missing_column(self, compute_mod): df = pl.DataFrame({"col1": [1.0, 2.0]}) with pytest.raises(ValueError, match="does not exist"): - outliers_mod.stack_outlier_columns(df, ["nonexistent"]) + compute_mod.stack_outlier_columns(df, ["nonexistent"]) - def test_raises_on_non_numeric_column(self, outliers_mod): + def test_raises_on_non_numeric_column(self, compute_mod): df = pl.DataFrame({"col": ["a", "b"]}) with pytest.raises(ValueError, match="cannot be converted"): - outliers_mod.stack_outlier_columns(df, ["col"]) + compute_mod.stack_outlier_columns(df, ["col"]) -# ============================================================================= +# ============================================================================ # TESTS: _create_box_plot and _create_descriptive_stats (via reimport) -# ============================================================================= +# ============================================================================ class TestCreateBoxPlot: """Test _create_box_plot function.""" - def test_returns_figure(self, outliers_mod): + def test_returns_figure(self, compute_mod): import plotly.graph_objects as go series = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0]) - fig = outliers_mod._create_box_plot(series, "Test Title") + fig = compute_mod._create_box_plot(series, "Test Title") assert isinstance(fig, go.Figure) - def test_figure_has_box_trace(self, outliers_mod): + def test_figure_has_box_trace(self, compute_mod): import plotly.graph_objects as go series = pd.Series([1.0, 2.0, 3.0]) - fig = outliers_mod._create_box_plot(series, "Col") + fig = compute_mod._create_box_plot(series, "Col") assert len(fig.data) == 1 assert isinstance(fig.data[0], go.Box) @@ -2442,1087 +1961,20 @@ def test_figure_has_box_trace(self, outliers_mod): class TestCreateDescriptiveStats: """Test _create_descriptive_stats function.""" - def test_returns_polars_dataframe(self, outliers_mod): + def test_returns_polars_dataframe(self, compute_mod): df = pl.DataFrame({"val": [1.0, 2.0, 3.0, 4.0, 5.0]}) - result = outliers_mod._create_descriptive_stats(df) + result = compute_mod._create_descriptive_stats(df) assert isinstance(result, pl.DataFrame) - def test_has_statistic_and_value_columns(self, outliers_mod): + def test_has_statistic_and_value_columns(self, compute_mod): df = pl.DataFrame({"val": [1.0, 2.0, 3.0]}) - result = outliers_mod._create_descriptive_stats(df) + result = compute_mod._create_descriptive_stats(df) assert "statistic" in result.columns assert "value" in result.columns - def test_renames_statistics(self, outliers_mod): + def test_renames_statistics(self, compute_mod): df = pl.DataFrame({"val": [1.0, 2.0, 3.0]}) - result = outliers_mod._create_descriptive_stats(df) + result = compute_mod._create_descriptive_stats(df) stat_names = result["statistic"].to_list() assert "Mean" in stat_names assert "Median (Q2)" in stat_names - - -# ============================================================================= -# TESTS: _validate_constraint_settings -# ============================================================================= - - -class TestValidateConstraintSettings: - """Test _validate_constraint_settings function.""" - - def test_valid_settings_returns_bounds_and_true(self): - result, valid = _validate_constraint_settings( - {"soft_min": 0.0, "soft_max": 100.0} - ) - assert valid is True - assert result is not None - - def test_invalid_hierarchy_returns_none_and_false(self): - with patch("datasure.checks.outliers.st") as st_mock: - result, valid = _validate_constraint_settings( - {"hard_min": 50.0, "soft_min": 10.0} - ) - assert valid is False - assert result is None - st_mock.error.assert_called_once() - - def test_all_none_settings_valid(self): - _result, valid = _validate_constraint_settings( - {"hard_min": None, "soft_min": None, "soft_max": None, "hard_max": None} - ) - assert valid is True - - -# ============================================================================= -# TESTS: _validate_outlier_settings -# ============================================================================= - - -class TestValidateOutlierSettings: - """Test _validate_outlier_settings function.""" - - def test_valid_settings_returns_config_and_true(self): - result, valid = _validate_outlier_settings( - { - "outlier_method": OutlierMethod.IQR.value, - "outlier_multiplier": 1.5, - "outlier_threshold": 20, - } - ) - assert valid is True - assert result is not None - - def test_invalid_multiplier_returns_none_and_false(self): - with patch("datasure.checks.outliers.st") as st_mock: - result, valid = _validate_outlier_settings( - { - "outlier_method": OutlierMethod.IQR.value, - "outlier_multiplier": 0.0, - "outlier_threshold": 20, - } - ) - assert valid is False - assert result is None - st_mock.error.assert_called_once() - - -# ============================================================================= -# TESTS: _format_constraint_validation_error -# ============================================================================= - - -class TestFormatConstraintValidationError: - """Test _format_constraint_validation_error function.""" - - def test_value_error_type(self): - try: - ConstraintBounds(hard_min=50.0, soft_min=10.0) - except ValidationError as e: - msg = _format_constraint_validation_error(e) - assert "Invalid constraint configuration" in msg - - def test_float_not_finite_type(self): - """Test float_not_finite error type produces finite-number message.""" - mock_error = MagicMock() - mock_error.errors.return_value = [ - { - "loc": ("hard_min",), - "msg": "value is not a finite number", - "type": "float_not_finite", - } - ] - msg = _format_constraint_validation_error(mock_error) - assert "Invalid constraint configuration" in msg - assert "finite number" in msg - - def test_value_error_type_uses_msg(self): - """Test value_error type includes the custom validation message.""" - mock_error = MagicMock() - mock_error.errors.return_value = [ - { - "loc": ("hard_min",), - "msg": "Bounds must follow hierarchy", - "type": "value_error", - } - ] - msg = _format_constraint_validation_error(mock_error) - assert "Bounds must follow hierarchy" in msg - - def test_other_error_type(self): - """Test other error types fall through to field: msg format.""" - try: - ConstraintBounds(hard_min="not_a_number") - except ValidationError as e: - msg = _format_constraint_validation_error(e) - assert "Invalid constraint configuration" in msg - - -# ============================================================================= -# TESTS: _format_outlier_validation_error -# ============================================================================= - - -class TestFormatOutlierValidationError: - """Test _format_outlier_validation_error function.""" - - def test_formats_error_message(self): - """Test that a ValidationError is formatted into a user-friendly string.""" - try: - OutlierOptionsConfig( - outlier_method=OutlierMethod.IQR.value, - outlier_multiplier=0.0, - outlier_threshold=20, - ) - except ValidationError as e: - msg = _format_outlier_validation_error(e) - assert "Invalid outlier configuration" in msg - - def test_includes_field_name(self): - """Test that the field name appears in the formatted error.""" - try: - OutlierOptionsConfig( - outlier_method=OutlierMethod.IQR.value, - outlier_multiplier=0.0, - outlier_threshold=20, - ) - except ValidationError as e: - msg = _format_outlier_validation_error(e) - assert "outlier_multiplier" in msg - - def test_value_error_number_not_ge_branch(self): - """Test the value_error.number.not_ge branch via mocked error.""" - mock_error = MagicMock() - mock_error.errors.return_value = [ - { - "loc": ("outlier_multiplier",), - "msg": "value must be greater than 0", - "type": "value_error.number.not_ge", - } - ] - msg = _format_outlier_validation_error(mock_error) - assert "Invalid outlier configuration" in msg - assert "greater than or equal" in msg - - def test_value_error_number_not_le_branch(self): - """Test the value_error.number.not_le branch via mocked error.""" - mock_error = MagicMock() - mock_error.errors.return_value = [ - { - "loc": ("outlier_multiplier",), - "msg": "value must be less than or equal to 10", - "type": "value_error.number.not_le", - } - ] - msg = _format_outlier_validation_error(mock_error) - assert "Invalid outlier configuration" in msg - assert "less than or equal" in msg - - -# ============================================================================= -# TESTS: _ensure_column_formats -# ============================================================================= - - -class TestEnsureColumnFormats: - """Test _ensure_column_formats function.""" - - def test_returns_polars_dataframe(self, outlier_column_config): - result = _ensure_column_formats(outlier_column_config) - assert isinstance(result, pl.DataFrame) - - def test_preserves_column_names(self, outlier_column_config): - result = _ensure_column_formats(outlier_column_config) - assert set(outlier_column_config.columns) == set(result.columns) - - def test_casts_types_correctly(self, outlier_column_config): - result = _ensure_column_formats(outlier_column_config) - assert result.schema["outlier_multiplier"] == pl.Float64 - assert result.schema["outlier_threshold"] == pl.Int64 - - -# ============================================================================= -# TESTS: _render_constraint_metrics -# ============================================================================= - - -@pytest.fixture -def sample_violation_data(): - """Create sample constraint violation data.""" - return pl.DataFrame( - { - "survey_key": ["K001", "K002", "K003"], - "column name": ["col1", "col1", "col2"], - "violation reason": [ - "below hard minimum", - "above soft maximum", - "no violation", - ], - } - ) - - -class TestRenderConstraintMetrics: - """Test _render_constraint_metrics function.""" - - def test_calls_st_metric(self, sample_violation_data): - with patch("datasure.checks.outliers.st") as st_mock: - st_mock.columns.side_effect = _columns_side_effect - _render_constraint_metrics(sample_violation_data) - assert st_mock.metric.called or st_mock.columns.called - - -# ============================================================================= -# TESTS: _render_outlier_metrics -# ============================================================================= - - -@pytest.fixture -def sample_outlier_data(): - """Create sample outlier data.""" - return pl.DataFrame( - { - "survey_key": ["K001", "K002"], - "column name": ["col1", "col1"], - "outlier reason": ["Value is below lower bound 5.00", "no outlier"], - "enumerator": ["E001", "E001"], - } - ) - - -class TestRenderOutlierMetrics: - """Test _render_outlier_metrics function.""" - - def test_with_enumerator(self, sample_outlier_data, outlier_settings): - with patch("datasure.checks.outliers.st") as st_mock: - st_mock.columns.side_effect = _columns_side_effect - _render_outlier_metrics(sample_outlier_data, outlier_settings) - assert st_mock.metric.called or st_mock.columns.called - - def test_without_enumerator(self, sample_outlier_data): - settings = OutlierSettings( - survey_key="survey_key", - survey_id="survey_id", - survey_date=None, - enumerator=None, - team=None, - ) - with patch("datasure.checks.outliers.st") as st_mock: - st_mock.columns.side_effect = _columns_side_effect - _render_outlier_metrics(sample_outlier_data, settings) - assert st_mock.columns.called - - -# ============================================================================= -# TESTS: _render_constraint_violations_table -# ============================================================================= - - -@pytest.fixture -def base_survey_data(): - """Create base survey data for rendering tests.""" - return pl.DataFrame( - { - "survey_key": ["K001", "K002"], - "survey_id": ["S001", "S002"], - "survey_date": ["2024-01-01", "2024-01-02"], - "enumerator": ["E001", "E002"], - "team": ["T1", "T2"], - } - ) - - -class TestRenderConstraintViolationsTable: - """Test _render_constraint_violations_table function.""" - - def test_empty_data_shows_info(self, base_survey_data, outlier_settings): - with patch("datasure.checks.outliers.st") as st_mock: - _render_constraint_violations_table( - base_survey_data, - pl.DataFrame(), - outlier_settings, - "settings.json", - ) - st_mock.info.assert_called_once() - - def test_non_empty_data_shows_dataframe(self, base_survey_data, outlier_settings): - violation_data = pl.DataFrame( - { - "survey_key": ["K001"], - "column name": ["col1"], - "violation reason": ["below soft minimum"], - } - ) - with ( - patch("datasure.checks.outliers.st") as st_mock, - patch("datasure.checks.outliers.load_check_settings", return_value={}), - patch("datasure.checks.outliers.save_check_settings"), - ): - st_mock.columns.side_effect = _columns_side_effect - st_mock.multiselect.return_value = [] - _render_constraint_violations_table( - base_survey_data, - violation_data, - outlier_settings, - "settings.json", - ) - st_mock.dataframe.assert_called_once() - - def test_non_empty_with_extra_display_cols( - self, base_survey_data, outlier_settings - ): - violation_data = pl.DataFrame( - { - "survey_key": ["K001"], - "violation reason": ["above hard maximum"], - } - ) - with ( - patch("datasure.checks.outliers.st") as st_mock, - patch("datasure.checks.outliers.load_check_settings", return_value={}), - patch("datasure.checks.outliers.save_check_settings"), - ): - st_mock.columns.side_effect = _columns_side_effect - st_mock.multiselect.return_value = [] - _render_constraint_violations_table( - base_survey_data, - violation_data, - outlier_settings, - "settings.json", - ) - st_mock.dataframe.assert_called_once() - - -# ============================================================================= -# TESTS: _render_outlier_table -# ============================================================================= - - -class TestRenderOutlierTable: - """Test _render_outlier_table function.""" - - def test_empty_data_shows_info(self, base_survey_data, outlier_settings): - with patch("datasure.checks.outliers.st") as st_mock: - _render_outlier_table( - base_survey_data, - pl.DataFrame(), - outlier_settings, - "settings.json", - ) - st_mock.info.assert_called_once() - - def test_non_empty_data_shows_dataframe(self, base_survey_data, outlier_settings): - outliers_data = pl.DataFrame( - { - "survey_key": ["K001"], - "column name": ["col1"], - "outlier reason": ["Value is above upper bound 50.00"], - } - ) - with ( - patch("datasure.checks.outliers.st") as st_mock, - patch("datasure.checks.outliers.load_check_settings", return_value={}), - patch("datasure.checks.outliers.save_check_settings"), - ): - st_mock.columns.side_effect = _columns_side_effect - st_mock.multiselect.return_value = [] - _render_outlier_table( - base_survey_data, - outliers_data, - outlier_settings, - "settings.json", - ) - st_mock.dataframe.assert_called_once() - - -# ============================================================================= -# TESTS: _render_outlier_column_inspection -# ============================================================================= - - -class TestRenderOutlierColumnInspection: - """Test _render_outlier_column_inspection function.""" - - def test_empty_outlier_data_shows_info(self, base_survey_data, outlier_settings): - with patch("datasure.checks.outliers.st") as st_mock: - _render_outlier_column_inspection( - base_survey_data, pl.DataFrame(), outlier_settings, "settings.json" - ) - st_mock.info.assert_called_once() - - def test_no_selected_col_returns_early(self, base_survey_data, outlier_settings): - outliers_data = pl.DataFrame( - {"survey_key": ["K001"], "column name": ["survey_key"]} - ) - with ( - patch("datasure.checks.outliers.st") as st_mock, - patch("datasure.checks.outliers.load_check_settings", return_value={}), - patch("datasure.checks.outliers.save_check_settings"), - ): - st_mock.columns.side_effect = _columns_side_effect - st_mock.selectbox.return_value = None - _render_outlier_column_inspection( - base_survey_data, outliers_data, outlier_settings, "settings.json" - ) - st_mock.info.assert_called() - - def test_col_not_in_data_raises(self, base_survey_data, outlier_settings): - outliers_data = pl.DataFrame( - {"survey_key": ["K001"], "column name": ["nonexistent_col"]} - ) - with ( - patch("datasure.checks.outliers.st") as st_mock, - patch("datasure.checks.outliers.load_check_settings", return_value={}), - patch("datasure.checks.outliers.save_check_settings"), - ): - st_mock.columns.side_effect = _columns_side_effect - st_mock.selectbox.return_value = "nonexistent_col" - with pytest.raises(ValueError, match="not present in the data"): - _render_outlier_column_inspection( - base_survey_data, - outliers_data, - outlier_settings, - "settings.json", - ) - - def test_normal_path_renders_chart_and_table(self, outlier_settings): - data = pl.DataFrame( - { - "survey_key": ["K001", "K002"], - "survey_id": ["S001", "S002"], - "survey_date": ["2024-01-01", "2024-01-02"], - "enumerator": ["E001", "E002"], - "team": ["T1", "T2"], - "numeric_col1": [1.0, 100.0], - } - ) - outliers_data = pl.DataFrame( - { - "survey_key": ["K001"], - "column name": ["numeric_col1"], - "outlier reason": ["Value is above upper bound 50.00"], - } - ) - with ( - patch("datasure.checks.outliers.st") as st_mock, - patch("datasure.checks.outliers.load_check_settings", return_value={}), - patch("datasure.checks.outliers.save_check_settings"), - patch("datasure.checks.outliers._create_descriptive_stats") as mock_desc, - patch("datasure.checks.outliers._create_box_plot") as mock_box, - ): - st_mock.columns.side_effect = _columns_side_effect - st_mock.selectbox.return_value = "numeric_col1" - st_mock.multiselect.return_value = [] - mock_desc.return_value = pl.DataFrame( - {"statistic": ["count"], "value": ["2"]} - ) - mock_box.return_value = MagicMock() - _render_outlier_column_inspection( - data, outliers_data, outlier_settings, "settings.json" - ) - st_mock.dataframe.assert_called() - - -# ============================================================================= -# TESTS: _create_search_type_info -# ============================================================================= - - -class TestCreateSearchTypeInfo: - """Test _create_search_type_info function.""" - - def test_exact_search_type(self): - with patch("datasure.checks.outliers.st") as st_mock: - _create_search_type_info(SearchType.EXACT.value) - st_mock.info.assert_called_once() - - def test_startswith_search_type(self): - with patch("datasure.checks.outliers.st") as st_mock: - _create_search_type_info(SearchType.STARTSWITH.value) - st_mock.info.assert_called_once() - - def test_endswith_search_type(self): - with patch("datasure.checks.outliers.st") as st_mock: - _create_search_type_info(SearchType.ENDSWITH.value) - st_mock.info.assert_called_once() - - def test_contains_search_type(self): - with patch("datasure.checks.outliers.st") as st_mock: - _create_search_type_info(SearchType.CONTAINS.value) - st_mock.info.assert_called_once() - - def test_regex_search_type(self): - with patch("datasure.checks.outliers.st") as st_mock: - _create_search_type_info(SearchType.REGEX.value) - st_mock.info.assert_called_once() - - def test_unknown_type(self): - with patch("datasure.checks.outliers.st") as st_mock: - _create_search_type_info("unknown_type") - st_mock.info.assert_called_once() - - -# ============================================================================= -# TESTS: _render_search_type_selection -# ============================================================================= - - -class TestRenderSearchTypeSelection: - """Test _render_search_type_selection function.""" - - def test_exact_search_type(self): - with patch("datasure.checks.outliers.st") as st_mock: - st_mock.columns.side_effect = _columns_side_effect - st_mock.selectbox.return_value = SearchType.EXACT.value - st_mock.multiselect.return_value = ["col1"] - search_type, pattern, cols, _lock = _render_search_type_selection( - ["col1", "col2"] - ) - assert search_type == SearchType.EXACT.value - assert pattern is None - assert cols == ["col1"] - - def test_pattern_search_type_with_pattern(self): - with patch("datasure.checks.outliers.st") as st_mock: - st_mock.selectbox.return_value = SearchType.STARTSWITH.value - st_mock.text_input.return_value = "num" - search_type, pattern, cols, _lock = _render_search_type_selection( - ["num_col1", "num_col2", "other"] - ) - assert search_type == SearchType.STARTSWITH.value - assert pattern == "num" - assert "num_col1" in cols - - def test_pattern_search_type_no_pattern(self): - with patch("datasure.checks.outliers.st") as st_mock: - st_mock.selectbox.return_value = SearchType.CONTAINS.value - st_mock.text_input.return_value = "" - _search_type, _pattern, cols, lock = _render_search_type_selection( - ["col1", "col2"] - ) - assert cols == [] - assert lock is None - - -# ============================================================================= -# TESTS: _render_column_grouping_options -# ============================================================================= - - -class TestRenderColumnGroupingOptions: - """Test _render_column_grouping_options function.""" - - def test_basic_render(self): - with patch("datasure.checks.outliers.st") as st_mock: - st_mock.columns.side_effect = _columns_side_effect - st_mock.toggle.return_value = False - group_cols, lock_cols = _render_column_grouping_options( - ["col1", "col2"], SearchType.EXACT.value - ) - assert isinstance(group_cols, bool) - assert isinstance(lock_cols, bool) - - def test_returns_toggle_values(self): - with patch("datasure.checks.outliers.st") as st_mock: - st_mock.columns.side_effect = _columns_side_effect - st_mock.toggle.side_effect = [True, False] - group_cols, lock_cols = _render_column_grouping_options( - ["col1", "col2"], SearchType.STARTSWITH.value - ) - assert group_cols is True - assert lock_cols is False - - -# ============================================================================= -# TESTS: _render_outlier_options -# ============================================================================= - - -class TestRenderOutlierOptions: - """Test _render_outlier_options function.""" - - def test_outliers_enabled_returns_settings(self): - with patch("datasure.checks.outliers.st") as st_mock: - st_mock.columns.side_effect = _columns_side_effect - st_mock.toggle.return_value = True - st_mock.selectbox.return_value = OutlierMethod.IQR.value - st_mock.number_input.side_effect = [1.5, 20] - enabled, settings, valid = _render_outlier_options() - assert enabled is True - assert settings is not None - assert valid is True - - def test_outliers_enabled_sd_method(self): - with patch("datasure.checks.outliers.st") as st_mock: - st_mock.columns.side_effect = _columns_side_effect - st_mock.toggle.return_value = True - st_mock.selectbox.return_value = OutlierMethod.SD.value - st_mock.number_input.side_effect = [3.0, 30] - enabled, _settings, valid = _render_outlier_options() - assert enabled is True - assert valid is True - - def test_outliers_disabled_returns_none(self): - with patch("datasure.checks.outliers.st") as st_mock: - st_mock.columns.side_effect = _columns_side_effect - st_mock.toggle.return_value = False - enabled, settings, valid = _render_outlier_options() - assert enabled is False - assert settings is None - assert valid is True - - -# ============================================================================= -# TESTS: _render_constraint_options -# ============================================================================= - - -class TestRenderConstraintOptions: - """Test _render_constraint_options function.""" - - def test_valid_settings_returns_true(self): - with patch("datasure.checks.outliers.st") as st_mock: - st_mock.columns.side_effect = _columns_side_effect - st_mock.number_input.return_value = None - _settings, valid = _render_constraint_options() - assert valid is True - - def test_invalid_settings_calls_error(self): - with patch("datasure.checks.outliers.st") as st_mock: - st_mock.columns.side_effect = _columns_side_effect - st_mock.number_input.side_effect = [50.0, 10.0, None, None] - _settings, valid = _render_constraint_options() - assert valid is False - st_mock.error.assert_called_once() - - -# ============================================================================= -# TESTS: _render_outlier_settings_table -# ============================================================================= - - -class TestRenderOutlierSettingsTable: - """Test _render_outlier_settings_table function.""" - - def test_renders_dataframe(self, outlier_column_config): - with patch("datasure.checks.outliers.st") as st_mock: - _render_outlier_settings_table(outlier_column_config) - st_mock.dataframe.assert_called_once() - - -# ============================================================================= -# TESTS: _render_outlier_column_actions -# ============================================================================= - - -class TestRenderOutlierColumnActions: - """Test _render_outlier_column_actions function.""" - - def test_empty_settings_shows_info(self): - with ( - patch("datasure.checks.outliers.st") as st_mock, - patch( - "datasure.checks.outliers.duckdb_get_table", - return_value=pl.DataFrame(), - ), - ): - st_mock.columns.side_effect = _columns_side_effect - _render_outlier_column_actions("proj1", "page1", ["col1"]) - assert st_mock.info.call_count >= 1 - - def test_non_empty_settings_calls_render_table(self, outlier_column_config): - with ( - patch("datasure.checks.outliers.st") as st_mock, - patch( - "datasure.checks.outliers.duckdb_get_table", - return_value=outlier_column_config, - ), - patch( - "datasure.checks.outliers._render_outlier_settings_table" - ) as mock_render, - patch("datasure.checks.outliers._delete_outlier_column"), - ): - st_mock.columns.side_effect = _columns_side_effect - _render_outlier_column_actions("proj1", "page1", ["col1"]) - mock_render.assert_called_once() - - -# ============================================================================= -# TESTS: _update_outlier_column_config -# ============================================================================= - - -class TestUpdateOutlierColumnConfig: - """Test _update_outlier_column_config function.""" - - def test_empty_existing_config_saves_new(self): - settings = OutlierOptionsConfig( - outlier_method=OutlierMethod.IQR, - outlier_multiplier=1.5, - outlier_threshold=20, - ) - bounds = ConstraintBounds(soft_min=0.0, soft_max=100.0) - with ( - patch( - "datasure.checks.outliers.duckdb_get_table", - return_value=pl.DataFrame(), - ), - patch("datasure.checks.outliers.duckdb_save_table") as mock_save, - ): - _update_outlier_column_config( - "proj1", - "page1", - "exact", - None, - ["col1"], - False, - False, - True, - settings, - bounds, - ) - mock_save.assert_called_once() - - def test_non_empty_existing_config_concatenates(self, outlier_column_config): - settings = OutlierOptionsConfig( - outlier_method=OutlierMethod.IQR, - outlier_multiplier=1.5, - outlier_threshold=20, - ) - bounds = ConstraintBounds(soft_min=0.0, soft_max=100.0) - with ( - patch( - "datasure.checks.outliers.duckdb_get_table", - return_value=outlier_column_config, - ), - patch("datasure.checks.outliers.duckdb_save_table") as mock_save, - ): - _update_outlier_column_config( - "proj1", - "page1", - "exact", - None, - ["col2"], - False, - False, - True, - settings, - bounds, - ) - mock_save.assert_called_once() - saved_df = mock_save.call_args[0][1] - assert len(saved_df) == 2 - - def test_outlier_settings_none(self): - bounds = ConstraintBounds(soft_min=0.0, soft_max=100.0) - with ( - patch( - "datasure.checks.outliers.duckdb_get_table", - return_value=pl.DataFrame(), - ), - patch("datasure.checks.outliers.duckdb_save_table") as mock_save, - ): - _update_outlier_column_config( - "proj1", - "page1", - "exact", - None, - ["col1"], - False, - False, - False, - None, - bounds, - ) - mock_save.assert_called_once() - - -# ============================================================================= -# TESTS: _delete_outlier_column -# ============================================================================= - - -class TestDeleteOutlierColumn: - """Test _delete_outlier_column function.""" - - def test_empty_settings_shows_info(self): - with patch("datasure.checks.outliers.st") as st_mock: - _delete_outlier_column("proj1", "page1", pl.DataFrame()) - st_mock.info.assert_called_once() - - def test_non_empty_shows_selectbox_and_button(self, outlier_column_config): - with patch("datasure.checks.outliers.st") as st_mock: - st_mock.selectbox.return_value = "0 - exact - " - st_mock.button.return_value = False - _delete_outlier_column("proj1", "page1", outlier_column_config) - st_mock.selectbox.assert_called_once() - - def test_delete_on_button_click(self, outlier_column_config): - with ( - patch("datasure.checks.outliers.st") as st_mock, - patch("datasure.checks.outliers.duckdb_save_table") as mock_save, - ): - st_mock.selectbox.return_value = "0 - exact - " - st_mock.button.return_value = True - _delete_outlier_column("proj1", "page1", outlier_column_config) - mock_save.assert_called_once() - - -# ============================================================================= -# TESTS: outliers_report_settings (via reimport) -# ============================================================================= - - -class TestOutliersReportSettings: - """Test outliers_report_settings function.""" - - def test_returns_outlier_settings(self, outliers_mod): - config = OutlierSettings( - survey_key="survey_key", - survey_id="survey_id", - survey_date="survey_date", - enumerator="enumerator", - team="team", - ) - with ( - patch( - "datasure.checks.outliers.load_default_settings", return_value=config - ), - patch("datasure.checks.outliers.load_check_settings", return_value={}), - patch("datasure.checks.outliers.save_check_settings"), - patch("datasure.checks.outliers.trigger_save"), - ): - outliers_mod.st.selectbox.return_value = "survey_key" - result = outliers_mod.outliers_report_settings( - "settings.json", - config, - ["survey_key", "survey_id", "enumerator", "team"], - ["survey_date"], - ) - assert isinstance(result, outliers_mod.OutlierSettings) - - -# ============================================================================= -# TESTS: _add_outlier_column (via reimport) -# ============================================================================= - - -class TestAddOutlierColumn: - """Test _add_outlier_column function.""" - - def test_no_cols_selected_does_not_save(self, outliers_mod): - """When no columns selected, skip grouping/options rendering.""" - mock_sel = MagicMock(return_value=(SearchType.EXACT.value, None, [], None)) - with patch.object(outliers_mod, "_render_search_type_selection", mock_sel): - outliers_mod._add_outlier_column("proj1", "page1", ["col1", "col2"]) - mock_sel.assert_called_once() - - def test_with_cols_selected_renders_options(self, outliers_mod): - """When columns are selected, all option panels are rendered.""" - settings = OutlierOptionsConfig( - outlier_method=OutlierMethod.IQR, - outlier_multiplier=1.5, - outlier_threshold=20, - ) - mock_sel = MagicMock( - return_value=(SearchType.EXACT.value, None, ["col1"], None) - ) - mock_grp = MagicMock(return_value=(False, False)) - mock_out = MagicMock(return_value=(True, settings, True)) - mock_con = MagicMock(return_value=(ConstraintBounds(), True)) - with ( - patch.object(outliers_mod, "_render_search_type_selection", mock_sel), - patch.object(outliers_mod, "_render_column_grouping_options", mock_grp), - patch.object(outliers_mod, "_render_outlier_options", mock_out), - patch.object(outliers_mod, "_render_constraint_options", mock_con), - ): - outliers_mod.st.button.return_value = False - outliers_mod._add_outlier_column("proj1", "page1", ["col1", "col2"]) - mock_grp.assert_called_once() - mock_out.assert_called_once() - mock_con.assert_called_once() - - -# ============================================================================= -# TESTS: outliers_report (main function) -# ============================================================================= - - -@pytest.fixture -def survey_columns_mock(): - """Create a mock ColumnByType for outliers_report tests.""" - from datasure.utils.dataframe_utils import ColumnByType - - return ColumnByType( - all_columns=[ - "survey_key", - "survey_id", - "survey_date", - "enumerator", - "team", - "col1", - ], - categorical_columns=["survey_key", "survey_id", "enumerator", "team"], - datetime_columns=["survey_date"], - numeric_columns=["col1"], - boolean_columns=[], - ) - - -@pytest.fixture -def outlier_config_dict(): - """Create a default outlier config dict for outliers_report tests.""" - return { - "survey_key": "survey_key", - "survey_id": "survey_id", - "survey_date": "survey_date", - "enumerator": "enumerator", - "team": "team", - } - - -class TestOutliersReport: - """Test outliers_report main function.""" - - def test_empty_column_config_returns_early( - self, base_survey_data, survey_columns_mock, outlier_config_dict - ): - settings = OutlierSettings(**outlier_config_dict) - with ( - patch("datasure.checks.outliers.st") as st_mock, - patch( - "datasure.checks.outliers.outliers_report_settings", - return_value=settings, - ), - patch("datasure.checks.outliers._render_outlier_column_actions"), - patch( - "datasure.checks.outliers.duckdb_get_table", - return_value=pl.DataFrame(), - ), - ): - st_mock.columns.side_effect = _columns_side_effect - outliers_report( - "proj1", - "page1", - base_survey_data, - "settings.json", - outlier_config_dict, - survey_columns_mock, - ) - st_mock.title.assert_called() - - def test_with_constraint_violations( - self, - base_survey_data, - survey_columns_mock, - outlier_config_dict, - outlier_column_config, - ): - settings = OutlierSettings(**outlier_config_dict) - constraint_violations = pl.DataFrame( - { - "survey_key": ["K001"], - "column name": ["col1"], - "violation reason": ["below soft minimum"], - } - ) - with ( - patch("datasure.checks.outliers.st") as st_mock, - patch( - "datasure.checks.outliers.outliers_report_settings", - return_value=settings, - ), - patch("datasure.checks.outliers._render_outlier_column_actions"), - patch( - "datasure.checks.outliers.duckdb_get_table", - return_value=outlier_column_config, - ), - patch("datasure.checks.outliers.duckdb_save_table"), - patch( - "datasure.checks.outliers.compute_constraint_violations", - return_value=constraint_violations, - ), - patch( - "datasure.checks.outliers.compute_outlier_output", - return_value=pl.DataFrame(), - ), - patch("datasure.checks.outliers._render_constraint_metrics"), - patch("datasure.checks.outliers._render_constraint_violations_table"), - ): - st_mock.columns.side_effect = _columns_side_effect - outliers_report( - "proj1", - "page1", - base_survey_data, - "settings.json", - outlier_config_dict, - survey_columns_mock, - ) - st_mock.info.assert_called() - - def test_with_outliers( - self, - base_survey_data, - survey_columns_mock, - outlier_config_dict, - outlier_column_config, - ): - settings = OutlierSettings(**outlier_config_dict) - outlier_data = pl.DataFrame( - { - "survey_key": ["K001"], - "column name": ["col1"], - "outlier reason": ["Value is above upper bound 50.00"], - } - ) - with ( - patch("datasure.checks.outliers.st") as st_mock, - patch( - "datasure.checks.outliers.outliers_report_settings", - return_value=settings, - ), - patch("datasure.checks.outliers._render_outlier_column_actions"), - patch( - "datasure.checks.outliers.duckdb_get_table", - return_value=outlier_column_config, - ), - patch("datasure.checks.outliers.duckdb_save_table"), - patch( - "datasure.checks.outliers.compute_constraint_violations", - return_value=pl.DataFrame(), - ), - patch( - "datasure.checks.outliers.compute_outlier_output", - return_value=outlier_data, - ), - patch("datasure.checks.outliers._render_outlier_metrics"), - patch("datasure.checks.outliers._render_outlier_column_inspection"), - ): - st_mock.columns.side_effect = _columns_side_effect - outliers_report( - "proj1", - "page1", - base_survey_data, - "settings.json", - outlier_config_dict, - survey_columns_mock, - ) - st_mock.info.assert_called() diff --git a/tests/checks/outliers/test_models.py b/tests/checks/outliers/test_models.py new file mode 100644 index 00000000..4c48e80b --- /dev/null +++ b/tests/checks/outliers/test_models.py @@ -0,0 +1,336 @@ +"""Tests for datasure.checks.outliers.models.""" + +import pytest +from pydantic import ValidationError + +from datasure.checks.outliers.models import ( + ConstraintBounds, + ConstraintMetrics, + OutlierBounds, + OutlierColumnConfig, + OutlierMethod, + OutlierMetrics, + OutlierOptionsConfig, + OutlierSettings, + OutlierStatistics, + SearchType, +) + +# ============================================================================ +# PYDANTIC MODEL TESTS +# ============================================================================ + + +class TestOutlierBounds: + """Test OutlierBounds Pydantic model.""" + + def test_valid_bounds(self): + """Test creating valid outlier bounds.""" + bounds = OutlierBounds(lower_bound=0.0, upper_bound=100.0) + assert bounds.lower_bound == 0.0 + assert bounds.upper_bound == 100.0 + + def test_negative_bounds(self): + """Test bounds with negative values.""" + bounds = OutlierBounds(lower_bound=-50.0, upper_bound=50.0) + assert bounds.lower_bound == -50.0 + assert bounds.upper_bound == 50.0 + + +class TestOutlierOptionsConfig: + """Test OutlierOptionsConfig Pydantic model.""" + + def test_valid_config(self): + """Test creating valid outlier options config.""" + config = OutlierOptionsConfig( + outlier_method=OutlierMethod.IQR, + outlier_multiplier=1.5, + outlier_threshold=20, + ) + assert config.outlier_method == OutlierMethod.IQR + assert config.outlier_multiplier == 1.5 + assert config.outlier_threshold == 20 + + def test_invalid_multiplier_zero(self): + """Test that zero multiplier raises validation error.""" + with pytest.raises(ValidationError): + OutlierOptionsConfig( + outlier_method=OutlierMethod.IQR, + outlier_multiplier=0.0, + outlier_threshold=20, + ) + + def test_invalid_multiplier_negative(self): + """Test that negative multiplier raises validation error.""" + with pytest.raises(ValidationError): + OutlierOptionsConfig( + outlier_method=OutlierMethod.IQR, + outlier_multiplier=-1.5, + outlier_threshold=20, + ) + + def test_invalid_threshold_zero(self): + """Test that zero threshold raises validation error.""" + with pytest.raises(ValidationError): + OutlierOptionsConfig( + outlier_method=OutlierMethod.IQR, + outlier_multiplier=1.5, + outlier_threshold=0, + ) + + +class TestConstraintBounds: + """Test ConstraintBounds Pydantic model.""" + + def test_valid_bounds_all_fields(self): + """Test creating valid constraint bounds with all fields.""" + bounds = ConstraintBounds( + hard_min=0.0, soft_min=10.0, soft_max=90.0, hard_max=100.0 + ) + assert bounds.hard_min == 0.0 + assert bounds.soft_min == 10.0 + assert bounds.soft_max == 90.0 + assert bounds.hard_max == 100.0 + + def test_valid_bounds_partial(self): + """Test creating valid constraint bounds with partial fields.""" + bounds = ConstraintBounds(soft_min=10.0, soft_max=90.0) + assert bounds.hard_min is None + assert bounds.soft_min == 10.0 + assert bounds.soft_max == 90.0 + assert bounds.hard_max is None + + def test_invalid_bounds_hierarchy(self): + """Test that invalid hierarchy raises validation error.""" + with pytest.raises(ValidationError, match="Bounds must follow hierarchy"): + ConstraintBounds( + hard_min=50.0, + soft_min=10.0, # hard_min > soft_min + ) + + def test_invalid_soft_bounds(self): + """Test that soft_min > soft_max raises validation error.""" + with pytest.raises(ValidationError): + ConstraintBounds(soft_min=90.0, soft_max=10.0) + + def test_negative_bounds(self): + """Test constraint bounds with negative values.""" + bounds = ConstraintBounds( + hard_min=-100.0, soft_min=-50.0, soft_max=50.0, hard_max=100.0 + ) + assert bounds.hard_min == -100.0 + + +class TestConstraintMetrics: + """Test ConstraintMetrics Pydantic model.""" + + def test_valid_metrics(self): + """Test creating valid constraint metrics.""" + metrics = ConstraintMetrics( + columns_checked=5, + total_violations=10, + hard_min_violations=2, + soft_min_violations=3, + soft_max_violations=3, + hard_max_violations=2, + ) + assert metrics.total_violations == 10 + + def test_negative_values_invalid(self): + """Test that negative values raise validation error.""" + with pytest.raises(ValidationError): + ConstraintMetrics( + columns_checked=-1, + total_violations=0, + hard_min_violations=0, + soft_min_violations=0, + soft_max_violations=0, + hard_max_violations=0, + ) + + +class TestOutlierMetrics: + """Test OutlierMetrics Pydantic model.""" + + def test_valid_metrics(self): + """Test creating valid outlier metrics.""" + metrics = OutlierMetrics( + columns_checked=5, + columns_with_outliers=3, + total_outliers=10, + enumerators_with_outliers=2, + ) + assert metrics.columns_checked == 5 + assert metrics.total_outliers == 10 + + +class TestOutlierStatistics: + """Test OutlierStatistics Pydantic model.""" + + def test_valid_statistics(self): + """Test creating valid outlier statistics.""" + stats = OutlierStatistics( + count=100, + min_value=0.0, + max_value=100.0, + mean=50.0, + median=48.0, + sd=15.0, + iqr=25.0, + lower_bound=10.0, + upper_bound=90.0, + ) + assert stats.count == 100 + assert stats.mean == 50.0 + assert stats.sd == 15.0 + + def test_alias_std(self): + """Test that 'sd' alias works for std field.""" + stats = OutlierStatistics( + count=100, + min_value=0.0, + max_value=100.0, + mean=50.0, + median=48.0, + sd=15.0, # Using alias + iqr=25.0, + lower_bound=10.0, + upper_bound=90.0, + ) + assert stats.sd == 15.0 + + +class TestOutlierColumnConfig: + """Test OutlierColumnConfig Pydantic model.""" + + def test_valid_config_exact(self): + """Test creating valid config with exact search.""" + config = OutlierColumnConfig( + search_type=SearchType.EXACT, + pattern=None, + outlier_cols=["col1", "col2"], + lock_cols=False, + grouped_cols=False, + outlier_method=OutlierMethod.IQR, + outlier_multiplier=1.5, + ) + assert config.search_type == SearchType.EXACT + + def test_invalid_pattern_required(self): + """Test that pattern is required for non-exact search types.""" + with pytest.raises(ValidationError): + OutlierColumnConfig( + search_type=SearchType.STARTSWITH, + pattern=None, # Should be required + outlier_cols=["col1"], + outlier_method=OutlierMethod.IQR, + outlier_multiplier=1.5, + ) + + def test_invalid_soft_bounds(self): + """Test that soft_max must be greater than soft_min.""" + with pytest.raises(ValidationError): + OutlierColumnConfig( + search_type=SearchType.EXACT, + outlier_cols=["col1"], + outlier_method=OutlierMethod.IQR, + outlier_multiplier=1.5, + soft_min=50.0, + soft_max=10.0, # Less than soft_min + ) + + +class TestOutlierSettings: + """Test OutlierSettings Pydantic model.""" + + def test_valid_settings(self): + """Test creating valid outlier settings.""" + settings = OutlierSettings( + survey_key="key", + survey_id="id", + survey_date="date", + enumerator="enum", + team="team", + ) + assert settings.survey_key == "key" + + def test_minimal_settings(self): + """Test creating minimal valid settings.""" + settings = OutlierSettings(survey_key="key") + assert settings.survey_key == "key" + assert settings.survey_id is None + + +# ============================================================================ +# Additional Model Validation Edge Case Tests +# ============================================================================ + + +class TestConstraintValidation: + """Test constraint bounds validation.""" + + def test_constraint_bounds_all_none(self): + """Test ConstraintBounds with all None values.""" + bounds = ConstraintBounds() + assert bounds.hard_min is None + assert bounds.soft_min is None + assert bounds.soft_max is None + assert bounds.hard_max is None + + def test_constraint_bounds_partial(self): + """Test ConstraintBounds with partial values.""" + bounds = ConstraintBounds(soft_min=10, soft_max=100) + assert bounds.soft_min == 10 + assert bounds.soft_max == 100 + assert bounds.hard_min is None + assert bounds.hard_max is None + + def test_constraint_bounds_invalid_order(self): + """Test ConstraintBounds with invalid hierarchy.""" + with pytest.raises(ValidationError, match="must be <="): + ConstraintBounds(hard_min=100, soft_min=50) + + def test_constraint_bounds_negative_values(self): + """Test ConstraintBounds with negative values.""" + bounds = ConstraintBounds( + hard_min=-100, soft_min=-50, soft_max=50, hard_max=100 + ) + assert bounds.hard_min == -100 + assert bounds.soft_min == -50 + + +class TestOutlierColumnConfigValidation: + """Test OutlierColumnConfig validation edge cases.""" + + def test_pattern_required_for_non_exact(self): + """Test that pattern is required for non-exact search types.""" + with pytest.raises(ValidationError, match="Pattern is required"): + OutlierColumnConfig( + search_type=SearchType.STARTSWITH, + pattern=None, + outlier_cols=["col1"], + outlier_multiplier=1.5, + ) + + def test_soft_max_validation(self): + """Test soft_max must be greater than soft_min.""" + with pytest.raises(ValidationError, match="soft_max must be greater"): + OutlierColumnConfig( + search_type=SearchType.EXACT, + outlier_cols=["col1"], + outlier_multiplier=1.5, + soft_min=100, + soft_max=50, + ) + + def test_valid_config_with_constraints(self): + """Test valid configuration with all constraints.""" + config = OutlierColumnConfig( + search_type=SearchType.EXACT, + outlier_cols=["col1"], + outlier_multiplier=1.5, + soft_min=10, + soft_max=100, + ) + assert config.soft_min == 10 + assert config.soft_max == 100 diff --git a/tests/checks/outliers/test_report_ui.py b/tests/checks/outliers/test_report_ui.py new file mode 100644 index 00000000..71353b2c --- /dev/null +++ b/tests/checks/outliers/test_report_ui.py @@ -0,0 +1,1139 @@ +"""Tests for datasure.checks.outliers.report_ui.""" + +import importlib +import sys +from unittest.mock import MagicMock, patch + +import polars as pl +import pytest +from pydantic import ValidationError + +from datasure.checks.outliers.models import ( + ConstraintBounds, + OutlierMethod, + OutlierOptionsConfig, + OutlierSettings, + SearchType, +) +from datasure.checks.outliers.report_ui import ( + _create_search_type_info, + _delete_outlier_column, + _ensure_column_formats, + _format_constraint_validation_error, + _format_outlier_validation_error, + _render_column_grouping_options, + _render_constraint_metrics, + _render_constraint_options, + _render_constraint_violations_table, + _render_outlier_column_actions, + _render_outlier_column_inspection, + _render_outlier_metrics, + _render_outlier_options, + _render_outlier_settings_table, + _render_outlier_table, + _render_search_type_selection, + _update_outlier_column_config, + _validate_constraint_settings, + _validate_outlier_settings, + outliers_report, +) +from tests.checks.outliers.conftest import _columns_side_effect, _make_st_mock + +# ============================================================================ +# REPORT_UI_MOD FIXTURE (reload compute/settings_ui/report_ui with mocked +# streamlit for @st.dialog decorator tests) +# ============================================================================ + + +@pytest.fixture +def report_ui_mod(): + """Reload the outliers submodules with mocked Streamlit for decorator tests.""" + mock_st = _make_st_mock() + original_st = sys.modules.get("streamlit") + sys.modules["streamlit"] = mock_st + + import datasure.checks.outliers.compute as compute_module + import datasure.checks.outliers.report_ui as report_ui_module + import datasure.checks.outliers.settings_ui as settings_ui_module + + try: + with patch( + "datasure.utils.onboarding_utils.demo_output_onboarding", + lambda tab: lambda f: f, + ): + # Reload in dependency order so decorators pick up the mocked st and + # cross-module references (report_ui imports from settings_ui) stay wired. + importlib.reload(compute_module) + importlib.reload(settings_ui_module) + importlib.reload(report_ui_module) + yield report_ui_module + finally: + if original_st is not None: + sys.modules["streamlit"] = original_st + else: + sys.modules.pop("streamlit", None) + importlib.reload(compute_module) + importlib.reload(settings_ui_module) + importlib.reload(report_ui_module) + + +# ============================================================================ +# FIXTURES +# ============================================================================ + + +@pytest.fixture +def sample_violation_data(): + """Create sample constraint violation data.""" + return pl.DataFrame( + { + "survey_key": ["K001", "K002", "K003"], + "column name": ["col1", "col1", "col2"], + "violation reason": [ + "below hard minimum", + "above soft maximum", + "no violation", + ], + } + ) + + +@pytest.fixture +def sample_outlier_data(): + """Create sample outlier data.""" + return pl.DataFrame( + { + "survey_key": ["K001", "K002"], + "column name": ["col1", "col1"], + "outlier reason": ["Value is below lower bound 5.00", "no outlier"], + "enumerator": ["E001", "E001"], + } + ) + + +@pytest.fixture +def base_survey_data(): + """Create base survey data for rendering tests.""" + return pl.DataFrame( + { + "survey_key": ["K001", "K002"], + "survey_id": ["S001", "S002"], + "survey_date": ["2024-01-01", "2024-01-02"], + "enumerator": ["E001", "E002"], + "team": ["T1", "T2"], + } + ) + + +@pytest.fixture +def survey_columns_mock(): + """Create a mock ColumnByType for outliers_report tests.""" + from datasure.utils.dataframe_utils import ColumnByType + + return ColumnByType( + all_columns=[ + "survey_key", + "survey_id", + "survey_date", + "enumerator", + "team", + "col1", + ], + categorical_columns=["survey_key", "survey_id", "enumerator", "team"], + datetime_columns=["survey_date"], + numeric_columns=["col1"], + boolean_columns=[], + ) + + +@pytest.fixture +def outlier_config_dict(): + """Create a default outlier config dict for outliers_report tests.""" + return { + "survey_key": "survey_key", + "survey_id": "survey_id", + "survey_date": "survey_date", + "enumerator": "enumerator", + "team": "team", + } + + +# ============================================================================ +# TESTS: _validate_constraint_settings +# ============================================================================ + + +class TestValidateConstraintSettings: + """Test _validate_constraint_settings function.""" + + def test_valid_settings_returns_bounds_and_true(self): + result, valid = _validate_constraint_settings( + {"soft_min": 0.0, "soft_max": 100.0} + ) + assert valid is True + assert result is not None + + def test_invalid_hierarchy_returns_none_and_false(self): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + result, valid = _validate_constraint_settings( + {"hard_min": 50.0, "soft_min": 10.0} + ) + assert valid is False + assert result is None + st_mock.error.assert_called_once() + + def test_all_none_settings_valid(self): + _result, valid = _validate_constraint_settings( + {"hard_min": None, "soft_min": None, "soft_max": None, "hard_max": None} + ) + assert valid is True + + +# ============================================================================ +# TESTS: _validate_outlier_settings +# ============================================================================ + + +class TestValidateOutlierSettings: + """Test _validate_outlier_settings function.""" + + def test_valid_settings_returns_config_and_true(self): + result, valid = _validate_outlier_settings( + { + "outlier_method": OutlierMethod.IQR.value, + "outlier_multiplier": 1.5, + "outlier_threshold": 20, + } + ) + assert valid is True + assert result is not None + + def test_invalid_multiplier_returns_none_and_false(self): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + result, valid = _validate_outlier_settings( + { + "outlier_method": OutlierMethod.IQR.value, + "outlier_multiplier": 0.0, + "outlier_threshold": 20, + } + ) + assert valid is False + assert result is None + st_mock.error.assert_called_once() + + +# ============================================================================ +# TESTS: _format_constraint_validation_error +# ============================================================================ + + +class TestFormatConstraintValidationError: + """Test _format_constraint_validation_error function.""" + + def test_value_error_type(self): + try: + ConstraintBounds(hard_min=50.0, soft_min=10.0) + except ValidationError as e: + msg = _format_constraint_validation_error(e) + assert "Invalid constraint configuration" in msg + + def test_float_not_finite_type(self): + """Test float_not_finite error type produces finite-number message.""" + mock_error = MagicMock() + mock_error.errors.return_value = [ + { + "loc": ("hard_min",), + "msg": "value is not a finite number", + "type": "float_not_finite", + } + ] + msg = _format_constraint_validation_error(mock_error) + assert "Invalid constraint configuration" in msg + assert "finite number" in msg + + def test_value_error_type_uses_msg(self): + """Test value_error type includes the custom validation message.""" + mock_error = MagicMock() + mock_error.errors.return_value = [ + { + "loc": ("hard_min",), + "msg": "Bounds must follow hierarchy", + "type": "value_error", + } + ] + msg = _format_constraint_validation_error(mock_error) + assert "Bounds must follow hierarchy" in msg + + def test_other_error_type(self): + """Test other error types fall through to field: msg format.""" + try: + ConstraintBounds(hard_min="not_a_number") + except ValidationError as e: + msg = _format_constraint_validation_error(e) + assert "Invalid constraint configuration" in msg + + +# ============================================================================ +# TESTS: _format_outlier_validation_error +# ============================================================================ + + +class TestFormatOutlierValidationError: + """Test _format_outlier_validation_error function.""" + + def test_formats_error_message(self): + """Test that a ValidationError is formatted into a user-friendly string.""" + try: + OutlierOptionsConfig( + outlier_method=OutlierMethod.IQR.value, + outlier_multiplier=0.0, + outlier_threshold=20, + ) + except ValidationError as e: + msg = _format_outlier_validation_error(e) + assert "Invalid outlier configuration" in msg + + def test_includes_field_name(self): + """Test that the field name appears in the formatted error.""" + try: + OutlierOptionsConfig( + outlier_method=OutlierMethod.IQR.value, + outlier_multiplier=0.0, + outlier_threshold=20, + ) + except ValidationError as e: + msg = _format_outlier_validation_error(e) + assert "outlier_multiplier" in msg + + def test_value_error_number_not_ge_branch(self): + """Test the value_error.number.not_ge branch via mocked error.""" + mock_error = MagicMock() + mock_error.errors.return_value = [ + { + "loc": ("outlier_multiplier",), + "msg": "value must be greater than 0", + "type": "value_error.number.not_ge", + } + ] + msg = _format_outlier_validation_error(mock_error) + assert "Invalid outlier configuration" in msg + assert "greater than or equal" in msg + + def test_value_error_number_not_le_branch(self): + """Test the value_error.number.not_le branch via mocked error.""" + mock_error = MagicMock() + mock_error.errors.return_value = [ + { + "loc": ("outlier_multiplier",), + "msg": "value must be less than or equal to 10", + "type": "value_error.number.not_le", + } + ] + msg = _format_outlier_validation_error(mock_error) + assert "Invalid outlier configuration" in msg + assert "less than or equal" in msg + + +# ============================================================================ +# TESTS: _ensure_column_formats +# ============================================================================ + + +class TestEnsureColumnFormats: + """Test _ensure_column_formats function.""" + + def test_returns_polars_dataframe(self, outlier_column_config): + result = _ensure_column_formats(outlier_column_config) + assert isinstance(result, pl.DataFrame) + + def test_preserves_column_names(self, outlier_column_config): + result = _ensure_column_formats(outlier_column_config) + assert set(outlier_column_config.columns) == set(result.columns) + + def test_casts_types_correctly(self, outlier_column_config): + result = _ensure_column_formats(outlier_column_config) + assert result.schema["outlier_multiplier"] == pl.Float64 + assert result.schema["outlier_threshold"] == pl.Int64 + + +# ============================================================================ +# TESTS: _render_constraint_metrics +# ============================================================================ + + +class TestRenderConstraintMetrics: + """Test _render_constraint_metrics function.""" + + def test_calls_st_metric(self, sample_violation_data): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + st_mock.columns.side_effect = _columns_side_effect + _render_constraint_metrics(sample_violation_data) + assert st_mock.metric.called or st_mock.columns.called + + +# ============================================================================ +# TESTS: _render_outlier_metrics +# ============================================================================ + + +class TestRenderOutlierMetrics: + """Test _render_outlier_metrics function.""" + + def test_with_enumerator(self, sample_outlier_data, outlier_settings): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + st_mock.columns.side_effect = _columns_side_effect + _render_outlier_metrics(sample_outlier_data, outlier_settings) + assert st_mock.metric.called or st_mock.columns.called + + def test_without_enumerator(self, sample_outlier_data): + settings = OutlierSettings( + survey_key="survey_key", + survey_id="survey_id", + survey_date=None, + enumerator=None, + team=None, + ) + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + st_mock.columns.side_effect = _columns_side_effect + _render_outlier_metrics(sample_outlier_data, settings) + assert st_mock.columns.called + + +# ============================================================================ +# TESTS: _render_constraint_violations_table +# ============================================================================ + + +class TestRenderConstraintViolationsTable: + """Test _render_constraint_violations_table function.""" + + def test_empty_data_shows_info(self, base_survey_data, outlier_settings): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + _render_constraint_violations_table( + base_survey_data, + pl.DataFrame(), + outlier_settings, + "settings.json", + ) + st_mock.info.assert_called_once() + + def test_non_empty_data_shows_dataframe(self, base_survey_data, outlier_settings): + violation_data = pl.DataFrame( + { + "survey_key": ["K001"], + "column name": ["col1"], + "violation reason": ["below soft minimum"], + } + ) + with ( + patch("datasure.checks.outliers.report_ui.st") as st_mock, + patch( + "datasure.checks.outliers.report_ui.load_check_settings", + return_value={}, + ), + patch("datasure.checks.outliers.report_ui.save_check_settings"), + ): + st_mock.columns.side_effect = _columns_side_effect + st_mock.multiselect.return_value = [] + _render_constraint_violations_table( + base_survey_data, + violation_data, + outlier_settings, + "settings.json", + ) + st_mock.dataframe.assert_called_once() + + def test_non_empty_with_extra_display_cols( + self, base_survey_data, outlier_settings + ): + violation_data = pl.DataFrame( + { + "survey_key": ["K001"], + "violation reason": ["above hard maximum"], + } + ) + with ( + patch("datasure.checks.outliers.report_ui.st") as st_mock, + patch( + "datasure.checks.outliers.report_ui.load_check_settings", + return_value={}, + ), + patch("datasure.checks.outliers.report_ui.save_check_settings"), + ): + st_mock.columns.side_effect = _columns_side_effect + st_mock.multiselect.return_value = [] + _render_constraint_violations_table( + base_survey_data, + violation_data, + outlier_settings, + "settings.json", + ) + st_mock.dataframe.assert_called_once() + + +# ============================================================================ +# TESTS: _render_outlier_table +# ============================================================================ + + +class TestRenderOutlierTable: + """Test _render_outlier_table function.""" + + def test_empty_data_shows_info(self, base_survey_data, outlier_settings): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + _render_outlier_table( + base_survey_data, + pl.DataFrame(), + outlier_settings, + "settings.json", + ) + st_mock.info.assert_called_once() + + def test_non_empty_data_shows_dataframe(self, base_survey_data, outlier_settings): + outliers_data = pl.DataFrame( + { + "survey_key": ["K001"], + "column name": ["col1"], + "outlier reason": ["Value is above upper bound 50.00"], + } + ) + with ( + patch("datasure.checks.outliers.report_ui.st") as st_mock, + patch( + "datasure.checks.outliers.report_ui.load_check_settings", + return_value={}, + ), + patch("datasure.checks.outliers.report_ui.save_check_settings"), + ): + st_mock.columns.side_effect = _columns_side_effect + st_mock.multiselect.return_value = [] + _render_outlier_table( + base_survey_data, + outliers_data, + outlier_settings, + "settings.json", + ) + st_mock.dataframe.assert_called_once() + + +# ============================================================================ +# TESTS: _render_outlier_column_inspection +# ============================================================================ + + +class TestRenderOutlierColumnInspection: + """Test _render_outlier_column_inspection function.""" + + def test_empty_outlier_data_shows_info(self, base_survey_data, outlier_settings): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + _render_outlier_column_inspection( + base_survey_data, pl.DataFrame(), outlier_settings, "settings.json" + ) + st_mock.info.assert_called_once() + + def test_no_selected_col_returns_early(self, base_survey_data, outlier_settings): + outliers_data = pl.DataFrame( + {"survey_key": ["K001"], "column name": ["survey_key"]} + ) + with ( + patch("datasure.checks.outliers.report_ui.st") as st_mock, + patch( + "datasure.checks.outliers.report_ui.load_check_settings", + return_value={}, + ), + patch("datasure.checks.outliers.report_ui.save_check_settings"), + ): + st_mock.columns.side_effect = _columns_side_effect + st_mock.selectbox.return_value = None + _render_outlier_column_inspection( + base_survey_data, outliers_data, outlier_settings, "settings.json" + ) + st_mock.info.assert_called() + + def test_col_not_in_data_raises(self, base_survey_data, outlier_settings): + outliers_data = pl.DataFrame( + {"survey_key": ["K001"], "column name": ["nonexistent_col"]} + ) + with ( + patch("datasure.checks.outliers.report_ui.st") as st_mock, + patch( + "datasure.checks.outliers.report_ui.load_check_settings", + return_value={}, + ), + patch("datasure.checks.outliers.report_ui.save_check_settings"), + ): + st_mock.columns.side_effect = _columns_side_effect + st_mock.selectbox.return_value = "nonexistent_col" + with pytest.raises(ValueError, match="not present in the data"): + _render_outlier_column_inspection( + base_survey_data, + outliers_data, + outlier_settings, + "settings.json", + ) + + def test_normal_path_renders_chart_and_table(self, outlier_settings): + data = pl.DataFrame( + { + "survey_key": ["K001", "K002"], + "survey_id": ["S001", "S002"], + "survey_date": ["2024-01-01", "2024-01-02"], + "enumerator": ["E001", "E002"], + "team": ["T1", "T2"], + "numeric_col1": [1.0, 100.0], + } + ) + outliers_data = pl.DataFrame( + { + "survey_key": ["K001"], + "column name": ["numeric_col1"], + "outlier reason": ["Value is above upper bound 50.00"], + } + ) + with ( + patch("datasure.checks.outliers.report_ui.st") as st_mock, + patch( + "datasure.checks.outliers.report_ui.load_check_settings", + return_value={}, + ), + patch("datasure.checks.outliers.report_ui.save_check_settings"), + patch( + "datasure.checks.outliers.report_ui._create_descriptive_stats" + ) as mock_desc, + patch("datasure.checks.outliers.report_ui._create_box_plot") as mock_box, + ): + st_mock.columns.side_effect = _columns_side_effect + st_mock.selectbox.return_value = "numeric_col1" + st_mock.multiselect.return_value = [] + mock_desc.return_value = pl.DataFrame( + {"statistic": ["count"], "value": ["2"]} + ) + mock_box.return_value = MagicMock() + _render_outlier_column_inspection( + data, outliers_data, outlier_settings, "settings.json" + ) + st_mock.dataframe.assert_called() + + +# ============================================================================ +# TESTS: _create_search_type_info +# ============================================================================ + + +class TestCreateSearchTypeInfo: + """Test _create_search_type_info function.""" + + def test_exact_search_type(self): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + _create_search_type_info(SearchType.EXACT.value) + st_mock.info.assert_called_once() + + def test_startswith_search_type(self): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + _create_search_type_info(SearchType.STARTSWITH.value) + st_mock.info.assert_called_once() + + def test_endswith_search_type(self): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + _create_search_type_info(SearchType.ENDSWITH.value) + st_mock.info.assert_called_once() + + def test_contains_search_type(self): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + _create_search_type_info(SearchType.CONTAINS.value) + st_mock.info.assert_called_once() + + def test_regex_search_type(self): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + _create_search_type_info(SearchType.REGEX.value) + st_mock.info.assert_called_once() + + def test_unknown_type(self): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + _create_search_type_info("unknown_type") + st_mock.info.assert_called_once() + + +# ============================================================================ +# TESTS: _render_search_type_selection +# ============================================================================ + + +class TestRenderSearchTypeSelection: + """Test _render_search_type_selection function.""" + + def test_exact_search_type(self): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + st_mock.columns.side_effect = _columns_side_effect + st_mock.selectbox.return_value = SearchType.EXACT.value + st_mock.multiselect.return_value = ["col1"] + search_type, pattern, cols, _lock = _render_search_type_selection( + ["col1", "col2"] + ) + assert search_type == SearchType.EXACT.value + assert pattern is None + assert cols == ["col1"] + + def test_pattern_search_type_with_pattern(self): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + st_mock.selectbox.return_value = SearchType.STARTSWITH.value + st_mock.text_input.return_value = "num" + search_type, pattern, cols, _lock = _render_search_type_selection( + ["num_col1", "num_col2", "other"] + ) + assert search_type == SearchType.STARTSWITH.value + assert pattern == "num" + assert "num_col1" in cols + + def test_pattern_search_type_no_pattern(self): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + st_mock.selectbox.return_value = SearchType.CONTAINS.value + st_mock.text_input.return_value = "" + _search_type, _pattern, cols, lock = _render_search_type_selection( + ["col1", "col2"] + ) + assert cols == [] + assert lock is None + + +# ============================================================================ +# TESTS: _render_column_grouping_options +# ============================================================================ + + +class TestRenderColumnGroupingOptions: + """Test _render_column_grouping_options function.""" + + def test_basic_render(self): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + st_mock.columns.side_effect = _columns_side_effect + st_mock.toggle.return_value = False + group_cols, lock_cols = _render_column_grouping_options( + ["col1", "col2"], SearchType.EXACT.value + ) + assert isinstance(group_cols, bool) + assert isinstance(lock_cols, bool) + + def test_returns_toggle_values(self): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + st_mock.columns.side_effect = _columns_side_effect + st_mock.toggle.side_effect = [True, False] + group_cols, lock_cols = _render_column_grouping_options( + ["col1", "col2"], SearchType.STARTSWITH.value + ) + assert group_cols is True + assert lock_cols is False + + +# ============================================================================ +# TESTS: _render_outlier_options +# ============================================================================ + + +class TestRenderOutlierOptions: + """Test _render_outlier_options function.""" + + def test_outliers_enabled_returns_settings(self): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + st_mock.columns.side_effect = _columns_side_effect + st_mock.toggle.return_value = True + st_mock.selectbox.return_value = OutlierMethod.IQR.value + st_mock.number_input.side_effect = [1.5, 20] + enabled, settings, valid = _render_outlier_options() + assert enabled is True + assert settings is not None + assert valid is True + + def test_outliers_enabled_sd_method(self): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + st_mock.columns.side_effect = _columns_side_effect + st_mock.toggle.return_value = True + st_mock.selectbox.return_value = OutlierMethod.SD.value + st_mock.number_input.side_effect = [3.0, 30] + enabled, _settings, valid = _render_outlier_options() + assert enabled is True + assert valid is True + + def test_outliers_disabled_returns_none(self): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + st_mock.columns.side_effect = _columns_side_effect + st_mock.toggle.return_value = False + enabled, settings, valid = _render_outlier_options() + assert enabled is False + assert settings is None + assert valid is True + + +# ============================================================================ +# TESTS: _render_constraint_options +# ============================================================================ + + +class TestRenderConstraintOptions: + """Test _render_constraint_options function.""" + + def test_valid_settings_returns_true(self): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + st_mock.columns.side_effect = _columns_side_effect + st_mock.number_input.return_value = None + _settings, valid = _render_constraint_options() + assert valid is True + + def test_invalid_settings_calls_error(self): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + st_mock.columns.side_effect = _columns_side_effect + st_mock.number_input.side_effect = [50.0, 10.0, None, None] + _settings, valid = _render_constraint_options() + assert valid is False + st_mock.error.assert_called_once() + + +# ============================================================================ +# TESTS: _render_outlier_settings_table +# ============================================================================ + + +class TestRenderOutlierSettingsTable: + """Test _render_outlier_settings_table function.""" + + def test_renders_dataframe(self, outlier_column_config): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + _render_outlier_settings_table(outlier_column_config) + st_mock.dataframe.assert_called_once() + + +# ============================================================================ +# TESTS: _render_outlier_column_actions +# ============================================================================ + + +class TestRenderOutlierColumnActions: + """Test _render_outlier_column_actions function.""" + + def test_empty_settings_shows_info(self): + with ( + patch("datasure.checks.outliers.report_ui.st") as st_mock, + patch( + "datasure.checks.outliers.report_ui.duckdb_get_table", + return_value=pl.DataFrame(), + ), + ): + st_mock.columns.side_effect = _columns_side_effect + _render_outlier_column_actions("proj1", "page1", ["col1"]) + assert st_mock.info.call_count >= 1 + + def test_non_empty_settings_calls_render_table(self, outlier_column_config): + with ( + patch("datasure.checks.outliers.report_ui.st") as st_mock, + patch( + "datasure.checks.outliers.report_ui.duckdb_get_table", + return_value=outlier_column_config, + ), + patch( + "datasure.checks.outliers.report_ui._render_outlier_settings_table" + ) as mock_render, + patch("datasure.checks.outliers.report_ui._delete_outlier_column"), + ): + st_mock.columns.side_effect = _columns_side_effect + _render_outlier_column_actions("proj1", "page1", ["col1"]) + mock_render.assert_called_once() + + +# ============================================================================ +# TESTS: _update_outlier_column_config +# ============================================================================ + + +class TestUpdateOutlierColumnConfig: + """Test _update_outlier_column_config function.""" + + def test_empty_existing_config_saves_new(self): + settings = OutlierOptionsConfig( + outlier_method=OutlierMethod.IQR, + outlier_multiplier=1.5, + outlier_threshold=20, + ) + bounds = ConstraintBounds(soft_min=0.0, soft_max=100.0) + with ( + patch( + "datasure.checks.outliers.report_ui.duckdb_get_table", + return_value=pl.DataFrame(), + ), + patch("datasure.checks.outliers.report_ui.duckdb_save_table") as mock_save, + ): + _update_outlier_column_config( + "proj1", + "page1", + "exact", + None, + ["col1"], + False, + False, + True, + settings, + bounds, + ) + mock_save.assert_called_once() + + def test_non_empty_existing_config_concatenates(self, outlier_column_config): + settings = OutlierOptionsConfig( + outlier_method=OutlierMethod.IQR, + outlier_multiplier=1.5, + outlier_threshold=20, + ) + bounds = ConstraintBounds(soft_min=0.0, soft_max=100.0) + with ( + patch( + "datasure.checks.outliers.report_ui.duckdb_get_table", + return_value=outlier_column_config, + ), + patch("datasure.checks.outliers.report_ui.duckdb_save_table") as mock_save, + ): + _update_outlier_column_config( + "proj1", + "page1", + "exact", + None, + ["col2"], + False, + False, + True, + settings, + bounds, + ) + mock_save.assert_called_once() + saved_df = mock_save.call_args[0][1] + assert len(saved_df) == 2 + + def test_outlier_settings_none(self): + bounds = ConstraintBounds(soft_min=0.0, soft_max=100.0) + with ( + patch( + "datasure.checks.outliers.report_ui.duckdb_get_table", + return_value=pl.DataFrame(), + ), + patch("datasure.checks.outliers.report_ui.duckdb_save_table") as mock_save, + ): + _update_outlier_column_config( + "proj1", + "page1", + "exact", + None, + ["col1"], + False, + False, + False, + None, + bounds, + ) + mock_save.assert_called_once() + + +# ============================================================================ +# TESTS: _delete_outlier_column +# ============================================================================ + + +class TestDeleteOutlierColumn: + """Test _delete_outlier_column function.""" + + def test_empty_settings_shows_info(self): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + _delete_outlier_column("proj1", "page1", pl.DataFrame()) + st_mock.info.assert_called_once() + + def test_non_empty_shows_selectbox_and_button(self, outlier_column_config): + with patch("datasure.checks.outliers.report_ui.st") as st_mock: + st_mock.selectbox.return_value = "0 - exact - " + st_mock.button.return_value = False + _delete_outlier_column("proj1", "page1", outlier_column_config) + st_mock.selectbox.assert_called_once() + + def test_delete_on_button_click(self, outlier_column_config): + with ( + patch("datasure.checks.outliers.report_ui.st") as st_mock, + patch("datasure.checks.outliers.report_ui.duckdb_save_table") as mock_save, + ): + st_mock.selectbox.return_value = "0 - exact - " + st_mock.button.return_value = True + _delete_outlier_column("proj1", "page1", outlier_column_config) + mock_save.assert_called_once() + + +# ============================================================================ +# TESTS: _add_outlier_column (via reimport) +# ============================================================================ + + +class TestAddOutlierColumn: + """Test _add_outlier_column function.""" + + def test_no_cols_selected_does_not_save(self, report_ui_mod): + """When no columns selected, skip grouping/options rendering.""" + mock_sel = MagicMock(return_value=(SearchType.EXACT.value, None, [], None)) + with patch.object(report_ui_mod, "_render_search_type_selection", mock_sel): + report_ui_mod._add_outlier_column("proj1", "page1", ["col1", "col2"]) + mock_sel.assert_called_once() + + def test_with_cols_selected_renders_options(self, report_ui_mod): + """When columns are selected, all option panels are rendered.""" + settings = OutlierOptionsConfig( + outlier_method=OutlierMethod.IQR, + outlier_multiplier=1.5, + outlier_threshold=20, + ) + mock_sel = MagicMock( + return_value=(SearchType.EXACT.value, None, ["col1"], None) + ) + mock_grp = MagicMock(return_value=(False, False)) + mock_out = MagicMock(return_value=(True, settings, True)) + mock_con = MagicMock(return_value=(ConstraintBounds(), True)) + with ( + patch.object(report_ui_mod, "_render_search_type_selection", mock_sel), + patch.object(report_ui_mod, "_render_column_grouping_options", mock_grp), + patch.object(report_ui_mod, "_render_outlier_options", mock_out), + patch.object(report_ui_mod, "_render_constraint_options", mock_con), + ): + report_ui_mod.st.button.return_value = False + report_ui_mod._add_outlier_column("proj1", "page1", ["col1", "col2"]) + mock_grp.assert_called_once() + mock_out.assert_called_once() + mock_con.assert_called_once() + + +# ============================================================================ +# TESTS: outliers_report (main function) +# ============================================================================ + + +class TestOutliersReport: + """Test outliers_report main function.""" + + def test_empty_column_config_returns_early( + self, base_survey_data, survey_columns_mock, outlier_config_dict + ): + settings = OutlierSettings(**outlier_config_dict) + with ( + patch("datasure.checks.outliers.report_ui.st") as st_mock, + patch( + "datasure.checks.outliers.report_ui.outliers_report_settings", + return_value=settings, + ), + patch("datasure.checks.outliers.report_ui._render_outlier_column_actions"), + patch( + "datasure.checks.outliers.report_ui.duckdb_get_table", + return_value=pl.DataFrame(), + ), + ): + st_mock.columns.side_effect = _columns_side_effect + outliers_report( + "proj1", + "page1", + base_survey_data, + "settings.json", + outlier_config_dict, + survey_columns_mock, + ) + st_mock.title.assert_called() + + def test_with_constraint_violations( + self, + base_survey_data, + survey_columns_mock, + outlier_config_dict, + outlier_column_config, + ): + settings = OutlierSettings(**outlier_config_dict) + constraint_violations = pl.DataFrame( + { + "survey_key": ["K001"], + "column name": ["col1"], + "violation reason": ["below soft minimum"], + } + ) + with ( + patch("datasure.checks.outliers.report_ui.st") as st_mock, + patch( + "datasure.checks.outliers.report_ui.outliers_report_settings", + return_value=settings, + ), + patch("datasure.checks.outliers.report_ui._render_outlier_column_actions"), + patch( + "datasure.checks.outliers.report_ui.duckdb_get_table", + return_value=outlier_column_config, + ), + patch("datasure.checks.outliers.report_ui.duckdb_save_table"), + patch( + "datasure.checks.outliers.report_ui.compute_constraint_violations", + return_value=constraint_violations, + ), + patch( + "datasure.checks.outliers.report_ui.compute_outlier_output", + return_value=pl.DataFrame(), + ), + patch("datasure.checks.outliers.report_ui._render_constraint_metrics"), + patch( + "datasure.checks.outliers.report_ui._render_constraint_violations_table" + ), + ): + st_mock.columns.side_effect = _columns_side_effect + outliers_report( + "proj1", + "page1", + base_survey_data, + "settings.json", + outlier_config_dict, + survey_columns_mock, + ) + st_mock.info.assert_called() + + def test_with_outliers( + self, + base_survey_data, + survey_columns_mock, + outlier_config_dict, + outlier_column_config, + ): + settings = OutlierSettings(**outlier_config_dict) + outlier_data = pl.DataFrame( + { + "survey_key": ["K001"], + "column name": ["col1"], + "outlier reason": ["Value is above upper bound 50.00"], + } + ) + with ( + patch("datasure.checks.outliers.report_ui.st") as st_mock, + patch( + "datasure.checks.outliers.report_ui.outliers_report_settings", + return_value=settings, + ), + patch("datasure.checks.outliers.report_ui._render_outlier_column_actions"), + patch( + "datasure.checks.outliers.report_ui.duckdb_get_table", + return_value=outlier_column_config, + ), + patch("datasure.checks.outliers.report_ui.duckdb_save_table"), + patch( + "datasure.checks.outliers.report_ui.compute_constraint_violations", + return_value=pl.DataFrame(), + ), + patch( + "datasure.checks.outliers.report_ui.compute_outlier_output", + return_value=outlier_data, + ), + patch("datasure.checks.outliers.report_ui._render_outlier_metrics"), + patch( + "datasure.checks.outliers.report_ui._render_outlier_column_inspection" + ), + ): + st_mock.columns.side_effect = _columns_side_effect + outliers_report( + "proj1", + "page1", + base_survey_data, + "settings.json", + outlier_config_dict, + survey_columns_mock, + ) + st_mock.info.assert_called() diff --git a/tests/checks/outliers/test_settings_ui.py b/tests/checks/outliers/test_settings_ui.py new file mode 100644 index 00000000..3203487b --- /dev/null +++ b/tests/checks/outliers/test_settings_ui.py @@ -0,0 +1,75 @@ +"""Tests for datasure.checks.outliers.settings_ui.""" + +import importlib +import sys +from unittest.mock import patch + +import pytest + +from datasure.checks.outliers.models import OutlierSettings +from tests.checks.outliers.conftest import _make_st_mock + +# ============================================================================ +# SETTINGS_UI_MOD FIXTURE (reload compute/settings_ui with mocked streamlit) +# ============================================================================ + + +@pytest.fixture +def settings_ui_mod(): + """Reload compute/settings_ui with mocked Streamlit and onboarding decorator.""" + mock_st = _make_st_mock() + original_st = sys.modules.get("streamlit") + sys.modules["streamlit"] = mock_st + + import datasure.checks.outliers.compute as compute_module + import datasure.checks.outliers.settings_ui as settings_ui_module + + try: + with patch( + "datasure.utils.onboarding_utils.demo_output_onboarding", + lambda tab: lambda f: f, + ): + importlib.reload(compute_module) + importlib.reload(settings_ui_module) + yield settings_ui_module + finally: + if original_st is not None: + sys.modules["streamlit"] = original_st + else: + sys.modules.pop("streamlit", None) + importlib.reload(compute_module) + importlib.reload(settings_ui_module) + + +# ============================================================================ +# TESTS: outliers_report_settings (via reimport) +# ============================================================================ + + +class TestOutliersReportSettings: + """Test outliers_report_settings function.""" + + def test_returns_outlier_settings(self, settings_ui_mod): + config = OutlierSettings( + survey_key="survey_key", + survey_id="survey_id", + survey_date="survey_date", + enumerator="enumerator", + team="team", + ) + with ( + patch( + "datasure.checks.outliers.settings_ui.load_default_settings", + return_value=config, + ), + patch("datasure.checks.outliers.settings_ui.save_check_settings"), + patch("datasure.checks.outliers.settings_ui.trigger_save"), + ): + settings_ui_mod.st.selectbox.return_value = "survey_key" + result = settings_ui_mod.outliers_report_settings( + "settings.json", + config, + ["survey_key", "survey_id", "enumerator", "team"], + ["survey_date"], + ) + assert isinstance(result, settings_ui_mod.OutlierSettings)