Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions src/datasure/processing/corrections.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,8 @@ def add_correction_entry(
key_value : str
The key value being corrected
current_id : str | None
The current ID value
The Survey ID value for this KEY, if a Survey ID column is
configured for the dataset
action : str
The correction action
column : str | None
Expand Down Expand Up @@ -231,6 +232,7 @@ def apply_correction(
current_value: Any | None = None,
new_value: Any | None = None,
reason: str | None = None,
survey_id_value: Any | None = None,
) -> pl.DataFrame:
"""Apply a single correction to the data.

Expand All @@ -252,6 +254,9 @@ def apply_correction(
The new value
reason : str | None
The reason for correction
survey_id_value : Any | None
The Survey ID value for this KEY, if a Survey ID column is
configured, recorded in the log's ID column

Returns
-------
Expand All @@ -278,7 +283,7 @@ def apply_correction(
self.add_correction_entry(
alias=alias,
key_value=key_value,
current_id=None, # Legacy field, not used in new implementation
current_id=survey_id_value,
action=action,
column=column,
current_value=current_value,
Expand Down
44 changes: 40 additions & 4 deletions src/datasure/views/correction_view.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ class TabConfig(BaseModel):
page_name: str = Field(..., description="Name of the page/check")
survey_data_name: str = Field(..., description="Name of the survey data alias")
survey_key: str = Field(..., description="Name of the survey KEY column")
survey_id: str | None = Field(
None, description="Name of the survey ID column, if configured"
)


class CorrectionFormState(BaseModel):
Expand Down Expand Up @@ -231,6 +234,7 @@ def load_tab_config(project_id: str, tab_index: int) -> TabConfig | None:
page_name=page_config.get("page_name"),
survey_data_name=page_config.get("survey_data_name"),
survey_key=page_config.get("survey_key"),
survey_id=page_config.get("survey_id"),
)


Expand Down Expand Up @@ -481,6 +485,7 @@ def render_add_correction_form(
key_col: str,
alias: str,
tab_index: int,
survey_id_col: str | None = None,
) -> None:
"""
Render the add correction step form.
Expand All @@ -498,6 +503,10 @@ def render_add_correction_form(
The data alias/table name.
tab_index : int
The tab index for unique widget keys.
survey_id_col : str | None
The name of the configured Survey ID column, if any. When set (and
present in the data), the corresponding Survey ID is shown once a
KEY is selected.
"""
corrected_data = correction_processor.get_corrected_data(alias)

Expand All @@ -519,6 +528,13 @@ def render_add_correction_form(
if not corr_key_val:
return

survey_id_value = None
if survey_id_col and survey_id_col in corrected_data.columns:
survey_id_value = get_current_value(
corrected_data, key_col, corr_key_val, survey_id_col
)
st.write(f"**Survey ID:** {survey_id_value}")

# Step 2: Select action
corr_action = st.selectbox(
label="Select Action",
Expand Down Expand Up @@ -547,6 +563,7 @@ def render_add_correction_form(
form_state=form_state,
reason=reason,
tab_index=tab_index,
survey_id_value=survey_id_value,
)


Expand Down Expand Up @@ -600,6 +617,7 @@ def _render_apply_button(
form_state: CorrectionFormState,
reason: str,
tab_index: int,
survey_id_value: Any = None,
) -> None:
"""
Render apply button and handle correction application.
Expand All @@ -620,6 +638,9 @@ def _render_apply_button(
Reason for correction.
tab_index : int
The tab index for unique widget keys.
survey_id_value : Any
The Survey ID value for the selected KEY, if a Survey ID column is
configured, to record alongside the correction log entry.
"""
apply_enabled = should_enable_apply_button(
form_state.action, reason, form_state.new_value
Expand All @@ -645,6 +666,7 @@ def _render_apply_button(
current_value=form_state.current_value,
new_value=form_state.new_value,
reason=reason,
survey_id_value=survey_id_value,
)


Expand All @@ -659,6 +681,7 @@ def _handle_apply_correction(
current_value: Any,
new_value: Any,
reason: str,
survey_id_value: Any = None,
) -> None:
"""
Handle the application of a correction with validation.
Expand All @@ -685,6 +708,9 @@ def _handle_apply_correction(
The new value (if applicable).
reason : str
The reason for correction.
survey_id_value : Any
The Survey ID value for this KEY, if a Survey ID column is
configured, to record alongside the correction log entry.
"""
try:
# Validate input
Expand All @@ -711,6 +737,7 @@ def _handle_apply_correction(
current_value=current_value,
new_value=new_value,
reason=reason,
survey_id_value=survey_id_value,
)

st.success("Correction applied successfully!")
Expand All @@ -725,6 +752,7 @@ def render_correction_input_form(
key_col: str,
alias: str,
tab_index: int,
survey_id_col: str | None = None,
) -> None:
"""
Render input form for corrections with add and remove functionality.
Expand All @@ -739,6 +767,8 @@ def render_correction_input_form(
The data alias/table name.
tab_index : int
The tab index for unique widget keys.
survey_id_col : str | None
The name of the configured Survey ID column, if any.
"""
corrected_data = correction_processor.get_corrected_data(alias)

Expand All @@ -754,6 +784,7 @@ def render_correction_input_form(
key_col=key_col,
alias=alias,
tab_index=tab_index,
survey_id_col=survey_id_col,
)

with fc2:
Expand Down Expand Up @@ -892,8 +923,9 @@ def _handle_remove_correction(
def _build_correction_log_display(correction_log: pl.DataFrame) -> pl.DataFrame:
"""Prepare a correction log for display in the Correction Log table.

Backfills the status columns for logs saved before they existed, and
orders columns so status/status_reason sit right after action.
Backfills the status columns for logs saved before they existed, orders
columns so status/status_reason sit right after action, and relabels the
"ID" column as "Survey ID" for display.

Parameters
----------
Expand All @@ -903,7 +935,8 @@ def _build_correction_log_display(correction_log: pl.DataFrame) -> pl.DataFrame:
Returns
-------
pl.DataFrame
The log with status columns present, in display column order.
The log with status columns present, in display column order, ready
for display.
"""
if "status" not in correction_log.columns:
correction_log = correction_log.with_columns(
Expand All @@ -926,7 +959,9 @@ def _build_correction_log_display(correction_log: pl.DataFrame) -> pl.DataFrame:
"new_value",
"reason",
]
return correction_log.select(display_columns)
# "ID" holds the Survey ID value recorded for the KEY, if one was
# configured - rename it for display so the column reads clearly.
return correction_log.select(display_columns).rename({"ID": "Survey ID"})


@st.fragment
Expand Down Expand Up @@ -1032,6 +1067,7 @@ def render_correction_tab(
key_col=config.survey_key,
alias=config.survey_data_name,
tab_index=tab_index,
survey_id_col=config.survey_id,
)

render_correction_log(
Expand Down
67 changes: 67 additions & 0 deletions tests/processing/test_corrections.py
Original file line number Diff line number Diff line change
Expand Up @@ -469,6 +469,73 @@ def test_apply_correction_without_reason(self, correction_processor, sample_data
# Should only save data, not log (since no reason)
assert mock_save.call_count == 1

def test_apply_correction_records_survey_id(
self, correction_processor, sample_data
):
"""The Survey ID for the corrected KEY is recorded in the log's ID column."""
processor, mock_get, mock_save = correction_processor
empty_log = pl.DataFrame(
{
"date": [],
"KEY": [],
"ID": [],
"action": [],
"column": [],
"current_value": [],
"new_value": [],
"reason": [],
}
)
mock_get.side_effect = [sample_data, empty_log]

processor.apply_correction(
alias="test_alias",
key_col="survey_key",
key_value="key2",
action="modify value",
column="name",
current_value="Jane",
new_value="Janet",
reason="Name correction",
survey_id_value="HH002",
)

saved_log = mock_save.call_args[1]["table_data"]
assert saved_log["ID"][0] == "HH002"

def test_apply_correction_without_survey_id_leaves_id_blank(
self, correction_processor, sample_data
):
"""No Survey ID configured/available means the ID column stays blank."""
processor, mock_get, mock_save = correction_processor
empty_log = pl.DataFrame(
{
"date": [],
"KEY": [],
"ID": [],
"action": [],
"column": [],
"current_value": [],
"new_value": [],
"reason": [],
}
)
mock_get.side_effect = [sample_data, empty_log]

processor.apply_correction(
alias="test_alias",
key_col="survey_key",
key_value="key2",
action="modify value",
column="name",
current_value="Jane",
new_value="Janet",
reason="Name correction",
)

saved_log = mock_save.call_args[1]["table_data"]
assert saved_log["ID"][0] is None

def test_get_data_summary(self, correction_processor, sample_data):
"""Test getting data summary."""
processor, _, _ = correction_processor
Expand Down
Loading
Loading