From 67005b6b2333372f183215fc85bc422c8f03748f Mon Sep 17 00:00:00 2001 From: Modeleon Date: Mon, 26 Oct 2026 13:27:32 +0500 Subject: [PATCH 1/2] modeleon 0.2.3 --- CHANGELOG.md | 678 + README.md | 36 +- pyproject.toml | 2 +- src/modeleon/__init__.py | 112 +- src/modeleon/compile/compat.py | 8 +- src/modeleon/compile/excel/header.py | 183 +- src/modeleon/compile/excel/layout.py | 179 +- src/modeleon/compile/excel/renderer.py | 597 +- src/modeleon/compile/excel/totals.py | 144 +- src/modeleon/compile/excel/totals_writer.py | 42 +- src/modeleon/compile/excel/view.py | 311 +- src/modeleon/compile/excel/writer.py | 231 +- src/modeleon/compile/renderer.py | 7 + src/modeleon/core/calendar.py | 598 + src/modeleon/core/entries.py | 326 +- src/modeleon/core/expr.py | 56 +- src/modeleon/core/loops.py | 525 +- src/modeleon/core/multi_variable.py | 177 +- src/modeleon/core/mv_context.py | 165 +- src/modeleon/core/mv_inspect.py | 15 +- src/modeleon/core/projection.py | 53 +- src/modeleon/core/regrain.py | 2 +- src/modeleon/core/shape.py | 33 +- src/modeleon/core/time.py | 466 +- src/modeleon/core/unit.py | 118 +- src/modeleon/core/variable.py | 400 +- src/modeleon/core/variable_init.py | 37 +- src/modeleon/core/variable_ops.py | 1276 +- src/modeleon/display/code.py | 4 +- src/modeleon/display/html.py | 6 +- src/modeleon/functions/__init__.py | 90 +- src/modeleon/functions/_helpers.py | 553 +- src/modeleon/functions/aggregate.py | 335 +- src/modeleon/functions/conditional.py | 184 +- src/modeleon/functions/criteria.py | 1581 + src/modeleon/functions/dates.py | 489 +- src/modeleon/functions/financial.py | 730 +- src/modeleon/functions/lag.py | 59 +- src/modeleon/functions/logical.py | 402 + src/modeleon/functions/lookup.py | 962 + src/modeleon/functions/mathfn.py | 530 +- src/modeleon/functions/recurrence.py | 281 +- src/modeleon/functions/registry.py | 235 + src/modeleon/functions/statistics.py | 1052 + src/modeleon/functions/text.py | 706 +- tests/golden/forecast.xlsx | Bin 6352 -> 6908 bytes tests/golden/time_calendar.json | 79458 ++++++++++++++++++ tests/test_aggregates_per_period.py | 893 + tests/test_calendar.py | 181 + tests/test_choose_keyed_line.py | 288 + tests/test_component.py | 19 + tests/test_criteria.py | 757 + tests/test_dates_extra.py | 647 + tests/test_day_month_totals.py | 87 + tests/test_entries.py | 54 + tests/test_error_cells_in_functions.py | 10 +- tests/test_excel_board_list_field.py | 194 + tests/test_excel_criteria.py | 525 + tests/test_excel_dates.py | 323 + tests/test_excel_financial.py | 331 + tests/test_excel_house_look.py | 56 + tests/test_excel_logical.py | 246 + tests/test_excel_lookup.py | 472 + tests/test_excel_math.py | 381 + tests/test_excel_statistics.py | 711 + tests/test_excel_text.py | 339 + tests/test_excel_time_header.py | 11 +- tests/test_excel_view.py | 220 +- tests/test_excel_view_pass.py | 171 + tests/test_financial_schedules.py | 694 + tests/test_function_ast_completeness.py | 164 +- tests/test_function_registry.py | 801 + tests/test_grain_vocabulary.py | 132 + tests/test_half_year.py | 114 + tests/test_helper_functions.py | 6 +- tests/test_hour.py | 343 + tests/test_indexed_by.py | 247 + tests/test_lag.py | 16 + tests/test_lines_on_other_windows.py | 1752 + tests/test_logical.py | 681 + tests/test_lookup.py | 733 + tests/test_loops.py | 205 + tests/test_mathfn.py | 666 + tests/test_minmax_elementwise.py | 5 +- tests/test_multi_variable.py | 172 + tests/test_operator_fast_paths.py | 209 + tests/test_passthrough_hygiene.py | 52 + tests/test_range_node.py | 628 + tests/test_records_scale.py | 293 + tests/test_series_arguments.py | 270 + tests/test_source_code_lazy.py | 158 + tests/test_statistics.py | 623 + tests/test_statistics_per_period.py | 617 + tests/test_text_functions.py | 778 + tests/test_time_axis.py | 24 +- tests/test_time_golden.py | 209 + tests/test_track_expr_list_refs.py | 73 + tests/test_unattached_aggregates.py | 30 +- tests/test_unit.py | 1390 +- tests/test_week.py | 270 + 100 files changed, 112494 insertions(+), 1211 deletions(-) create mode 100644 src/modeleon/core/calendar.py create mode 100644 src/modeleon/functions/criteria.py create mode 100644 src/modeleon/functions/logical.py create mode 100644 src/modeleon/functions/lookup.py create mode 100644 src/modeleon/functions/registry.py create mode 100644 src/modeleon/functions/statistics.py create mode 100644 tests/golden/time_calendar.json create mode 100644 tests/test_aggregates_per_period.py create mode 100644 tests/test_calendar.py create mode 100644 tests/test_choose_keyed_line.py create mode 100644 tests/test_criteria.py create mode 100644 tests/test_dates_extra.py create mode 100644 tests/test_day_month_totals.py create mode 100644 tests/test_entries.py create mode 100644 tests/test_excel_board_list_field.py create mode 100644 tests/test_excel_criteria.py create mode 100644 tests/test_excel_dates.py create mode 100644 tests/test_excel_financial.py create mode 100644 tests/test_excel_logical.py create mode 100644 tests/test_excel_lookup.py create mode 100644 tests/test_excel_math.py create mode 100644 tests/test_excel_statistics.py create mode 100644 tests/test_excel_text.py create mode 100644 tests/test_excel_view_pass.py create mode 100644 tests/test_financial_schedules.py create mode 100644 tests/test_function_registry.py create mode 100644 tests/test_grain_vocabulary.py create mode 100644 tests/test_half_year.py create mode 100644 tests/test_hour.py create mode 100644 tests/test_lines_on_other_windows.py create mode 100644 tests/test_logical.py create mode 100644 tests/test_lookup.py create mode 100644 tests/test_mathfn.py create mode 100644 tests/test_operator_fast_paths.py create mode 100644 tests/test_passthrough_hygiene.py create mode 100644 tests/test_range_node.py create mode 100644 tests/test_records_scale.py create mode 100644 tests/test_series_arguments.py create mode 100644 tests/test_source_code_lazy.py create mode 100644 tests/test_statistics.py create mode 100644 tests/test_statistics_per_period.py create mode 100644 tests/test_text_functions.py create mode 100644 tests/test_time_golden.py create mode 100644 tests/test_week.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bf0531..5a9d49c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,684 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.3] — 2026-10-26 + +### Added + +- **A named view, applied.** A model may keep several looks beside its + own — `model.board = mo.ExcelView('Board pack', font=…, bands=…)`, + `model.banker = mo.ExcelView('Banker', …)` — and `with + mo.applied_view(model.board): model.to_excel(path)` draws the book + under one of them: its fonts, bands, columns and formats over the + model's own view, whatever it leaves unset coming, as before, from the + model, a `mo.house_view(...)` around the call and the engine's defaults + (`mo.default_excel_view`); a `sheet={'order': …}` it declares orders + the book. The same model, the same numbers and formulas, in as many + looks as it declares. The period axis stays the model's: `orient` is + geometry, not a look. +- **Errors caught, and many branches without nesting.** `mo.IFERROR(value, + fallback)` shows the fallback where a division by zero or a lookup that + finds nothing would spread an error through the model (`=IFERROR(B7 / B8, + 0)` in every period); `mo.ISERROR` and `mo.ISNUMBER` test a cell, `mo.XOR` + joins conditions, and `mo.IFS(cond, value, …)` / `mo.SWITCH(expr, case, + value, …, default)` choose among many branches without nested `IF`s. Each + is one call per period over a row and runs inside a loop container; + `XOR`, `IFS` and `SWITCH` are stored as the `_xlfn.` names Excel expects. + `SWITCH` compares numbers to about 15 significant digits, as the + spreadsheet does; text given to `XOR` is refused with a pointer to compare + it instead. The values `IFERROR`, `IFS` and `SWITCH` may return are in + one unit, inside a loop container too (`mo.IFERROR(usd, eur)` is + refused; a plain `0` or a line without a unit goes with any), and + `SWITCH` compares its expression with each case in one unit + (`mo.SWITCH(hours, limit_usd, …)` is refused). +- **Rounding, powers and logarithms.** `mo.ROUNDUP`, `mo.ROUNDDOWN`, + `mo.TRUNC`, `mo.CEILING`, `mo.FLOOR`, `mo.POWER`, `mo.SQRT`, `mo.EXP`, + `mo.LN`, `mo.LOG`, `mo.LOG10` and `mo.SIGN`, one call per period, and + `mo.PRODUCT`: over one row the product of its cells, written over its + range (`=PRODUCT(C3:N3)`), over two or more operands with a series among + them one product per period (`=PRODUCT(C3, C4)` in each column), as `a * + b` multiplies them, and over scalars only one number. They round the + number as the cell shows it (`ROUNDUP(0.1 * 3, 1)` is 0.3, not 0.4), read + a date as its serial number, and give `#NUM!` where no cell could hold + the result; the same functions work inside a `mo.recurrence` template + (`"ROUNDUP({prev} * 1.05, 0)"`). A rounded amount keeps its unit; + `mo.POWER` and `mo.SQRT` raise it as `**` does (`mo.POWER(usd, 2)` is in + `USD^2`, `mo.SQRT` of a line in `USD^2` is in `USD`, `mo.SQRT(usd)` has + none); `mo.PRODUCT` multiplies its operands' units (`USD/t` times `t` is + `USD`), and over one row runs along pure numbers such as growth factors: + a row in a unit is refused, its cells' product being that unit to the + power of their count; `mo.CEILING` and `mo.FLOOR` take a number and its + multiple in one unit; `mo.EXP`, `mo.LN`, `mo.LOG`, `mo.LOG10` and + `mo.SIGN` carry none. The same holds inside a loop container. +- **Loan schedules and depreciation.** `mo.NPER`, `mo.RATE`, `mo.IPMT`, + `mo.PPMT`, `mo.CUMIPMT` and `mo.CUMPRINC` — how long a loan runs, what + rate it carries, and the interest and principal of each payment — and + `mo.SLN`, `mo.DB`, `mo.DDB` and `mo.SYD`. One call per period, so + `mo.PPMT(rate, mo.time.index, n, loan)` is a whole amortization row + (`=PPMT(I!B1, 1, 4, I!B2)`, `=PPMT(I!B1, 2, 4, I!B2)`, …), inside a loop + container too. `RATE` iterates as the spreadsheet does, retrying from its + default guesses, and is `#NUM!` where no rate balances the flows. The + amounts each one reads — a loan and its balloon, a payment, a cost and + its salvage — are in one unit, the result's (`mo.SLN(cost, salvage, + life)` with both in USD is in USD; a salvage in EUR is refused), and + `NPER` and `RATE` carry none. +- **Date arithmetic.** `mo.DATEDIF(start, end, unit)` with all six units, + Excel's quirks included (`"MD"` from 31 January to 1 March is -2 in a + common year); `mo.WEEKDAY` and `mo.WEEKNUM` with every return type (21 is + the ISO week); `mo.NETWORKDAYS` and `mo.WORKDAY`, whose holidays line is + written as the range every period's call reads (`=NETWORKDAYS(B2, C2, + Inputs!B5:D5)`). +- **Text.** `mo.LEFT`, `mo.RIGHT`, `mo.MID`, `mo.TRIM`, `mo.FIND`, + `mo.SEARCH`, `mo.SUBSTITUTE`, `mo.VALUE`, `mo.TEXT` (number and date + formats: `TEXT(d, "yyyy-mm-dd")`, `TEXT(x, "#,##0.00")`), `mo.REPT` and + `mo.TEXTJOIN`. An operand reads as the cell would: a number as its General + text, a date as its serial number, a blank as empty text. +- **Conditional sums and counts.** `mo.SUMIF`, `mo.SUMIFS`, `mo.COUNTIF`, + `mo.COUNTIFS`, `mo.AVERAGEIF`, `mo.AVERAGEIFS`, `mo.MAXIFS` and + `mo.MINIFS` over a row, or over a field down the records of a table + (`=SUMIFS(Deals!C3:C5, Deals!B3:B5, "North")`), with Excel's criteria — + `">=10"`, `"<>0"`, `"North*"`, `("<", threshold)` joined to a cell as + `"<" & B2` — and `mo.COUNT` / `mo.COUNTA`: over one row the count of its + cells (`=COUNT(C5:N5)`), over two or more operands with a series among + them one count per period (`=COUNT(C5, C9)` in each column), over scalars + only one number. The lines a `SUMIF`, `SUMIFS`, `AVERAGEIF`, + `AVERAGEIFS`, `MAXIFS` or `MINIFS` call totals, averages or picks from + are in one unit, which the result carries, and each criterion is compared + in its range's unit, one joined to a cell too: `mo.SUMIF(hours, ("<", + cap), revenue)` with the cap in USD is refused, inside a loop container + as outside, and an error the joined cell holds is the call's result. A + criterion a spreadsheet reads one way in some locales and another way in + others (`'1,000'`, `'1/31/2026'`) is refused when the model is built, + with the form to write instead (`'1000'`, `'2026-01-31'`). Where the + cells a conditional call reads cannot form one range — lines on two + sheets or with a line between them, or a row split by a totals column — + the call is written as its computed number and `check_compat` lists the + cell; `mo.COUNT` and `mo.COUNTA` name such cells one by one + (`=COUNT(East!B1, East!B3)`). +- **Lookups.** `mo.XLOOKUP`, `mo.INDEX`, `mo.MATCH`, `mo.VLOOKUP` and + `mo.HLOOKUP` over a row, a list of rows, or the records of a table; a key + that is a series gives one lookup per period (`=INDEX(Inputs!B3:D3, + MATCH(B2, Inputs!B2:D2, 0))`, then `MATCH(C2, …)`), so a year-varying + assumption is read from its table instead of being retyped. Approximate + matches need sorted cells, as in Excel; `XLOOKUP`'s binary searches + (`search_mode` 2 / -2) are refused, and `None` typed as a key is refused + (the formula would write it as 0). A key is compared with the cells + searched in one unit (`mo.XLOOKUP(amount_usd, thresholds_eur, fees)` is + refused), and the cells a lookup may return — a list of rows, a block's + column or row, an `if_not_found` — are in one unit, as `mo.CHOOSE`'s + choices are; a number or a line without a unit goes with any. Where the + cells searched or returned cannot form one range — lines on two sheets + or with a line between them, or a row split by a totals column — the + lookup is written as its computed number and `check_compat` lists the + cell. +- **Statistics.** `mo.MEDIAN`, `mo.STDEV_S`, `mo.STDEV_P`, `mo.VAR_S`, + `mo.VAR_P`, `mo.PERCENTILE_INC`, `mo.PERCENTILE_EXC`, `mo.QUARTILE_INC`, + `mo.QUARTILE_EXC`, `mo.LARGE`, `mo.SMALL`, `mo.RANK_EQ` and `mo.RANK_AVG` + — Excel's dot is an underscore in the name, and the file stores + `_xlfn.PERCENTILE.INC` — with the older `mo.STDEV`, `mo.STDEVP`, + `mo.VAR`, `mo.VARP`, `mo.PERCENTILE`, `mo.QUARTILE` and `mo.RANK` that + older workbooks carry. Over a row or one field down the records of + a table: the median of a comps multiple is `=MEDIAN(Comps!C3:C9)`, its + first quartile `=_xlfn.PERCENTILE.INC(Comps!C3:C9, 0.25)`, and each + record's rank among its peers one call per record against the same + range (`=_xlfn.RANK.EQ(C3, C3:C9)`). Several series are combined period + by period, as `a + b` combines them — the rule every aggregate follows: + `mo.MEDIAN(plan, actual)` is `=MEDIAN(C5, C9)` in every column, + with scalars broadcast, series of other lengths refused and a line with + tracks taken track by track; a list of two or more yearly lines (one + field down a records table, or sibling lines) is one sample per year — + the peers' median revenue is `=MEDIAN(Peers!C3:C9)`, then + `=MEDIAN(Peers!D3:D9)`, …, their first quartile and each record's rank + among them one call per year over the same column of cells + (`=_xlfn.RANK.EQ(D3, D3:D9)`). One series alone is reduced over its + periods (`=MEDIAN(C5:N5)`) and scalars only stay one number; a list of + one yearly line is refused, as no sample. Inside a `loop=True` container + a call over two or more values runs one period at a time. Blanks, text + and booleans in a range are skipped; a `k` or a quart out of range is + `#NUM!`; `LARGE` takes a fractional `k` up and `SMALL` down; the values + a call reads are in one unit. +- **A rank, a conditional total or a lookup per record reads its range + once.** Every call over the same lines — `mo.RANK_EQ(rec.multiple, + peers)` for each record of a table, or a `SUMIF`, `COUNTIFS`, `MATCH` or + `XLOOKUP` per record — shares one range, read once and checked before + each call: a line given another value (replaced, or its list changed in + place), another unit or another place is read again. Its cells are + sorted once for `RANK`, `LARGE`, `PERCENTILE` and `MEDIAN`, and indexed + by value once a second equality in `SUMIF`, `COUNTIFS`, `MATCH`, + `XLOOKUP` or their kin looks among them; the range's address is worked + out once per sheet. Each formula still lists all its peers, so the build + of a very large table keeps a cost per record and peer. +- **Hours.** `default_grain='hour', default_start='2026-01-05T13'` (or a + `datetime` on the hour, or a date — its midnight): the wall clock of one + fixed-offset zone, declared once on the window (`default_zone='+01:00'`; + UTC when omitted; a named zone such as `'Europe/Paris'` is refused with the + offset to write, since a zone that moves its clocks would skip and repeat + hours). An hour is `[13:00, 14:00)`: `mo.time.end` is where it stops, + `mo.time.days` is 1/24 (`mo.time.months` is refused), `regrain` weights a + mean by it. An hour nests in its day and everything above: `.at('day')`, + `.at('month')`, `.at('week')`; `timeline={'totals': ['day']}` + draws a day total after each day's last hour, `05 Jan 2026 | 22:00 23:00 | + Total 05 Jan 2026`, a live `=SUM` over its hours; a time row's day total + reads its hours' cells too — the first hour's start, the last hour's + stop, the `SUM` of their days. The header is a closed + form off the start cell (`start + i/24`, never a chain that would drift + over thousands of columns), captions are the hour's start as a clock + (`dd mmm yyyy hh:mm`), the days row shows its fraction. Keys are + `YYYY-MM-DDTHH`; the finance spelling is `05 Jan 2026 13:00`. +- **A window wider than a sheet is refused.** A sheet holds 16,384 columns + and a period is one (a total another): `default_periods` past 16,383 is + refused where it is declared, and the layout refuses a sheet whose label + columns, periods and totals together pass the last column + (`SheetWidthError`), naming the count, what the grain fits (hours about 22 + months) and the two ways out — a coarser grain, or the data kept as rows. +- **An instant keeps its clock in a formula.** A `datetime` off midnight + inlined into a formula is written `(DATE(2026, 1, 5)+TIME(13, 0, 0))` — in + brackets, so `end - start` subtracts the whole instant; it was cut to its + day. +- **An ISO year in its own book.** A window of ISO years + (`default_grain='iso_year'`), or a weekly book viewed as ISO years, + writes its header off the start cell: the ISO year of the start cell + is `YEAR(start + 3)`, each year starts on `DATE(y,1,4) - + WEEKDAY(DATE(y,1,4),3)` and ends the day before the next one does; an + anchor chains `start = previous end + 1` and ends each year off its own + start, so a first year opened on a Monday inside it is short and right. +- **ISO weeks, and the ISO year that totals them.** `default_grain='week'`, + `default_start='2026-W05'` (or a date that is a Monday): weeks run Monday to + Sunday and are numbered in their ISO year — `2026-W01` starts on 29 December + 2025, and 2026 has 53 weeks; `mo.time` reads each week (7 days; `months` is + refused, as a day's is). A week lies inside one ISO year and in no month, + quarter or calendar year: `.at('iso_year')` rolls it up, and + `timeline={'totals': ['iso_year']}` draws a total after each ISO year's last + week; a day sheet may total by `'week'` (with `'iso_year'` above it). A + weekly header is a live chain off the start cell (`end = start + 6`). +- **A day sheet totals by month.** `ExcelView(timeline={'totals': ['month']})` + (beside `'quarter'`, `'half'`, `'year'`) on a day window draws a month total + after each month's last day — `30 31 | Total Jan | 01 02 03 | Total Feb` — + under a month tier (`Jan`; the year on the tier above when it is declared), + the days grouped under their month in the outline, each total a live `=SUM` + over its days (`=` the last day for a stock). Code that reads the totals + plan (`modeleon.compile.excel.totals.TotalsPlan`) finds a sheet's own periods + keyed `('period', i)`, where they were `('month', i)` whatever the grain; + `month_col` stays as an alias of `native_col`. +- **The half-year.** A fifth grain between quarter and year: a window of + its own (`default_grain='half', default_start='2026-H1'`, or a date — + a half opened on the 1st of any month is a short first period, as a + quarter's is), a totals column (`ExcelView(timeline={'totals': ['half']})` + draws `Total H1` after June and `Total H2` after December, under the + year's tier and over the quarters'), a target of `.at('half')` on a line + or on a whole model (`model.at('half')`), with its `mo.time` (181 / 184 + days, 6 months) and its header formulas (`EOMONTH(start, 5)`). Keys are + `YYYY-Hn`; the finance spelling is `H1 2026`, the compact one `H1-26`. +- **The calendar's answers for a window.** + `modeleon.core.time.period_bounds('2026-01', 'month', 3)` returns each + period's first and last day as two lists (1 to 31 January, 1 to 28 + February, 1 to 31 March; for an hour, where it starts and stops), and + `year_back_index('2026-01', 'month', 15)` the index of the period a year + earlier in the same window (`None` for the first twelve months, then `0, + 1, 2`). `modeleon.core.calendar` holds a ruler for each grain (hour, day, + week, month, quarter, half, year, iso_year): `GREGORIAN.ruler('quarter')` + finds the period holding a day (`start_of`) and where the next one starts + (`next_start`), writes and reads a period (`label(date(2026, 4, 1), + 'finance')` is `'Q2 2026'`, `parse('2026-Q2')` gives back 1 April) and + names its grain (`word` is `'Quarterly'`, `letter` `'Q'`). A chart's time + axis or a growth on last year in your own code can take its dates from + these instead of doing calendar arithmetic of its own. + +### Changed + +- **`mo.SUM` and `mo.AVERAGE` over two or more operands work period by + period.** `mo.SUM(product_a, product_b)` is the two lines' total in every + period — a subtotal, written `=SUM(C5, C9)` in each column — and + `mo.AVERAGE(plan, actual)` their mean, with scalars broadcast as `a + b` + broadcasts them, series of other lengths refused and an operand with + tracks taken track by track; inside a `loop=True` container they are + allowed. Before, two series were reduced to one number across both rows + (`=SUM(B2:F2, B3:F3)`). Over one operand `mo.SUM(row)` stays the total of + its periods (`=SUM(B2:F2)`), and over scalars only one number (`=SUM(B2, + B3)`). The total of several rows over all their periods is now + `mo.SUM(a) + mo.SUM(b)`. Every aggregate follows the rule `mo.MIN` / + `mo.MAX` follow since 0.2.2: several series are combined period by + period, one series alone is reduced over its periods, scalars only stay + one number. The operands of a call are added left to right, as `+` adds + them (Python's `sum` compensates the rounding), so `mo.SUM(a, b, c)` is + `a + b + c` to the last digit — over scalars only too, where a value can + move in its last digit (`mo.SUM(0.1, 0.2, -0.3)`). Lines on other time + windows and lines along keys are read as `a + b` reads them, as in every + function (the next entry). A `None` typed into `mo.SUM`, + `mo.AVERAGE`, `mo.MIN`, `mo.MAX` or `mo.PRODUCT` (or into a list typed + in) is the 0 the formula holds, as the sheet reads `=MIN(C2, 0)` and as + the counts read it (`mo.AVERAGE`, `mo.MIN` and `mo.MAX` skipped it); + text among the values is refused with a message naming + the function and the value. A per-period `SUM` or `AVERAGE` re-grains + as `a + b` does — its quarter is the call over its operands' quarters, + written live in a totals column (`=SUM(E5, E9)`) — and a per-period + `mo.PRODUCT` as `a * b` does. +- **Every function reads a line on another time window by date, and lines + along keys key by key — as `a + b` reads them, in Python and in the + workbook.** A located series on another window (a sub-container's own + `default_start`, a line's `start=`) is aligned onto the combined window: + each line continued by its own `extend=` rule — `mo.zero()` for a flow, + `mo.hold()` for a rate — the result on that window, two grains refused. + In 0.2.2 `mo.AND`, `mo.OR`, `mo.CHOOSE`, `mo.CONCAT`, `mo.DATE`, + `mo.DAYS360`, `mo.YEARFRAC` and `mo.DAYS` paired such lines by position — + January beside a tranche's March, the line's last value standing in for + the months it lacks — `mo.IF` paired them by position and raised + `IndexError` past the shorter branch (`mo.IF(rev > 2, rev, fee)` gave + `[10, 20, 3, 4]` for a fee from March; with the fee's + `extend=mo.zero()` it is now `[0, 0, 3, 4]`), `mo.MIN` and + `mo.MAX` refused lines of other lengths (`Cannot broadcast lists of + different lengths`), and `mo.MOD` and the comparisons paired two windows + of one length by position and refused others; now each reads the line + on its date, as do the functions added since (rounding, powers, loan + schedules, `mo.IFERROR` and its kin, the lookups' keys and positions, + the statistics, the counts), a `mo.recurrence` template's lines, which + 0.2.2 paired by position, and a loop container, whose rows read such a line by date in every period (a + one-period line was broadcast into every period of the loop, a line of + another length refused). A model that relied on the positional reading + meets a refusal naming the line and the rule to declare: `'fee' does + not cover the combined window (…). Say what continues it: + extend=mo.zero() (a flow) or extend=mo.hold() (a rate)`. An unnamed + formula runs on the window of the lines it reads — `rev * 1` on the + model's, `fee > 15` on the tranche's — and meets other windows by date; + it carries no `extend=` rule, so where it does not cover the combined + window it is refused, naming the line to make of it — and it is placed + the same whatever the order the model was built in: a line attached to + the model after a formula was built over it, or a window moved after, + places the formula as a fresh one is placed. The commonest case + is a condition on a line of another window, `mo.IF(fee > 15, rev, 0)`, + which 0.2.2 read by position: its migration is a line with a rule on + that container, `m.t.big = mo.Variable(m.t.fee > 15, extend=mo.zero())`, + then `mo.IF(m.t.big, rev, 0)`; a condition on the model's own lines, + `mo.IF(rev > 2, fee, 0)`, needs nothing. A one-value line on a window + of several periods is the constant it always was; a line along keys + runs on no window; a plain line beside a tracked one is read on the + tracked line's window (`mo.SUM(rev, once, trk)` reads `once` on its one + date — 0.2.2 refused the call, and `trk + once` read it in every + period) and a tracked line on its own window only; windows are ordered + by date, never as text (`'2026-9'` comes before `'2026-10'`: `rev + fee` + dropped September). A line of dates or text with `extend=mo.zero()` is + refused where the zero would stand in, pointing to `mo.hold()` (0.2.2 + let the zero stand in without a word). A line of another length with no + window at all is refused, as `a + b` refuses it; `mo.IF`, `mo.AND`, + `mo.OR`, `mo.CONCAT`, the dates and a `mo.recurrence` template, which + paired it by position, now name it (`'pair' cannot be paired with the + call's other series period by period`); a list typed in keeps its positional reading, + its last value standing in for the periods it lacks as the cell writes + it (in `mo.IF` too, which raised `IndexError` there), and `mo.IF` with + one condition takes the chosen branch whole, as before. + Lines along the same keys in another order are combined key by key in + every function, the result keeping their keys, and a line along keys + against a period series is refused, as `a + b` refuses it — the + functions zipped them cell by cell. What a workbook written now does + differently from 0.2.2: where Python aligns two windows — `+`, `-`, `*`, + `/`, `%`, `**`, the comparisons, every function and a loop container's + rows — a formula reading a line on a sub-container's own window inside + the sheet's takes the line's cell of the same date and, for a period + the line has no column for, its `extend=` rule, written as `0` for a + flow (`=SUM(C5, 0)`) or as its first or last cell for a rate held; + 0.2.2 wrote the cell in the same position — January read the line's + first cell, March's — and, past the line's last column, its first cell + again. Where Python reads one line alone — `fee * 2` over a line on + another window, set on the model's sheet — the workbook reads it by + position, as Python does and as 0.2.2 did. A formula whose operands met + by date on a window of its own, set on a sheet of another window + (`m.x.y = mo.IFERROR(m.b.bz + m.c.cz, 0)` with `B` from November on a + sheet from January), is refused by the export as the bare sum was: its + January column would hold November. A line along keys in another order + is read at its key's cell (`=B6 * D8`, Bear × Bear), where 0.2.2 wrote + the cell in the same position (Bear × Bull) beside Python's key-by-key + value; a function's result over one keyed line keeps its keys and is + laid out under a caption row of them (`mo.ROUND(growth, 1)`); where the + line above it runs along periods or other keys that caption row is new + and the rows below move down one — 0.2.2 dropped the keys and wrote the + values under the period captions (or refused the line when the keys and + the periods differed in number). A window + reaching beyond the sheet's is refused at export, as before. +- **Functions that add, compare or pick take their values in one unit, as + `+` does.** `mo.SUM(revenue, hours)`, with `revenue` in USD and `hours` + in hours, is refused, naming both lines (`mo.SUM takes its values in one + unit; 'revenue' and 'hours' are in USD and hours`), with the advice `+` + gives; so are `mo.AVERAGE`, `mo.MIN`, `mo.MAX`, the two branches of + `mo.IF`, the choices of `mo.CHOOSE` and a comparison of two lines + (`revenue > hours`: `Cannot compare values with incompatible units: USD + and hours`). Before, these ran in silence and the result had no unit, so + the next formula could not catch the mix: `mo.SUM(revenue, hours) + cost` + passed, in USD. A number or a line without a unit goes with any + (`mo.MAX(revenue, 0)` is in USD). A condition, an index or a case is not + one of the values: `mo.CHOOSE(scenario, revenue, cost)` with `scenario` + labelled `#` is in USD, where it had no unit. Inside a loop container, + over a row read before its line, the check is made when the loop closes. + The functions this release adds take their values the same way (above). +- **A series where a function took one value is one call per period.** + `mo.ROUND`'s digits, `mo.EDATE` / `mo.EOMONTH`'s months and every + argument of `mo.PMT`, `mo.FV` and `mo.PV` may be a line — each period + computes the call over its own values, written in its column as the + spreadsheet writes it (`=ROUND(C5, C9)`, `=EDATE(C5, C9)`, `=PMT(C9 / + 100, 12, 1000, 0, 0)`), a scalar standing in every period and lines read + as above; these raised a bare `TypeError`. `mo.NPV`'s rate and `mo.IRR` + / `mo.XIRR`'s guess stay one value for the whole row: a series there is + refused by name, as `mo.XNPV`'s rate already was. +- **A function says how it is written.** Which names the Excel writer + passes through as `=NAME(...)`, which arguments it writes as ranges, which + names the file stores with the `_xlfn.` prefix, and which functions a loop + container runs one period at a time, is one registry + (`modeleon.functions.registry`) each function fills from its own module; + the renderer's and the loop container's hand-kept name sets are gone. A + function name the registry does not know is still written as given, with + a warning that points at it. +- **A condition's `source_code` follows its formula.** For `mo.IF`, + `mo.AND`, `mo.OR`, `mo.NOT`, `mo.ISBLANK` and `mo.CHOOSE`, and the new + `mo.IFERROR`, `mo.ISERROR`, `mo.ISNUMBER`, `mo.XOR`, `mo.IFS` and + `mo.SWITCH`, `source_code` is the line's `.formula`, rendered when it is + read, so it follows names given to the lines later and a formula set by + hand (`flag.formula = '...'`): `mo.IF(a > 0, b, 0.0)` built before `a` + and `b` were named reads `IF((a > 0), b, 0.0)` once they are. Before, it + kept the text of the call, `IF((v1088264b0 > 0), v107cf6b70, 0.0)`. A + text written with `flag.source_code = '...'` still wins; nothing in the + workbook changes. +- **A recurrence template whose power or function outgrows a cell gives + `#NUM!`.** `mo.recurrence(1e200, "{prev} ** 2", periods=3)` is + `[1e+200, '#NUM!', '#NUM!']`, carried on as a division by zero is; 0.2.2 + raised `OverflowError`. The functions new in this release do the same in + a template (`"EXP({prev} * 1000)"`); an overflow by `*` or `+` still + gives `inf`, as in Python. +- **A recurrence is in its start's unit.** `mo.recurrence(opening, + "{prev} * (1 + {g})", g=growth)` from an `opening` in USD is in USD, and + so is `mo.recurrence_sum(opening, additions)` over USD additions: a + revenue line built so is labelled `Revenue (USD)` in the workbook and + the notebook repr, and adding an hours line to it is refused. Before, + both had no unit unless `unit=` was given: the label read `Revenue`, and + the sum with hours passed, in hours. A declared `unit=` still decides. +- **Grains nest by a relation, not by their order.** The calendar says + which grain lies inside which: in `modeleon.core.calendar`, + `GREGORIAN.nests('day', 'week')` is true and `GREGORIAN.nests('week', + 'month')` false, and `GREGORIAN.verify()` checks each such claim period + by period. `modeleon.core.time.GRAINS` lists `hour, day, week, month, + quarter, half, year, iso_year` in that order (ranks 0 to 7). `.at()` on a line or on a whole + model, and arithmetic between two lines, are refused across grains that + do not nest (a weekly line's `.at('month')`), with the reason and the way + out: a grain both share (days). A totals column a sheet cannot draw (a + month column over weeks, weeks beside months over days) is left out with + a `TotalsWarning` that names the sheet and says why (`'month' totals are + not drawn: a week does not lie inside one month (weeks straddle months). + A week sheet totals by iso_year.`). In 0.2.2 `GRAINS` was `day, month, + quarter, year`, and a line rolled up to any grain after its own. +- **Large models build faster.** Arithmetic between lines does less work + for each value, a condition (`mo.IF` and its kin) renders its text only + when it is read, and under a blend of tracks (`mo.blend`) what a line + reads is read once for the model, not again for every line built on it: + in our runs a model of 2,000 lines of arithmetic or conditions builds in + about three quarters of the time 0.2.2 took. Values, formulas and + workbooks are the same; writing the workbook takes about as long as + before. + +### Fixed + +- **A tracked row reads by position.** `closing[-1]` of a plan/actual row + raised "Index -1 on a scalar Variable", and `[0]` copied the whole row as + numbers. Both now read that period of every track, written in the book as + one live cell per track; a period not entered yet reads 0, as a blank cell + does in a formula. An index past `[0]` on a one-cell row says to read the + row by its name instead of pointing at `var.value`. +- **A view goes where it is read.** An unnamed `mo.ExcelView(...)` hung + under a name that says "view" — `model.excel_view = …`, `model.view = …`, + `mo.Model(..., view=…)` — became a container nobody reads: the workbook + kept its look and no total column appeared, or the view's own settings + were refused as a series (`'totals' has 2 value(s) in a 24-period + window`). `to_excel` now warns (`UnreadViewWarning`, on the author's + line) with the slot to use, spelled by the author's own name for the + model — `model.default_excel_view = mo.ExcelView(...)`. A view the slot + points at is read, also through `extend`, through the copy a projection + or a second parent makes, through an ancestor's slot, and under + `mo.house_view(...)`. A view given a name of its own + (`model.board_view = mo.ExcelView('Board pack', ...)`, kept to be applied + with `mo.applied_view`) and a view kept under a name without "view" in it + draw no warning, and no view's settings are read as a series over the + model's periods. +- **A horizon given by assignment counts.** `m.default_periods = 36` marks + the model last given a horizon, as the constructor does, so a line + reading `mo.time` outside the model's `with` block finds the window; a + refused change (the lines already read the window) marks nothing. +- **A container cannot hold itself.** `model.m = model` adopted the model + as its own component and recursed until Python gave up; so did assigning + an ancestor. Refused with the alias to write (`with model as m:`). A read + of a component that was never assigned (`'Model' has no component + 'inputs'`) now says how to assign it, and that a row read before its own + line belongs in a loop container. Both messages spell the container by + the author's own name for it (`model`, `income`, `model.pnl`), never by + its declared or display name, so the line they suggest runs. +- **A large table is written in linear time.** A container with thousands + of children copied its whole list of children for every child it read, + and checked a new child's name against the list one by one: a records + table of 10,000 records took two minutes to write and now takes about + three seconds (3,000 records: 11 s to under a second), and its notebook + repr and `.code` follow. +- **A blank typed into `LEN`, `UPPER`, `LOWER` or `CONCAT` is written as + `""`.** It was written as `0`, which the workbook reads as the text "0" + (`=LEN(0)` is 1 where Python gave 0). +- **`mo.DAYS360` reads a method that varies by period.** A line given as + `method` was read as one TRUE — any list is — so every period counted + the European way, where the workbook reads each period's own cell + (`=DAYS360(B4, B11, B10)`); now each period takes its own method. +- **A list of lines given as one argument says what it means, or is + refused.** In 0.2.2 `mo.SUM([a, b])` held a Variable as its value and the + cell its text (`Variable(...)`), as did `mo.AVERAGE`, `mo.MIN` and + `mo.MAX` over scalar lines; over a list holding a series `mo.MIN` / + `mo.MAX` raised "The truth value of a list Variable is ambiguous". A + list of scalar lines is its lines one by one in `mo.SUM`, `mo.AVERAGE`, + `mo.MIN`, `mo.MAX` and `mo.PRODUCT` (`mo.SUM([p, q])` is `mo.SUM(p, q)`, + `=SUM(B1, B2)`) and the range they form in `mo.COUNT` / `mo.COUNTA` + (`=COUNT(B1:B2)`). A list holding a series reads two ways and is + refused, both spelled with the lines' own names: per period, `mo.SUM(a, + b)`; the total over all their periods, `mo.SUM(a) + mo.SUM(b)` (each + function its own: `mo.MIN(mo.MIN(a), mo.MIN(b))`, …). The statistics + read a list of series lines as one sample per period (above). +- **A grain or totals kind the engine does not know is refused where it is + given.** `m.default_grain = 'fortnight'` is refused on assignment, as the + constructor refuses it (`Unknown default_grain 'fortnight'; supported: + hour, day, week, ...`); it used to be accepted and fail later, in + `to_excel` or at the first `mo.time` read, on `Unknown time grain + 'fortnight'`, which does not name `default_grain`. + `ExcelView(timeline={'totals': ['fortnight']})` is refused by name + (`ExcelView timeline totals may name day, week, month, quarter, half, + year, iso_year; got 'fortnight'`); it used to be accepted and draw no + column, without a word. `.at('fortnight')` on a month line names the + grains it rolls up to (`a month rolls up to quarter, half, year`), where + it raised a bare `KeyError: 'fortnight'`. A day window's start must be + `YYYY-MM-DD`: `default_start='2026-W01'` is refused, where it used to + start the window on Monday 29 December 2025, the first day of that ISO + week. +- **Months beside years point to the explicit step.** Two units that + differ only in a month word against a year word (`months` / `yrs`, + `USD/month` / `USD/yr`) are refused with the step written out both ways + — `(x / 12).set_unit('yrs')`, `(x * 12).set_unit('months')` — since a + unit is a label that never rescales: dividing by 12 leaves months in + months. A refusal with no hint no longer opens its advice with + "Otherwise". +- **A remainder keeps its unit.** `usd % 12` is in USD, and `usd % hours` + and `mo.MOD(usd, hours)` are refused, inside a loop container too; + before, `%` dropped the unit, and both let any two units through, + leaving none. `100 // usd` is in `1/USD`, as `100 / usd` is. +- **A whole power raises the unit, inside a loop container too.** + `usd ** 2.0` is in `USD^2`, as `usd ** 2` is, and `(usd * usd) ** 0.5` + is in `USD`; a power that comes out fractional (`usd ** 0.5`), or an + exponent that is a line, still leaves none. A unit string reads `^n` as + a power, so a line declared in `'USD^2'` adds to `usd * usd`. Inside a + loop container a row takes the unit it takes outside one: `balance ** 2` + is in `USD^2`, and `balance + balance ** 2` is refused. Before, a power + written as a float and a root had no unit, so `usd ** 2.0 + usd` + passed; a loop row built with `**` had none either, so + `balance + balance ** 2` passed, stamped USD; and `'USD^2'` was a word + of its own, so adding that line to `usd * usd` was refused as + `USD^2 and USD^2`. +- **A `pyformula` result has no unit unless declared.** The engine cannot + see inside the function, and a ratio of two USD lines was stamped USD, + then added to amounts without a word. Declare the result's unit with + the new `pyformula(unit='USD')`, or with `set_unit`. +- **A `pyformula` named like a spreadsheet function is not run as that + function in a loop container.** A `@mo.pyformula` function named `max` + (or given `name='MAX'`), applied inside a `loop=True` container to a row + of the loop, is refused when the loop closes, as one of any other name is + (`MAX(...) cannot be computed again when the loop closes`): build that + row after the loop's last line. Before, the loop ran `mo.MAX` in its + place without a word — a function doubling a row of 100, 200 and 310 + gave the row back unchanged. +- **A unit with several slashes reads as written.** `'USD/seat/month'` + divides at each slash, so a seat count times a price in + `'USD/seat/month'` is in `USD/month` and adds to a fee in `USD/month`. + A bracketed group is a unit of its own (`'USD/(seat/month)'` is + `USD * month/seat`), and a word may carry a note in brackets: hours in + `'hours (billable)'` times a rate in `'USD/hours (billable)/month'` are + in `USD/month`. A chain prints as typed; a string whose brackets do not + balance keeps the old reading and prints as typed. Before, only the + first `/` was read — `USD` per one symbol `seat/month` — so `seat` + never cancelled (the product was in `seat * USD/seat/month`, and adding + the fee was refused), and a note lost its closing bracket + (`hours (billable`). +- **A unit-keyed number format finds its line by unit equality.** + `ExcelView(formats={'USD/seat/month': '0.00'})` formats the price a + model works out, `revenue / seats / months`, whose unit prints + `USD/(seat * month)`, and `formats={'seat * month': '#,##0'}` formats + `months * seats`, whose unit prints `month * seat`. Before, a format was + found by the unit's printed text, so both lines were left in General. A + pure number matches by its label, so a `'%'` format is never applied + to a `bps` line. +- **A loop container's rows get the units their formulas give.** When a + loop closes, a row's unit waits for the rows it reads (`+` takes the + other side's unit, a product or a power waits for its factors), a lag's + fill gives the row it lags its unit, and a row nothing settles (a flag, + an index, a discount factor) is a pure number. With + `opening = mo.lag(closing, fill=start)` and `start`, `cap` and `inflow` + in USD, `opening / cap` is a pure number and a closing of + `opening + inflow + opening ** 2 * k` (`k` in `1/USD`) is in USD; 0.2.2 + read a row that had no unit yet as a pure number, so the ratio printed + `1/USD` and that closing was refused (`USD and 1/USD`). A unit that + never settles is refused: `x = mo.lag(x, fill=1.0) * cap` (0.2.2 printed + `USD^4`) meets `its unit does not settle`, with the two ways out, a + factor with no unit or a declared unit on the row (`.set_unit(...)`). A + mix is reported under the row that writes it on every build, where 0.2.2 + sometimes named a row reading it through a lag; and a formula built in + the loop that fails when the loop closes passes its error to the rows + built on it, where 0.2.2 handed them the numbers it held while the loop + was open. +- **Inside a loop container, `line[0]` and a recurrence keep their unit.** + A row written before the loop closes as `x[k]`, `x[a:b]`, + `mo.recurrence`, `mo.recurrence_sum` or `mo.cumsum` kept no unit, so + `balance[0] + fee_usd` passed in silence and `balance / balance[0]` + printed EUR. They now carry the line's (the start's) unit, as built + outside a loop, and a mix is refused when the loop closes. +- **A loop refused when it closes hands out no numbers.** A loop + container whose unit check refused it at close (an `EUR` fee added to a + `USD` amount) still handed its rows out with the numbers computed from + the mix and wrote them into the book. Every row, and + every line built on one, now refuses its value with the refusal, as a + loop refused before it computes does; the way out is to build the + container again. +- **A count in `bps` or `pp` beside a share in `%` is refused.** `bps` and + `pp` are labels that never rescale (50 bps is written 0.005, 2 pp is + 0.02), so a `bps` or `pp` line holding a count (50) is refused beside a + `%` line holding a fraction (0.05) where the two are added or taken as + one: `+` and `-`; every function that combines them (`mo.SUM`, + `mo.AVERAGE`, `mo.MIN`, `mo.MAX`, `mo.IF`, `mo.IFS`, `mo.SWITCH`, + `mo.CHOOSE`, `mo.IFERROR`, `mo.MOD`, `mo.CEILING`, `mo.FLOOR`, the + conditional totals, the statistics, the cells a lookup returns); a + recurrence template (`mo.recurrence(share, '{prev} + {x}', x=counts)`); + a lag's fill (`mo.lag`, `.shift`); and a loop container, when the loop + closes, under the row. `rate + step`, with `rate` holding 0.05 in `%` + and `step` 50 in `bps`, is refused with the way to write it: `A bps line + holds the fraction (50 bps is 0.005), or the count is converted first: + step / 10_000.` A share written as points (30 % + 2 pp) and a fraction + stepped by a fraction (0.05 % + 0.005 bps) add as before. Before, + nothing made the check, and `rate + step` came out 50.05 without a word. +- **`.shift`, `NPV`, `XNPV`, `PMT`, `FV` and `PV` carry units, and a lag's + fill is checked.** A lag is in its line's unit, and so must be its fill: + `mo.lag(balance, fill=opening)`, the balance in USD and the opening in + EUR, is refused (`mo.lag takes its values in one unit; 'balance' and + 'opening' are in USD and EUR`), and inside a loop container the check is + made when the loop closes. A number or a line without a unit fills any + line, and a line without a unit takes any fill. `balance.shift(1)` is in + USD, as `mo.lag(balance)` is. `mo.NPV` and `mo.XNPV` are in their flows' + unit, and `mo.PMT`, `mo.FV` and `mo.PV` take their amounts — a present or + future value, a payment — in one unit, the result's: `mo.PMT(rate, 24, + loan, balloon)` with the loan in USD and the balloon in EUR is refused; + the rate and the number of periods are not amounts. Before, a fill or a + balloon in any unit passed, and `balance.shift(1)` and the five functions + had no unit: `mo.NPV(rate, flows) + fee` passed with the fee in EUR. +- **`.shift(n, fill_value=)` keeps the fill as a live cell.** A + line given as the fill was frozen into the formula as its number + (`=1000.0` in the first cell), so the workbook did not follow an edit of + the opening cell. The fill is now a reference to its cell (`=inp!B2`), + as `mo.lag(x, fill=)` keeps it, and `.shift` follows `mo.lag`'s + unit rule (above): a fill in another unit is refused, in a loop + container when it closes; a list fill or a fill assigned further down + is refused, as `mo.lag` refuses them. The 0.2.0 note that `.shift` + inlines its fill no longer holds. +- **`NPV`, `IRR`, `XNPV` and `XIRR` over a list of lines read each + line.** `mo.NPV(rate, [y1, y2])` with lines in two units ran in silence + and stamped no unit, so `usd + NPV(rate, [eur, eur])` passed; the + book wrote the computed number with a false "reduces a literal list" + warning, and inside a loop container a list holding a total of a row + read before its line kept the number it held while the loop was open. + The list is now + the lines' cells: they must share one unit (two are refused), the result + carries it (`NPV`, `XNPV`), the lines are the call's dependencies, + and the book writes `=NPV(B1, B3:B4)` / `=IRR(B2:B4, 0.1)`. A list + of numbers typed in stays a value, as before. +- **A recurrence template is read for units.** `mo.recurrence(eur, + '{prev} + {x}', x=usd_line)` had no unit and ran in silence; the + template is now read as any formula is: a mix is refused with the + operators' own message (`recurrence("{prev} + {x}"): Cannot + add/subtract values with incompatible units: EUR and USD`), a start + without a unit takes the template's, a template that multiplies the + line by a dimension every period is refused as a unit that does not + settle, and inside a loop container the check is made when the loop + closes, under the row. A declared `unit=` decides what the line + prints, never what its template may add. +- **`CHOOSE` over a keyed line.** A choice that runs along keys — `mo.Variable({'Bear': + 0.9, 'Base': 1.0, 'Bull': 1.1})`, or a line on one non-time `indexed_by` axis — is, on + its own, one choice per key in key order: `mo.CHOOSE(pick, growth)` is `mo.CHOOSE(pick, + growth['Bear'], growth['Base'], growth['Bull'])`, written `=CHOOSE(B1, B3, C3, D3)` over + the line's cells (one such formula per period when the index is a series; keys that + are not text spread by position, as `[]` and the workbook read them). Before, the line + was read as one period series: its keys were lost, a 3-key line in a 4-period window + was refused for its length, and the workbook held `=CHOOSE(B1, B3)` in every period. Refused + with an error that names the fix instead of a silent number: a keyed line beside other choices + (the numbering would depend on its key count), a computed keyed line with no cells of + its own (name it first), a tracked or keyed index, a series index over tracked + choices, a line on two axes. A scalar index over tracked choices still switches track + by track, as before. +- **A line along keys against a period series is refused.** `base * growth`, with + `base` a period series — a list, a located list (`start=`/`extend=`), a tracked line, + `mo.time.days` — and `growth` a keyed list (or a line on one finite non-time axis) of + the same length, was zipped cell by cell — January with Bear, February with Base — and + the result carried the keys, so the numbers looked right and were wrong; unequal + lengths died with the generic length error. Every arithmetic operator and comparison + now raises, naming both lines, the pairing a zip would make, and the fixes that run + (pick one key, `growth['Bear']`; give the series the same keys or lay it along the + same axis; or switch with `mo.CHOOSE(pick, growth)`). A keyed list against a line on + an axis of other labels is refused too. Scalars, one-value bare lists, a raw Python + list (`keyed * other.value`), keyed × keyed, axis × axis and series × series + broadcast as before. The layout, the operators and `CHOOSE` use the same test of + whether a line runs along keys. +- **A wrapping Variable keeps the keys and axes of the line it wraps.** + `mo.Variable(growth * 2, display_name='Twice')` ran along nothing — the keys of the + product were dropped, as its unit once was — so the named line lost its caption row + and its place in `CHOOSE`. A declared `keys=` or `indexed_by=` on the wrapper still wins. +- **A per-period field inside an `orient='columns'` block is refused, not dropped.** The + matrix holds one cell per field per instance; a list field (a period series, a keyed + line, a line on an axis) was skipped without a word — its row vanished and every + aggregate reading it was written as a dead number. The layout now records the refusal + and `to_excel` raises it before writing a book, naming the + line, the instance and the block, with the forms that hold such a line (the sheet, or + `orient='records'` for a period series); `check_compat` reports it as a `layout` + issue; the notebook repr and every other reader keep their addresses. A one-value + bare list field is a cell, as a one-value list is a single value everywhere else. +- **A name whose line was overwritten by a plain value can be removed.** After + `inputs.rate = 0.05` written over the line `rate`, `inputs.remove('rate')`, + `del inputs.rate`, and moving `inputs` to another container raised + `AttributeError: 'float' object has no attribute '_qualified_id'` and could leave the + container half detached. The plain value is removed or carried along like a line. + ## [0.2.2] — 2026-10-19 ### Added diff --git a/README.md b/README.md index dfe38b9..bee0f6a 100644 --- a/README.md +++ b/README.md @@ -166,13 +166,13 @@ Balance B5: =Inputs!B6 - Inputs!C4 - B4 C5: =B5 - Inputs!D4 - C4 ``` A cycle that does not pass through `mo.lag` is refused by name — it -would need a fixed point, not an order. So is `SUM` (or `MAX`/`MIN` over -one operand) of a loop row inside the loop: those read a whole series. A -per-period floor or cap is `mo.MAX(x, 0)` / `mo.MIN(sweep, balance)` — -`MIN`/`MAX` over two or more operands work period by period, inside the -loop too, and write `=MIN(C2, C3)` in every column. A class is a loop -container for every instance with `class Tranche(mo.MultiVariableClass, -loop=True)`. +would need a fixed point, not an order. So is `SUM`, `MAX` or `MIN` over +one operand of a loop row inside the loop: it reads a whole series. Over +two or more operands they work period by period, inside the loop too: a +per-period floor or cap is `mo.MAX(x, 0)` / `mo.MIN(sweep, balance)`, a +subtotal `mo.SUM(a, b)`, written `=MIN(C2, C3)` / `=SUM(C2, C3)` in every +column. A class is a loop container for every instance with `class +Tranche(mo.MultiVariableClass, loop=True)`. --- @@ -185,12 +185,32 @@ Standard library functions emit live formulas, not values: ```python mo.IRR(cf) # =IRR(B2:F2, 0.1) mo.NPV(0.1, cf) # =NPV(0.1, B2:F2) +mo.SUM(revenue) # =SUM(B2:F2) — the total over the periods +mo.SUM(product_a, product_b) # =SUM(B2, B3) in every period — a subtotal of two lines mo.IF(rev > 0, rev * 0.25, 0) # =IF(B1>0, B1*0.25, 0) mo.CHOOSE(scenario, base, bull) # =CHOOSE(B1, B4, B5) in every period — one scenario cell switches the row mo.EOMONTH(start, 1) # =EOMONTH(B1, 1) +mo.IFERROR(cost / units, 0) # =IFERROR(B7 / B8, 0) in every period — a zero divisor shows 0 +mo.SUMIFS(amount, region, "North") # =SUMIFS(C6:E6, C5:E5, "North") +mo.XLOOKUP(year, years, inflation) # =_xlfn.XLOOKUP(B4, C2:E2, C3:E3) +mo.PERCENTILE_INC(multiples, 0.25) # =_xlfn.PERCENTILE.INC(Comps!C3:C9, 0.25) — a field down a table +mo.MEDIAN([p.revenue for p in peers]) # each year: =MEDIAN(Peers!C3:C9), =MEDIAN(Peers!D3:D9), … ``` -Also: `IRR`, `NPV`, `XIRR`, `XNPV`, `PMT`, `FV`, `PV` · `SUM`, `MAX`, `MIN`, `AVERAGE` · `IF`, `AND`, `OR`, `NOT`, `CHOOSE` · `ABS`, `ROUND`, `INT`, `MOD` · `YEAR`, `MONTH`, `DAY`, `DAYS`, `YEARFRAC`, `EDATE`, `EOMONTH`, `DATE`, `TODAY` · `CONCAT`, `UPPER`, `LOWER`, `LEN`. All emit live formulas. +| Family | Functions | +|---|---| +| Aggregates | `SUM`, `MAX`, `MIN`, `AVERAGE`, `PRODUCT` | +| Conditional aggregates | `SUMIF`, `SUMIFS`, `COUNTIF`, `COUNTIFS`, `AVERAGEIF`, `AVERAGEIFS`, `MAXIFS`, `MINIFS`, `COUNT`, `COUNTA` | +| Logic and errors | `IF`, `IFS`, `SWITCH`, `CHOOSE`, `AND`, `OR`, `NOT`, `XOR`, `IFERROR`, `ISERROR`, `ISNUMBER`, `ISBLANK` | +| Lookup | `XLOOKUP`, `INDEX`, `MATCH`, `VLOOKUP`, `HLOOKUP` | +| Math and rounding | `ABS`, `ROUND`, `ROUNDUP`, `ROUNDDOWN`, `TRUNC`, `INT`, `MOD`, `CEILING`, `FLOOR`, `POWER`, `SQRT`, `EXP`, `LN`, `LOG`, `LOG10`, `SIGN` | +| Financial | `IRR`, `NPV`, `XIRR`, `XNPV`, `PMT`, `FV`, `PV`, `NPER`, `RATE`, `IPMT`, `PPMT`, `CUMIPMT`, `CUMPRINC` | +| Depreciation | `SLN`, `DB`, `DDB`, `SYD` | +| Dates | `DATE`, `YEAR`, `MONTH`, `DAY`, `DAYS`, `DAYS360`, `YEARFRAC`, `DATEDIF`, `EDATE`, `EOMONTH`, `WEEKDAY`, `WEEKNUM`, `NETWORKDAYS`, `WORKDAY`, `TODAY` | +| Text | `LEN`, `UPPER`, `LOWER`, `CONCAT`, `LEFT`, `RIGHT`, `MID`, `TRIM`, `FIND`, `SEARCH`, `SUBSTITUTE`, `VALUE`, `TEXT`, `REPT`, `TEXTJOIN` | +| Statistics | `MEDIAN`, `STDEV_S`, `STDEV_P`, `VAR_S`, `VAR_P`, `PERCENTILE_INC`, `PERCENTILE_EXC`, `QUARTILE_INC`, `QUARTILE_EXC`, `LARGE`, `SMALL`, `RANK_EQ`, `RANK_AVG`, and the older `STDEV`, `STDEVP`, `VAR`, `VARP`, `PERCENTILE`, `QUARTILE`, `RANK` | + +All emit live formulas. A function of one value per period, given a row, is one call per period (`=ROUNDUP(C9, 1)`, `=ROUNDUP(D9, 1)`, …) and runs inside a loop container too. A row given where a function takes one value is read per period as well: `mo.ROUND(x, digits)` is `=ROUND(C9, C10)` and `mo.PMT(rate / 100, 12, 1000)` is `=PMT(C9 / 100, 12, 1000, 0, 0)` in each column. Every function reads rows on other time windows by date and rows along keys key by key, as `a + b` reads them (the workbook reads the cell of the same date, and the row's `extend=` rule where it has none); a loop container's rows read them so too. A formula over a row runs on that row's window and, carrying no `extend=` rule, must cover the combined window: `mo.IF(fee > 15, rev, 0)` with `fee` on a tranche's window asks for a line with a rule, `m.t.big = mo.Variable(m.t.fee > 15, extend=mo.zero())`. Every aggregate follows one rule: several series given to it — `SUM`, `AVERAGE`, `MIN`, `MAX`, `PRODUCT`, `COUNT`, `COUNTA`, `MEDIAN`, `STDEV_S`, `STDEV_P`, `VAR_S` or `VAR_P` — or listed as a percentile's or a rank's range, are combined period by period, as `a + b` is (`mo.SUM(a, b)` is `=SUM(C5, C9)` and `mo.MEDIAN(plan, actual)` is `=MEDIAN(C5, C9)` in each column); one row alone is reduced over its periods (`mo.SUM(a)` is `=SUM(C5:N5)`); scalars only stay one number. The total of several rows over all their periods is `mo.SUM(a) + mo.SUM(b)`. A function Excel added after 2007 is stored with the `_xlfn.` prefix the file format expects, and a name Excel spells with a dot is written with an underscore (`mo.PERCENTILE_INC` is `PERCENTILE.INC`). --- diff --git a/pyproject.toml b/pyproject.toml index d224a83..1c96a2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "modeleon" -version = "0.2.2" +version = "0.2.3" description = "Financial Model Engineering — Python code compiles to live Excel formulas" requires-python = ">=3.12" license = "Apache-2.0" diff --git a/src/modeleon/__init__.py b/src/modeleon/__init__.py index 8c6371a..3241892 100644 --- a/src/modeleon/__init__.py +++ b/src/modeleon/__init__.py @@ -44,14 +44,15 @@ Package layout: core/ - Variable, MultiVariable, QPath, Unit, Expr AST - functions/ - IF, SUM, MAX, MIN, AVERAGE, ABS, ROUND, INT, MOD, - recurrence, IRR, NPV, XIRR, PMT, FV, PV, … + functions/ - IF, IFERROR, SUM, SUMIFS, XLOOKUP, INDEX, MATCH, ROUND, + MEDIAN, PERCENTILE_INC, recurrence, IRR, NPV, PMT, RATE, + SLN, … (the registry says how each is written) compile/ - AST renderers + layout + .xlsx writer (Excel, JSON) display/ - notebook HTML rendering plugins.py - extension point registry """ -__version__ = "0.2.2" +__version__ = "0.2.3" # Plugin system — extensions register via entry points from modeleon.plugins import load_plugins as _load_plugins @@ -81,7 +82,7 @@ from modeleon.compile import check_compat, to_json # Excel presentation view — declared-once, cascading layout/styling -from modeleon.compile.excel.view import ExcelView, house_view +from modeleon.compile.excel.view import ExcelView, applied_view, house_view # Pre-built DSL helpers from modeleon.functions import ( @@ -89,18 +90,30 @@ pyformula, # Aggregates SUM, MAX, MIN, AVERAGE, + # Conditional aggregates and counts + SUMIF, SUMIFS, COUNTIF, COUNTIFS, AVERAGEIF, AVERAGEIFS, MAXIFS, MINIFS, COUNT, COUNTA, # Math - ABS, ROUND, INT, MOD, + ABS, ROUND, INT, MOD, ROUNDUP, ROUNDDOWN, TRUNC, CEILING, FLOOR, POWER, SQRT, EXP, LN, + LOG, LOG10, SIGN, PRODUCT, # Conditional / logical - IF, AND, OR, NOT, CHOOSE, ISBLANK, + IF, AND, OR, NOT, CHOOSE, ISBLANK, IFERROR, ISERROR, ISNUMBER, XOR, IFS, SWITCH, + # Lookup / reference + MATCH, INDEX, XLOOKUP, VLOOKUP, HLOOKUP, # Dates YEAR, MONTH, DAY, DATE, EDATE, EOMONTH, DAYS360, YEARFRAC, DAYS, TODAY, + DATEDIF, WEEKDAY, WEEKNUM, NETWORKDAYS, WORKDAY, # Text - LEN, UPPER, LOWER, CONCAT, + LEN, UPPER, LOWER, CONCAT, LEFT, RIGHT, MID, TRIM, FIND, SEARCH, SUBSTITUTE, VALUE, TEXT, + REPT, TEXTJOIN, # Recurrence / cohort / shift cumsum, lag, recurrence, recurrence_sum, cohort_retention, schedule, # Financial IRR, NPV, XIRR, XNPV, PMT, FV, PV, + NPER, IPMT, PPMT, RATE, CUMIPMT, CUMPRINC, SLN, DB, DDB, SYD, + # Statistics + MEDIAN, STDEV_S, STDEV_P, VAR_S, VAR_P, PERCENTILE_INC, PERCENTILE_EXC, QUARTILE_INC, + QUARTILE_EXC, LARGE, SMALL, RANK_EQ, RANK_AVG, + STDEV, STDEVP, VAR, VARP, PERCENTILE, QUARTILE, RANK, ) # The process-wide default view — EVERY value the engine uses, made explicit so @@ -140,6 +153,7 @@ "Model", "Unit", "ExcelView", + "applied_view", "default_excel_view", "house_view", # Exceptions / warnings @@ -159,14 +173,51 @@ "NOT", "ISBLANK", "CHOOSE", + "IFERROR", + "ISERROR", + "ISNUMBER", + "XOR", + "IFS", + "SWITCH", "SUM", "MAX", "MIN", "AVERAGE", + # Conditional aggregates and counts + "SUMIF", + "SUMIFS", + "COUNTIF", + "COUNTIFS", + "AVERAGEIF", + "AVERAGEIFS", + "MAXIFS", + "MINIFS", + "COUNT", + "COUNTA", + # Math "ABS", "ROUND", "INT", "MOD", + "ROUNDUP", + "ROUNDDOWN", + "TRUNC", + "CEILING", + "FLOOR", + "POWER", + "SQRT", + "EXP", + "LN", + "LOG", + "LOG10", + "SIGN", + "PRODUCT", + # Lookup / reference + "MATCH", + "INDEX", + "XLOOKUP", + "VLOOKUP", + "HLOOKUP", # Dates "YEAR", "MONTH", @@ -178,11 +229,27 @@ "YEARFRAC", "DAYS", "TODAY", + "DATEDIF", + "WEEKDAY", + "WEEKNUM", + "NETWORKDAYS", + "WORKDAY", # Text "LEN", "UPPER", "LOWER", "CONCAT", + "LEFT", + "RIGHT", + "MID", + "TRIM", + "FIND", + "SEARCH", + "SUBSTITUTE", + "VALUE", + "TEXT", + "REPT", + "TEXTJOIN", # Domain helpers (no Excel equivalent — stay lowercase) "cumsum", "cohort_retention", @@ -203,4 +270,35 @@ "PMT", "FV", "PV", + "NPER", + "IPMT", + "PPMT", + "RATE", + "CUMIPMT", + "CUMPRINC", + "SLN", + "DB", + "DDB", + "SYD", + # Statistics + "MEDIAN", + "STDEV_S", + "STDEV_P", + "VAR_S", + "VAR_P", + "PERCENTILE_INC", + "PERCENTILE_EXC", + "QUARTILE_INC", + "QUARTILE_EXC", + "LARGE", + "SMALL", + "RANK_EQ", + "RANK_AVG", + "STDEV", + "STDEVP", + "VAR", + "VARP", + "PERCENTILE", + "QUARTILE", + "RANK", ] diff --git a/src/modeleon/compile/compat.py b/src/modeleon/compile/compat.py index b9d4b87..5e10aa4 100644 --- a/src/modeleon/compile/compat.py +++ b/src/modeleon/compile/compat.py @@ -36,7 +36,7 @@ class CompatIssue: func: str # the call; for a cell, the call written as a number, # else the sub-expression itself - kind: str # "render", "compute", "value" or "inlined" + kind: str # "render", "compute", "value", "inlined" or "layout" declared: frozenset # the FuncCall's declared backend set (a cell: empty) target: str # the target that was checked qpath: Optional[str] = None # a cell: the line's qualified path @@ -45,6 +45,8 @@ class CompatIssue: literal: Optional[str] = None # an inlined cell: the number written in its place def __str__(self) -> str: + if self.kind == "layout": + return self.func if self.kind == "inlined": return ( f"{self.qpath}[{self.period}]: {self.inlined} is written as " @@ -222,6 +224,10 @@ def _excel_cell_issues(root: Any, target: str) -> List[CompatIssue]: time=book_time_for(engine, roots, sheet_names, addresses, var_to_sheet), ) issues: List[CompatIssue] = [] + issues.extend( + CompatIssue(func=msg, kind="layout", declared=frozenset(), target=target) + for msg in engine.refusals + ) for sheet_mv, sheet_name in zip(roots, sheet_names): for _name, comp in sheet_mv.walk(): if not isinstance(comp, Variable): diff --git a/src/modeleon/compile/excel/header.py b/src/modeleon/compile/excel/header.py index 9d0aa71..ba8b3ad 100644 --- a/src/modeleon/compile/excel/header.py +++ b/src/modeleon/compile/excel/header.py @@ -28,15 +28,70 @@ from __future__ import annotations from dataclasses import dataclass, field -from datetime import date +from datetime import date, datetime from typing import Any, Callable, Dict, List, Optional +from ...core.calendar import GREGORIAN from ...core.multi_variable import MultiVariableBase -from ...core.time import time_field_of +from ...core.time import _has_stub, time_field_of from ...core.variable import Variable -#: Months in one period of each grain. -STEP = {'month': 1, 'quarter': 3, 'year': 12} +#: Months in one period of each grain the header steps by months — the +#: calendar's rulers of whole months (a day is stepped by days). +STEP = {r.name: r.months for r in GREGORIAN.rulers if r.months} + + +def _months_per(grain: str) -> int: + """Months in one period of ``grain`` — the stride every header + formula steps by. A grain the book cannot step is refused by name, + never spelled as if it were a month.""" + months = GREGORIAN.ruler(grain).months if GREGORIAN.knows(grain) else None + if not months: + spelled = [r.name for r in GREGORIAN.rulers + if r.months or r.stride_days or r.stride_hours or r.week_years] + raise ValueError( + f"A workbook spells periods of {', '.join(spelled[:-1])} or " + f"{spelled[-1]}; got grain {grain!r}." + ) + return months + + +def _stride_days(grain: str) -> Optional[int]: + """Days in one period of a fixed-length ruler (a week: 7), else + ``None``. The day has its own branch in ``spell_inline`` / + ``spell_anchor``.""" + if not GREGORIAN.knows(grain): + return None + d = GREGORIAN.ruler(grain).stride_days + return d if d and d > 1 else None + + +def _week_years(grain: str) -> bool: + """Whether ``grain`` is the ISO week-numbering year: its bounds are + spelled off the ISO year number of a day (``YEAR(day + 3)`` — the + Thursday of a week lies in the week's ISO year).""" + return GREGORIAN.knows(grain) and GREGORIAN.ruler(grain).week_years + + +def iso_year_start(year: str) -> str: + """The Monday ISO year ``year`` starts on: the Monday of the week + holding 4 January.""" + return f"(DATE({year},1,4)-WEEKDAY(DATE({year},1,4),3))" + + +def iso_year_end_of(day: str) -> str: + """The Sunday that ends the ISO year ``day`` lies in.""" + return f"DATE(YEAR({day}+3)+1,1,4)-WEEKDAY(DATE(YEAR({day}+3)+1,1,4),3)-1" + + +def _stride_hours(grain: str) -> Optional[int]: + """Hours in one period of a clock ruler (an hour: 1), else ``None``: + its time is the start cell plus i/24, one closed form across the + row — never a running chain, which would drift over thousands of + columns.""" + if not GREGORIAN.knows(grain): + return None + return GREGORIAN.ruler(grain).stride_hours #: No ``default_header`` anywhere up the tree. UNSET = object() @@ -161,7 +216,19 @@ def window_key(window: Any) -> Optional[tuple]: def date_literal(d: date) -> str: - return f"DATE({d.year},{d.month},{d.day})" + """A day as Excel writes it, ``DATE(2026,1,5)``; an instant off + midnight carries its clock, ``DATE(2026,1,5)+TIME(13,0,0)``.""" + day = f"DATE({d.year},{d.month},{d.day})" + if isinstance(d, datetime) and (d.hour or d.minute or d.second): + return f"{day}+TIME({d.hour},{d.minute},{d.second})" + return day + + +def _atom(ref: str) -> str: + """``ref`` fit to stand as one operand: a cell or a call as it is, an + instant's ``DATE(…)+TIME(…)`` in brackets — else ``x - start`` reads + ``(x - DATE(…)) + TIME(…)``.""" + return f"({ref})" if "+" in ref else ref def _offset(start0: str, k: int, first_day: Optional[date]) -> str: @@ -188,13 +255,38 @@ def spell_inline(field_: str, i: int, grain: str, start0: str, first_day: Optional[date] = None) -> str: """``mo.time.`` of period ``i``, off the start cell alone. ``first_day`` set: the window's first period is short — it starts at - ``start0`` and ends with its quarter / year.""" + ``start0`` and ends with its quarter / year. A clock ruler's period + ends where the next starts (``start0 + (i+1)/24``).""" + h = _stride_hours(grain) + if h is not None: + def at(j: int) -> str: + return f"({start0}+{h * j}/24)" if j else _atom(start0) + return {'start': at(i), 'end': at(i + 1), 'days': f"{h}/24", 'months': '0', + 'index': str(i + 1)}[field_] + if _week_years(grain): + # An ISO year: its number is the start cell's (YEAR(start + 3)), + # each later one a year on, every bound a closed form off the start + # cell. A first period opened on a Monday inside its ISO year runs + # from that Monday to the year's last Sunday. + def year(j: int) -> str: + return f"YEAR({start0}+3)+{j}" if j else f"YEAR({start0}+3)" + start = start0 if i == 0 else iso_year_start(year(i)) + end = f"({iso_year_start(year(i + 1))}-1)" + return {'start': start, 'end': end, 'days': f"({end}-{start}+1)", 'months': '0', + 'index': str(i + 1)}[field_] if grain == 'day': day = f"({start0}+{i})" if i else start0 return {'start': day, 'end': day, 'days': '1', 'months': '0', 'index': str(i + 1)}[field_] - k = STEP[grain] - if first_day is not None and grain in ('quarter', 'year'): + d = _stride_days(grain) + if d is not None: + # A week: every period d days from the start cell, one closed + # form across the row (never a running chain). + start = f"({start0}+{d * i})" if i else start0 + return {'start': start, 'end': f"({start0}+{d * i + d - 1})", 'days': str(d), + 'months': '0', 'index': str(i + 1)}[field_] + k = _months_per(grain) + if first_day is not None and _has_stub(grain): off = _offset(start0, k, first_day) def end_at(j: int) -> str: @@ -239,10 +331,43 @@ def spell_anchor(field_: str, i: int, grain: str, start0: str, with its quarter / year whatever month it starts in, and counts its months from its dates — again one formula across the row, which stays right when the start cell is edited in the workbook.""" - if first_day is not None and grain in ('quarter', 'year'): + if _week_years(grain) and field_ != 'index': + # An ISO year, chained like a hand-built timing sheet: start = the + # previous end + 1, end = the last Sunday of the ISO year the start + # lies in, days = end - start + 1 — right for a first period opened + # on a Monday inside its year too. + own_start, own_end = cell('start', i), cell('end', i) + if field_ == 'start': + if i == 0: + opening = cell('end', -1) + return f"{opening}+1" if opening else start0 + prev_end = cell('end', i - 1) + return f"{prev_end}+1" if prev_end else spell_inline('start', i, grain, start0) + if field_ == 'end': + return iso_year_end_of(own_start) if own_start else spell_inline('end', i, grain, start0) + if field_ == 'days': + if own_start and own_end: + return f"{own_end}-{own_start}+1" + return spell_inline('days', i, grain, start0) + return '0' # months: an ISO year holds no calendar months + if first_day is not None and _has_stub(grain) and not _week_years(grain): stub = _spell_anchor_stub(field_, i, grain, start0, cell, first_day) if stub is not None: return stub + h = _stride_hours(grain) + if h is not None and field_ != 'index': + # An hour: every start a closed form off the start cell (never the + # previous cell plus 1/24 — thousands of columns would drift), the + # end where it stops, the days its fraction. + if field_ == 'start': + return start0 if i == 0 else f"{start0}+{h * i}/24" + if field_ == 'end': + own = cell('start', i) + return f"{own}+{h}/24" if own else spell_inline('end', i, grain, start0) + if field_ == 'days': + own_start, own_end = cell('start', i), cell('end', i) + return f"{own_end}-{own_start}" if own_start and own_end else f"{h}/24" + return '0' # months: an hour holds none (mo.time.months is refused) if grain == 'day': if field_ == 'start': if i == 0: @@ -256,7 +381,34 @@ def spell_anchor(field_: str, i: int, grain: str, start0: str, return '1' if field_ == 'months': return '0' - k = STEP.get(grain, 1) + d = _stride_days(grain) + if d is not None and field_ != 'index': + # A week, chained like a hand-built timing sheet: start = the + # previous end + 1, end = start + 6, days = end - start + 1. + if field_ == 'start': + if i == 0: + opening = cell('end', -1) + return f"{opening}+1" if opening else start0 + prev_end = cell('end', i - 1) + if prev_end: + return f"{prev_end}+1" + prev_start = cell('start', i - 1) + return f"{prev_start}+{d}" if prev_start else spell_inline('start', i, grain, start0) + if field_ == 'end': + own = cell('start', i) + return f"{own}+{d - 1}" if own else spell_inline('end', i, grain, start0) + if field_ == 'days': + own_start, own_end = cell('start', i), cell('end', i) + return f"{own_end}-{own_start}+1" if own_start and own_end else str(d) + return '0' # months: a week holds none (mo.time.months is refused) + if field_ == 'index': + # The period's number chains off the previous one whatever the + # grain — it needs no month stride (a day window has none). + prev = cell('index', i - 1) + if prev: + return f"{prev}+1" + return str(i + 1) + k = _months_per(grain) if field_ == 'start': if i == 0: opening = cell('end', -1) @@ -285,13 +437,8 @@ def spell_anchor(field_: str, i: int, grain: str, start0: str, if own_start: return f"EOMONTH({own_start},{k - 1})-{own_start}+1" return spell_inline('days', i, grain, start0) - if field_ == 'months': - return _months_cell(i, start0, cell) or str(k) - # index - prev = cell('index', i - 1) - if prev: - return f"{prev}+1" - return str(i + 1) + # months + return _months_cell(i, start0, cell) or str(k) def _spell_anchor_stub(field_: str, i: int, grain: str, start0: str, @@ -299,7 +446,7 @@ def _spell_anchor_stub(field_: str, i: int, grain: str, start0: str, first_day: date) -> Optional[str]: """``spell_anchor`` for a window whose first period is short; ``None`` where the whole-period chain is already right (start and index).""" - k = STEP[grain] + k = _months_per(grain) own_start, own_end = cell('start', i), cell('end', i) if field_ == 'end': if own_start: diff --git a/src/modeleon/compile/excel/layout.py b/src/modeleon/compile/excel/layout.py index fc4fded..f9ef0ce 100644 --- a/src/modeleon/compile/excel/layout.py +++ b/src/modeleon/compile/excel/layout.py @@ -19,6 +19,7 @@ from .addresses import VariableAddresses from ...core.multi_variable import MultiVariableBase +from ...core.shape import line_keys from ...core.variable import Variable @@ -181,7 +182,7 @@ def _number_outline( out[var.id] = f"{prefix}{count[0]}" return for name in mv._component_order: - comp = mv._components[name] + comp = mv._component(name) if isinstance(comp, Variable): if _takes_number(comp, rows): count[0] += 1 @@ -301,26 +302,6 @@ def _is_periodic(var: Variable) -> bool: return bool(getattr(var, '_indexed_by', ()) or ()) -def line_keys(var: Variable) -> Optional[List[str]]: - """The keys a line runs ALONG instead of periods — a keyed list - (``mo.Variable({'FY2023': 400, 'FY2024': 450})``) or a line on one - finite axis that is not time (``indexed_by=[regions]``) — one per - cell, in cell order; ``None`` for a series, a constant and a - tracked line. Such a line stands under a caption row of these keys - (``key_captions_by_sheet``) and counts for no period column. Public - so a renderer captions the line by the keys the layout placed it - under.""" - keys = getattr(var, '_keys', None) - if keys and isinstance(getattr(var, '_value', None), list): - return [str(k) for k in keys] - axes: Sequence[Any] = getattr(var, '_indexed_by', None) or () - if len(axes) == 1 and getattr(axes[0], '_axis_kind', None) != 'time': - labels = getattr(axes[0], '_value', None) - if isinstance(labels, list): - return [str(label) for label in labels] - return None - - def _sheet_has_periods(mv: MultiVariableBase, flatten: bool = False) -> bool: """Does this SHEET carry any per-period line? @@ -394,6 +375,48 @@ def _reject_rank2_emission(var: Variable) -> None: ) +def _board_list_field_refusal( + mv: MultiVariableBase, + iname: str, + inst: MultiVariableBase, + fname: str, + var: Variable, + cells: int, +) -> str: + """The refusal of a field of several cells in an ``orient='columns'`` block. + + The matrix holds ONE cell per field per instance. A list field — a + period series, a keyed line, a line on an axis — was skipped here + without a word: its row vanished from the sheet, and every aggregate + reading it was written as its computed number, a dead cell. Loud + instead, with the forms that do hold such a line. + """ + keys = line_keys(var) + label = (getattr(var, '_display_name', None) + or getattr(var, '_python_name', None) or fname) + instance = getattr(inst, '_display_name', None) or inst.python_name or iname + section = getattr(mv, '_display_name', None) or mv.python_name or mv.id + if keys is None: + along = "period" + fix = ( + f"Put '{label}' outside '{section}', where it gets a row of its " + f"own on the sheet, or lay '{section}' out with orient='records', " + f"where a per-period field gets a block of its own beneath the table." + ) + else: + along = "key" + fix = ( + f"Put '{label}' outside '{section}', where it gets a row of its " + f"own on the sheet, under a caption row of its keys." + ) + return ( + f"'{label}' in '{instance}' is a line of {cells} cell(s), one per " + f"{along}, and '{section}' is laid out with orient='columns' — one " + f"column per instance, one cell per field — which has no row for it. " + f"{fix}" + ) + + def layout_sheet_names(sheets: Sequence[MultiVariableBase]) -> List[str]: """The key the layout files each sheet under, in order: its :func:`sheet_name_for`, suffixed ``1``, ``2``, … when an earlier sheet @@ -518,6 +541,17 @@ def __init__( self._totals_override = ( tuple(totals_override) if totals_override is not None else None ) + if self._totals_override: + # A kind the engine has no bucket for is refused by name — a + # typo or a grain the engine does not know must not vanish into "no + # totals" without a word. + from ...core.time import GRAINS + unknown = [k for k in self._totals_override if k not in GRAINS] + if unknown: + raise ValueError( + f"A totals column is one of {', '.join(GRAINS)}; got " + f"{', '.join(repr(k) for k in unknown)}." + ) self.roots = roots self._flatten_nested_sheets = flatten_nested_sheets #: The workbook's header rows (``default_header``) — mirrors of the @@ -568,6 +602,10 @@ def __init__( #: one row the lines that share its keys share. A renderer hides #: it with its lines (:func:`hidden_key_caption_rows`). self.key_caption_rows: Dict[str, int] = {} + #: What the layout refused to place, one message each that says how to fix it — + #: a workbook is not written while any stands (the writer raises + #: the first); other readers keep their addresses. + self.refusals: List[str] = [] #: var id -> the row its unit stands in, for a caption that heads #: no row of the sheet (a records-table field, a scalar across a #: board) on a sheet with a unit column: the UNIT ROW its table @@ -645,6 +683,17 @@ def __init__( # in-process writer keeps the id() keying above; the deduped name # is unique per compute_addresses pass, so no clone collision). self.time_headers_by_sheet: Dict[str, tuple] = {} + #: sheet_name → the day its window's short first period starts + #: on (``DefaultWindow.first_day``), ``None`` when whole — the + #: one window fact ``time_headers_by_sheet`` does not carry. + self.time_first_day_by_sheet: Dict[str, Any] = {} + #: sheet_name → the window's fixed offset ('+01:00'), when it + #: declares one — kept for a caller; a workbook's dates carry no zone. + self.time_zone_by_sheet: Dict[str, Any] = {} + #: Why a declared totals kind is not drawn on a sheet (a month + #: column over weeks) — ``": "``, each also raised as + #: a ``TotalsWarning``; kept here for a caller that collects them. + self.totals_notes: List[str] = [] #: id(sheet_mv) -> the time window its period columns follow #: (``None`` when the model declares none) — recorded whether or #: not a header shows it, because the columns follow it anyway. @@ -854,22 +903,25 @@ def compute_addresses(self) -> Dict[str, VariableAddresses]: # Subtotal columns ride the time header — resolved # here where the view and window are in hand, # applied after addresses exist (_apply_totals). + # Only kinds the sheet's periods nest in, and that + # nest with each other, are drawn; the rest are reported. + from .totals import TotalsWarning, drawable_kinds + + _usable, _notes = drawable_kinds(tuple(_tot_kinds), window.grain) + for _note in _notes: + self.totals_notes.append(f"{sheet_name}: {_note}") + import warnings as _warnings + _warnings.warn(f"{sheet_name}: {_note}", TotalsWarning, stacklevel=2) self._totals_decl[sheet_name] = ( - tuple(_tot_kinds), + _usable, window.start, window.grain, style, view.time_totals_word or "Total", ) self._totals_ids[sheet_name] = id(sheet_mv) - from .totals import _coarser as _tot_coarser - - _usable = [ - k for k in _tot_kinds - if _tot_coarser(k, window.grain) - ] # The hierarchical header: one row per bucket kind - # ABOVE the month row — data moves down with it. + # ABOVE the periods' row — data moves down with it. _extra_header_rows = len(_usable) if style not in TIME_LABEL_STYLES: raise ValueError( @@ -882,6 +934,8 @@ def compute_addresses(self) -> Dict[str, VariableAddresses]: window.grain, style, ) + self.time_first_day_by_sheet[sheet_name] = getattr(window, 'first_day', None) + self.time_zone_by_sheet[sheet_name] = getattr(window, 'zone', None) # Track column groups: the sheet's STRIDE is the widest # slot any of its lines carries — resolved here, once, @@ -1022,6 +1076,7 @@ def compute_addresses(self) -> Dict[str, VariableAddresses]: self._record_period_starts() self._apply_totals() + self._check_sheet_width() self._record_freezes() self._record_numbers() return self.addresses @@ -1155,7 +1210,7 @@ def _iter_sheet_variables(self, sheet_mv: MultiVariableBase) -> Iterator[Variabl while stack: mv = stack.pop() for name in mv._component_order: - comp = mv._components[name] + comp = mv._component(name) if isinstance(comp, Variable): yield comp elif isinstance(comp, MultiVariableBase): @@ -1246,8 +1301,9 @@ def _layout_mv(self, mv: MultiVariableBase, row: int, col: int, sheet_name: str) # REPEATING children spread ACROSS — one column per sub-MV # (instance), one row per scalar field. The matrix form for # instances: metrics × instances instead of N stacked - # blocks. List fields don't fit a scalar matrix and are - # skipped. + # blocks. A list field has no cell in a scalar matrix: the + # layout records its refusal (``refusals``) and skips it; the + # writer refuses the book. if _own_orient(mv) == 'columns': return self._layout_mv_columns(mv, row, col, sheet_name) # Records orientation: one ROW per child record, one COLUMN @@ -1266,7 +1322,7 @@ def _layout_mv(self, mv: MultiVariableBase, row: int, col: int, sheet_name: str) # share the caption row. prev_keyed: Optional[tuple] = None for name in mv._component_order: - comp = mv._components[name] + comp = mv._component(name) if isinstance(comp, MultiVariableBase): if _is_presentation(comp): @@ -1301,7 +1357,7 @@ def _layout_mv(self, mv: MultiVariableBase, row: int, col: int, sheet_name: str) # components sit under it. The label cell stays the # section's; the child gets value addresses only. for _hname in comp._component_order: - _hchild = comp._components[_hname] + _hchild = comp._component(_hname) if (isinstance(_hchild, Variable) and (_hchild._excel_props or {}).get('header_row')): _haddr = self._create_var_address( @@ -1654,11 +1710,26 @@ def _line_row(var: Variable, row: int) -> int: inst_id = inst.id or inst.python_name or iname self.section_header_rows[inst_id] = (row, first + j) field_order: list[str] = [] - for _, inst in instances: + for iname, inst in instances: for fname, f in inst._components.items(): - if (isinstance(f, Variable) - and not isinstance(getattr(f, '_value', None), list) - and fname not in field_order): + if not isinstance(f, Variable): + continue + value = getattr(f, '_value', None) + if isinstance(value, list) and ( + len(value) > 1 or line_keys(f) is not None + or getattr(f, '_grain', None) is not None + ): + # A field is ONE cell of the matrix; a line of + # several has no place in it. The layout records + # the refusal and skips the line — a workbook is + # refused by the writer, a compatibility check + # reports it, every other reader keeps its + # addresses. + self.refusals.append( + _board_list_field_refusal(mv, iname, inst, fname, f, len(value)) + ) + continue + if fname not in field_order: field_order.append(fname) row += 1 for fname in field_order: @@ -1756,6 +1827,32 @@ def _record_period_starts(self) -> None: if roles is not None: roles['period'] = start + def _check_sheet_width(self) -> None: + """The width law: a sheet holds 16,384 columns, and a period is + one (a total another). A window wider than its sheet — counting + the label columns before it and the totals between — is refused + with the count and the two ways out, never cut.""" + from ...core.time import SHEET_COLUMNS, SheetWidthError, width_refusal + for sheet, var_ids in self.sheet_map.items(): + last = 0 + plan = self.totals_by_sheet.get(sheet) + if plan is not None and plan.col_of: + last = max(plan.col_of.values()) + plan.stride - 1 + for v in var_ids: + addr = self.addresses.get(v) + if addr is not None and addr.values: + last = max(last, _col_number(addr.values[-1])) + if last <= SHEET_COLUMNS: + continue + header = self.time_headers_by_sheet.get(sheet) + grain = header[1] if header else 'period' + start = self.period_start_by_sheet.get(sheet) or 2 + lead = start - 1 + periods = plan.n if plan is not None else last - lead + raise SheetWidthError( + f"{sheet}: " + width_refusal(grain, periods, lead, last - lead) + ) + def _apply_totals(self) -> None: """Interleave subtotal columns into every declaring sheet. @@ -1766,7 +1863,7 @@ def _apply_totals(self) -> None: bucket, recorded on the plan for the writer and any other consumer. """ - from .totals import build_totals_plan + from .totals import NATIVE, build_totals_plan for sheet, decl in self._totals_decl.items(): kinds, start, grain, style, totals_word = decl @@ -1793,7 +1890,7 @@ def _apply_totals(self) -> None: row = _row_number(a.values[0]) slot = self._slot_by_id.get(v, 0) moved = [ - f"{self._col_to_letter(plan.cell(('month', i), slot))}{row}" + f"{self._col_to_letter(plan.cell((NATIVE, i), slot))}{row}" for i in range(len(a.values)) ] self.addresses[v] = VariableAddresses( @@ -1887,7 +1984,7 @@ def _layout_mv_transposed( """ col = start_col for name in mv._component_order: - comp = mv._components[name] + comp = mv._component(name) if isinstance(comp, Variable): self.addresses[comp.id] = self._create_var_address_down( comp, header_row, col diff --git a/src/modeleon/compile/excel/renderer.py b/src/modeleon/compile/excel/renderer.py index 9ba176f..5dc5ed0 100644 --- a/src/modeleon/compile/excel/renderer.py +++ b/src/modeleon/compile/excel/renderer.py @@ -20,8 +20,9 @@ import logging import re import warnings +from dataclasses import dataclass from datetime import date, datetime, timedelta -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Dict, Iterator, List, Optional from .addresses import VariableAddresses from ...core.errors import CrossScopeReferenceWarning @@ -44,6 +45,14 @@ VarRef, ) from ..renderer import RenderCtx +from ...functions.registry import ( + excel_name, + file_prefix, + is_passthrough, + is_range_taking, + per_period_arity, + range_positions, +) logger = logging.getLogger(__name__) @@ -85,6 +94,58 @@ def _fold_sum(parts: List[str]) -> Optional[str]: f"{prefix}{first}:{last}" if last else f"{prefix}{first}" for first, last in runs)) +@dataclass(frozen=True) +class _Frame: + """The value a formula is being written an element of, as Python + computed it: ``var`` (the cell's own line, or an unnamed intermediate + written inline) and the element's index less the period being written + (``offset``: a lag that writes the previous period moves both). + ``window`` is the window (a sheet window key) ``var``'s operands were + read onto by DATE — an operator or a function that is one call per + period aligning two windows — and None where they were read by + position (or by key).""" + + var: Any + offset: int + window: Optional[tuple] + + +def _aligned_key(var: Any) -> Optional[tuple]: + """The window a formula's operands were aligned onto by date — the + combined window an operator or a one-call-per-period function puts its + result on and marks (``_read_by_date``) — as a sheet window key; None + for a value its operands gave by position (a line on a window of its + own read otherwise: ``start=``, a re-grain, a copy).""" + if not getattr(var, '_read_by_date', False): + return None + if getattr(var, '_grain', None) is None or getattr(var, '_start', None) is None: + return None + value = getattr(var, '_value', None) + n = len(value) if isinstance(value, list) else getattr(value, 'time_length', None) + if not n: + return None + first_day = getattr(var, '_first_day', None) + key = (var._grain, var._start, n) + return key if first_day is None else (*key, first_day) + + +def _by_position(var: Any) -> bool: + """A value whose elements are read at the cell's own period: not + aligned by date, along no keys — writing it needs no frame.""" + return not getattr(var, '_read_by_date', False) and not getattr(var, '_keys', None) + + +def _periods_from(a: tuple, b: tuple) -> int: + """The periods from window ``a``'s first to window ``b``'s, both of one + grain: negative when ``b`` starts earlier.""" + from ...core.time import _count, _first_day, _parse + grain = a[0] + here, there = _parse(a[1], grain), _parse(b[1], grain) + if _first_day(here, grain) <= _first_day(there, grain): + return _count(here, there, grain) - 1 + return 1 - _count(there, here, grain) + + _PY_TO_EXCEL_OP: Dict[str, str] = { "+": "+", "-": "-", @@ -125,12 +186,14 @@ def _fold_sum(parts: List[str]) -> Optional[str]: # The translator renders those args as ranges (``B2:F2``) and leaves # scalar VarRefs / literals as normal. Covers classic aggregates (SUM, # MAX, MIN, …) and financial-series funcs (IRR, NPV, XIRR) where Excel -# expects a range for the cash-flow / dates argument. ``MIN`` / ``MAX`` -# over two or more operands with a series among them are the exception -# (``_per_period_call``): one call per period, its operands this -# period's cells. +# expects a range for the cash-flow / dates argument. A call over enough +# values with a series among them is the exception (``_per_period_call``): +# one call per period, its operands this period's cells — ``SUM(a, b)``, +# ``MIN`` / ``MAX`` over two or more operands, ``MEDIAN(a, b)``, a +# percentile over a list of yearly lines. def _contiguous_run(refs: list) -> bool: - """Do these cell refs sit side by side on one row? + """Do these cell refs sit side by side on one row, or one under + another in one column, in order? A range is only honest when they do. The interleave that breaks it is ordinary — a declared quarter total puts a column between March @@ -145,54 +208,70 @@ def _contiguous_run(refs: list) -> bool: except Exception: return True # unparseable: emit the plain range rows = {r for r, _c in cells} - if len(rows) != 1: + cols = {c for _r, c in cells} + if len(rows) == 1: + run = [c for _r, c in cells] + elif len(cols) == 1: + run = [r for r, _c in cells] + else: + return False + return all(b - a == 1 for a, b in zip(run, run[1:])) + + +# Which names are written as ranges, which pass through as ``=NAME(args)``, +# and what the file stores before a name (``_xlfn.DAYS``) is the function +# registry's to say (``modeleon.functions.registry``); each function +# registers itself from its own module. + + +def _per_period_call(call: FuncCall, + lists_series: Optional[Callable[[ListExpr], bool]] = None) -> bool: + """A call over enough values to be one call per period, a series among + them (the registry's ``per_period_arity``) — ``SUM(a, b)`` is a + subtotal of a period's cells, ``MIN`` / ``MAX`` over two or more + operands are a cap or a floor, ``MEDIAN(a, b)`` the median of a + period's two cells, ``PERCENTILE_INC([a, b, c], k)`` the percentile of + a period's cells of three lines: its values are written as this + period's cells, never as the lines' ranges. The values are the + arguments read as ranges, each line a list of lines holds counted; + ``lists_series`` answers whether a list of lines holds a series.""" + func = call.func.upper() + arity = per_period_arity(func) + if arity is None: return False - cols = [c for _r, c in cells] - return all(b - a == 1 for a, b in zip(cols, cols[1:])) + positions = range_positions(func) + values: List[Any] = [] + lists: List[ListExpr] = [] + count = 0 + for i, arg in enumerate(call.args): + if positions is not None and i not in positions: + continue + if isinstance(arg, ListExpr): + lists.append(arg) + count += len(arg.items) + else: + values.append(arg) + count += 1 + if count < arity: + return False + if any(isinstance(arg, Literal) and isinstance(arg.value, list) for arg in values): + return True + series = lists_series or _lists_series + return any(_is_series_ref(arg) for arg in values) or any(series(arg) for arg in lists) -_RANGE_TAKING_FUNCS = { - "SUM", "AVERAGE", "MIN", "MAX", "COUNT", - "IRR", "NPV", "XIRR", "XNPV", - # What ``SUM(a * b)`` over two rows is written as; its arrays are - # ranges only (a comma list would be read as more arrays). - "SUMPRODUCT", -} +def _lists_series(node: ListExpr) -> bool: + return any(_is_series_ref(item) for item in node.items) -def _per_period_call(call: FuncCall) -> bool: - """``MIN`` / ``MAX`` over two or more operands with a series among - them is a cap or a floor, one call per period, not a reduction: its - operands are written as this period's cells, never as ranges.""" - if call.func.upper() not in ("MIN", "MAX") or len(call.args) < 2: +def _is_series_ref(node: Any) -> bool: + """A reference to a line holding a series (a row, or a row per track).""" + if not isinstance(node, VarRef): return False - for arg in call.args: - if isinstance(arg, Literal) and isinstance(arg.value, list): - return True - if isinstance(arg, VarRef): - value = getattr(arg.var, "_value", None) - if isinstance(value, list) or hasattr(value, "roles"): - return True - return False + value = getattr(node.var, "_value", None) + return isinstance(value, list) or hasattr(value, "roles") -# Function names that pass through to Excel unchanged. AST FuncCall nodes -# use Excel casing directly (``IF``, ``ABS``, ``ROUND``, etc.), so the -# translator just emits them verbatim — no rename map needed. -_PASSTHROUGH_FUNCS = _RANGE_TAKING_FUNCS | { - "IF", "AND", "OR", "NOT", "CHOOSE", "ISBLANK", "ABS", "ROUND", "INT", - "MOD", - "EDATE", "EOMONTH", "YEAR", "MONTH", "DAY", "DATE", "DAYS360", "TODAY", - "YEARFRAC", "DAYS", - "LEN", "UPPER", "LOWER", "TEXT", # CONCAT renders as the `&` operator - "PMT", "FV", "PV", "RATE", -} - -#: Excel 2013+ functions are stored in the file with the ``_xlfn.`` prefix: -#: written bare, Excel shows #NAME? (it reads ``_xlfn.DAYS`` and displays -#: ``=DAYS(...)``). -_FILE_PREFIX = {"DAYS": "_xlfn."} - def _value_at_period(value: Any, period_idx: int) -> Any: """Pick the period-th element from a list-valued Variable, or return @@ -218,6 +297,12 @@ def _value_to_excel_literal(value: Any) -> str: if isinstance(value, bool): return "TRUE" if value else "FALSE" if isinstance(value, datetime): + if value.hour or value.minute or value.second: + # An instant keeps its clock: 13:00 is not the day's start. A + # sum, it stands in brackets: ``x - start`` must not read + # ``(x - DATE(…)) + TIME(…)``. + return (f"(DATE({value.year}, {value.month}, {value.day})" + f"+TIME({value.hour}, {value.minute}, {value.second}))") value = value.date() if isinstance(value, date): return f"DATE({value.year}, {value.month}, {value.day})" @@ -369,6 +454,13 @@ def __init__( #: before each walk. self.inlined: List[tuple[Expr, str]] = [] self.lossy: List[Expr] = [] + #: A list of lines many calls share (one range read once, see + #: ``functions.criteria``) is read once per render: whether it + #: holds a series, and the text of its cells per sheet and period — + #: keyed by the node, which each entry holds so its id stays its + #: own. Nothing a render reads moves while it lasts. + self._listed: Dict[int, tuple[ListExpr, bool]] = {} + self._ranges: Dict[tuple[Any, ...], tuple[ListExpr, Optional[str]]] = {} self.walker: Any = None # set by Walker(renderer) constructor def _note_inlined(self, node: Expr, literal: str, ctx: RenderCtx) -> str: @@ -458,7 +550,9 @@ def _render_time(self, host: Any, field: str, ctx: RenderCtx) -> str: anchor's own chain, this sheet's header row, the anchor's row, or the period spelled off its window's start — and only through a header or anchor that counts the same window as the formula.""" - from ...core.time import _first_day, _parse, resolve_default_window, spelled_first_day + from ...core.time import ( + GRAINS, _first_day, _parse, resolve_default_window, spelled_first_day, + ) from .header import date_literal, spell_anchor, spell_inline, window_key i = ctx.period_idx self_var = ctx.self_var @@ -467,8 +561,7 @@ def _render_time(self, host: Any, field: str, ctx: RenderCtx) -> str: window = resolve_default_window(self_var) if self_var is not None else None if window_key(window) is None: loc = getattr(host, 'time', None) - if loc is None or loc.start is None or loc.grain not in ('day', 'month', - 'quarter', 'year'): + if loc is None or loc.start is None or loc.grain not in GRAINS: return self._note_inlined(TimeRef(field), _value_to_excel_literal( _value_at_period(getattr(host, '_value', None), i)), ctx) start = loc.first_day or _first_day(_parse(loc.start, loc.grain), loc.grain) @@ -545,13 +638,18 @@ def render_opening(self, var: Any, sheet: Optional[str]) -> Optional[str | int]: window = resolve_default_window(var) if window is None or window.start is None: return None + from ...core.calendar import GREGORIAN + if GREGORIAN.knows(window.grain) and GREGORIAN.ruler(window.grain).sub_day: + # "Period 0" of a clock ruler stops where the first period starts. + return f"{self._start_cell(window, sheet)}" return f"{self._start_cell(window, sheet)}-1" def render_varref(self, node: VarRef, ctx: RenderCtx) -> str: target = self._varref_target(node, ctx) if not isinstance(target, Expr): return target - with self._chain_passes(node, target, ctx): + inline = node.var if target is node.var._expr else None + with self._chain_passes(node, target, ctx), self._entered(inline, ctx): return self.walker.render(target, ctx) def _varref_target(self, node: VarRef, ctx: RenderCtx) -> Any: @@ -572,6 +670,9 @@ def _varref_target(self, node: VarRef, ctx: RenderCtx) -> Any: if coord is not None: return coord if var_id in self.addresses: + dated = self._dated_cell(node, var_id, ctx) + if dated is not None: + return dated return self._resolve_var_addr(var_id, ctx.period_idx, ctx.current_sheet) # An address-less operand may still LIVE in a laid-out row: a # loop-built per-month intermediate is the same object as that @@ -902,27 +1003,44 @@ def _render_binop_terms(self, node: BinOp, ctx: RenderCtx) -> str: # outgrew Python's stack — the cell fell back to its number. level = next((ops for ops in _CHAIN_LEVELS if op in ops), ()) chain = [node] - operand: Any = node.left - while True: - target = operand - if isinstance(operand, VarRef) and operand is not ctx.own_chain: - target = self._varref_target(operand, ctx) - if not (isinstance(target, BinOp) and target.op in level - and self._folded_chain(target, ctx) is None): - break - chain.append(target) - operand = target.left - if target is operand: - rendered = self.walker.render(operand, ctx) - elif isinstance(target, Expr): - with self._chain_passes(operand, target, ctx): - rendered = self.walker.render(target, ctx) - else: - rendered = target - parts = [self._wrap_operand(operand, rendered, chain[-1].op, side="left")] - for step in reversed(chain): - parts.append(_PY_TO_EXCEL_OP.get(step.op, step.op)) - parts.append(self._render_operand(step.right, step.op, ctx, side="right")) + # Each step of the run down an unnamed intermediate is written in + # that intermediate's element (``_entered``), its left side read + # in it too: entered on the way down, left on the way back up. + entered: List[Any] = [] + try: + operand: Any = node.left + while True: + target = operand + if isinstance(operand, VarRef) and operand is not ctx.own_chain: + target = self._varref_target(operand, ctx) + if not (isinstance(target, BinOp) and target.op in level + and self._folded_chain(target, ctx) is None): + break + chain.append(target) + step_in = self._entered(operand.var if isinstance(operand, VarRef) + and target is operand.var._expr else None, ctx) + step_in.__enter__() + entered.append(step_in) + operand = target.left + if target is operand: + rendered = self.walker.render(operand, ctx) + elif isinstance(target, Expr): + last = operand.var if (isinstance(operand, VarRef) + and target is operand.var._expr) else None + with self._chain_passes(operand, target, ctx), self._entered(last, ctx): + rendered = self.walker.render(target, ctx) + else: + rendered = target + parts = [self._wrap_operand(operand, rendered, chain[-1].op, side="left")] + for k in range(len(chain) - 1, -1, -1): + while len(entered) > k: + entered.pop().__exit__(None, None, None) + step = chain[k] + parts.append(_PY_TO_EXCEL_OP.get(step.op, step.op)) + parts.append(self._render_operand(step.right, step.op, ctx, side="right")) + finally: + while entered: + entered.pop().__exit__(None, None, None) text = " ".join(parts) # A sum of a thousand rows outgrows Excel's formula: as a run # of cells it is ``SUM(B2:B1001)``, the way a hand-built book @@ -1091,7 +1209,7 @@ def render_funccall(self, node: FuncCall, ctx: RenderCtx) -> str: if node.render_backends is not None and 'excel' not in node.render_backends: return self._fallback_funccall(node, ctx) - if func in _RANGE_TAKING_FUNCS and not _per_period_call(node): + if is_range_taking(func) and not _per_period_call(node, self._lists_series): # Reached here, the call is the cell's whole formula (a call # inside a larger one is met through its Variable first). inlined = self._inline_unlaid_aggregate(node, None, ctx) @@ -1101,8 +1219,28 @@ def render_funccall(self, node: FuncCall, ctx: RenderCtx) -> str: ranges = self._product_ranges(node, ctx) if ranges is not None: return f"SUMPRODUCT({', '.join(ranges)})" - parts = [self._render_range_or_cell(a, ctx, func) for a in node.args] - return f"{func}({', '.join(parts)})" + # A range position is the row's range; any other position + # is this period's cell (a per-period criterion, a key). + positions = range_positions(func) + parts = [ + self._render_range_or_cell(a, ctx, func) + if positions is None or i in positions + else self.walker.render(a, ctx) + for i, a in enumerate(node.args) + ] + return f"{file_prefix(func)}{excel_name(func)}({', '.join(parts)})" + + if is_range_taking(func): + # One call per period: a list of lines at a range position is + # the run of their cells in this period's column. + positions = range_positions(func) + parts = [ + self._render_cell_list(a, ctx, func, ctx.period_idx) + if isinstance(a, ListExpr) and (positions is None or i in positions) + else self.walker.render(a, ctx) + for i, a in enumerate(node.args) + ] + return f"{file_prefix(func)}{excel_name(func)}({', '.join(parts)})" parts = [self.walker.render(a, ctx) for a in node.args] @@ -1117,13 +1255,13 @@ def render_funccall(self, node: FuncCall, ctx: RenderCtx) -> str: ] return " & ".join(pieces) - if func in _PASSTHROUGH_FUNCS: - return f"{_FILE_PREFIX.get(func, '')}{func}({', '.join(parts)})" + if is_passthrough(func): + return f"{file_prefix(func)}{excel_name(func)}({', '.join(parts)})" logger.warning( "Unknown function %r in formula — emitting as-is. If this is a " - "valid Excel function, add it to _PASSTHROUGH_FUNCS; otherwise " - "the cell will be invalid.", + "valid Excel function, register it in modeleon.functions.registry; " + "otherwise the cell will be invalid.", node.func, ) return f"{node.func}({', '.join(parts)})" @@ -1441,6 +1579,9 @@ def _expr_reads_own_cell(self, expr: Any, seen: set) -> bool: def _render_range_or_cell( self, arg: Expr, ctx: RenderCtx, func: Optional[str] = None, ) -> str: + if isinstance(arg, ListExpr): + block = self._render_block(arg, ctx) + return self._render_cell_list(arg, ctx, func) if block is None else block if isinstance(arg, VarRef): cid = self._coord_var_id(arg.var, ctx) if cid is None: @@ -1489,14 +1630,190 @@ def _render_range_or_cell( ) return self.walker.render(arg, ctx) + def _render_cell_list( + self, node: ListExpr, ctx: RenderCtx, func: Optional[str], + period: Optional[int] = None, + ) -> str: + """A list of scalar lines as one range argument: their cells, one + under another in a column (a field down the records of a table) + or side by side on a row, are the range ``Deals!C3:C5``. Cells + that form no run cannot be one range — a comma list is read as + further arguments by every function but the variadic ones — so + the cell carries its computed value. Given a ``period``, a line + holding a series stands for its cell of that period: a field of + yearly lines down the records is the run in that year's column.""" + # A period's cells are those of the element being written: read by + # date or by position as Python read them (``_frame``). + frame = self._frame(ctx) if period is not None else None + key = (id(node), ctx.current_sheet, ctx.track_role, period, + func in self._LIST_SAFE_FUNCS, + None if frame is None else (frame.offset, frame.window, + tuple(getattr(frame.var, '_keys', None) or ()))) + hit = self._ranges.get(key) + if hit is not None and hit[0] is node: + return self._ship_value(node, ctx) if hit[1] is None else hit[1] + text = self._cell_list_text(node, ctx, func, period) + if not self._mirrored(node, ctx): + self._ranges[key] = (node, text) + return self._ship_value(node, ctx) if text is None else text + + # Does a list of lines hold a series — read once per render. + def _lists_series(self, node: ListExpr) -> bool: + hit = self._listed.get(id(node)) + if hit is None or hit[0] is not node: + hit = self._listed[id(node)] = (node, _lists_series(node)) + return hit[1] + + # Is a line of ``node`` mirrored into this sheet's header? Its cell then + # depends on the formula being rendered (the mirror reads the row + # itself), so the text is read for each formula. + def _mirrored(self, node: ListExpr, ctx: RenderCtx) -> bool: + book = self.time + own = book.sheets.get(ctx.current_sheet) if book is not None else None + if own is None or not own.alias: + return False + return any(isinstance(item, VarRef) and item.var.id in own.alias + for item in node.items) + + # The text of ``_render_cell_list``, or None where the cell carries its + # computed value. + def _cell_list_text( + self, node: ListExpr, ctx: RenderCtx, func: Optional[str], + period: Optional[int], + ) -> Optional[str]: + cells: List[tuple[str, str]] = [] + for item in node.items: + found = self._cell_of(item, ctx, period) + if found is None: + return None + cells.append(found) + refs = [cell for cell, _vid in cells] + sheets = {self.var_to_sheet.get(vid) for _cell, vid in cells} + first, vid = cells[0] + if len(sheets) == 1 and _contiguous_run(refs): + start = self._maybe_qualify(first, vid, ctx.current_sheet) + return start if len(refs) == 1 else f"{start}:{refs[-1]}" + if func in self._LIST_SAFE_FUNCS: + return ", ".join( + self._maybe_qualify(cell, vid, ctx.current_sheet) for cell, vid in cells + ) + return None + + def _render_block(self, node: ListExpr, ctx: RenderCtx) -> Optional[str]: + """A list of rows as one rectangular range argument — the records + of a table, each a list of its cells behind a literal label, or + laid-out rows one under another — ``Deals!A3:C5``, ``Inputs!B2:D3``. + A record's label is the cell before its first field, in the + sheet's first column, where the records layout writes it; with + anything between the two (a unit column) no cell can be named for + it. None when ``node`` is a list of cells, not of rows; when the + rows form no rectangle on one sheet the cell carries its computed + value.""" + from openpyxl.utils.cell import coordinate_to_tuple + + rows: List[List[str]] = [] + owners: List[str] = [] + for item in node.items: + if isinstance(item, ListExpr): + cells = item.items + labelled = bool(cells) and isinstance(cells[0], Literal) and isinstance( + cells[0].value, str) + refs: List[str] = [] + for sub in cells[1 if labelled else 0:]: + found = self._cell_of(sub, ctx) + if found is None: + return self._ship_value(node, ctx) + refs.append(found[0]) + owners.append(found[1]) + if not refs: + return self._ship_value(node, ctx) + if labelled: + row_no, col = coordinate_to_tuple(refs[0]) + if col != 2: + return self._ship_value(node, ctx) + refs.insert(0, f"A{row_no}") + rows.append(refs) + elif isinstance(item, VarRef) and ( + isinstance(item.var._value, list) or hasattr(item.var._value, "roles") + ): + vid = self._coord_var_id(item.var, ctx) + addr = self.addresses.get(vid) if vid is not None else None + if vid is None or addr is None: + return self._ship_value(node, ctx) + rows.append(list(addr.values)) + owners.append(vid) + else: + return None + if not rows: + return None + if len({self.var_to_sheet.get(vid) for vid in owners}) != 1: + return self._ship_value(node, ctx) + grid = [[coordinate_to_tuple(ref) for ref in refs] for refs in rows] + cols = [c for _r, c in grid[0]] + tops = [row[0][0] for row in grid] + if ( + any([c for _r, c in row] != cols for row in grid) + or any(len({r for r, _c in row}) != 1 for row in grid) + or any(b - a != 1 for a, b in zip(cols, cols[1:])) + or any(b - a != 1 for a, b in zip(tops, tops[1:])) + ): + return self._ship_value(node, ctx) + first = self._maybe_qualify(rows[0][0], owners[0], ctx.current_sheet) + last = rows[-1][-1] + return first if rows[0][0] == last else f"{first}:{last}" + + def _cell_of(self, item: Expr, ctx: RenderCtx, + period: Optional[int] = None) -> Optional[tuple[str, str]]: + """The one cell ``item`` stands for — a scalar line's, one element + of a laid-out row (``flow[0]``), or, given a ``period``, a row's + cell of that period, as Python read it (``_read_index``: of the + same date where lines on other windows were aligned) — with the id + it is addressed through; None when no cell holds it.""" + if isinstance(item, VarRef): + var_id = self._coord_var_id(item.var, ctx) + addr_obj = self.addresses.get(var_id) if var_id is not None else None + if addr_obj is not None and var_id is not None: + cells = addr_obj.values + read = (self._read_index(item.var, var_id, addr_obj, ctx) + if period is not None and isinstance(item.var._value, list) else None) + if read is not None and read[1]: + return (cells[read[0]], var_id) if 0 <= read[0] < len(cells) else None + if len(cells) == 1: + return cells[0], var_id + if read is not None and 0 <= read[0] < len(cells): + return cells[read[0]], var_id + return None + inner = getattr(item.var, "_expr", None) + if isinstance(inner, Subscript): + item = inner + if ( + isinstance(item, Subscript) + and isinstance(item.base, VarRef) + and isinstance(item.key, int) + and not isinstance(item.key, bool) + ): + var_id = self._coord_var_id(item.base.var, ctx) + addr_obj = self.addresses.get(var_id) if var_id is not None else None + if addr_obj is not None and var_id is not None: + n = len(addr_obj.values) + key = item.key + idx = key + n if key < 0 and self._is_native_row(item.base.var, addr_obj) else key + if 0 <= idx < n: + return addr_obj.values[idx], var_id + return None + def _unlaid_list_arg(self, call: FuncCall, ctx: RenderCtx) -> Optional[Expr]: """The first list argument of ``call`` that has no cells in this workbook — a Variable never attached to the model, an intermediate such as ``x * 2`` or ``x[0:2]`` of one, a literal ``[10, 20, 30]`` — or None when every list can be written as a range. Scalar arguments never count: one value is one cell or - one literal either way.""" - for arg in call.args: + one literal either way, and nor does a list at a position the + registry writes as a cell.""" + positions = range_positions(call.func.upper()) + for i, arg in enumerate(call.args): + if positions is not None and i not in positions: + continue if isinstance(arg, Literal): if isinstance(arg.value, list): return arg @@ -1591,7 +1908,7 @@ def _inline_unlaid_aggregate( of the formula stays live. When the call is the cell's whole formula (``owner`` None) the cell carries its computed value. """ - if call.func.upper() not in _RANGE_TAKING_FUNCS or _per_period_call(call): + if not is_range_taking(call.func.upper()) or _per_period_call(call, self._lists_series): return None arg = self._unlaid_list_arg(call, ctx) if arg is None: @@ -1649,6 +1966,122 @@ def _lookup_key_index(self, base_var: Any, key: Any, addr_obj: VariableAddresses return keys.index(key) return None + def _dated_cell(self, node: VarRef, var_id: str, ctx: RenderCtx) -> Optional[str]: + """The cell of ``node``'s line this formula reads, where it is not + the one of the formula's own column: the cell Python read for the + value being written (``_frame``). By DATE where two windows were + aligned — an operator or a one-call-per-period function over a + line on a window of its own inside the sheet's (a tranche from + March in a model from January: its cell of the same date, as + ``_onto_window`` reads it) — a period the line has no column for + continued by its declared ``extend=`` rule: ``0`` for a flow, its + first or last cell for a rate held, the value where no rule + continues it. By KEY where lines along the same keys in another + order were combined; by position otherwise — the column's cell, + so a formula over one other window, which Python read by + position, reads it by position too. ``None`` where that is the + cell of the formula's own column.""" + if ctx.frame is None and _by_position(ctx.self_var): + return None # the column's own cell, as before + if not isinstance(getattr(node.var, '_value', None), list): + return None + addr_obj = self.addresses.get(var_id) + if addr_obj is None: + return None + read = self._read_index(node.var, var_id, addr_obj, ctx) + if read is None: + return None + j, dated = read + cells = addr_obj.values + if 0 <= j < len(cells): + if j == ctx.period_idx: + return None + return self._maybe_qualify(cells[j], var_id, ctx.current_sheet) + if not dated: + return None + rule = getattr(node.var, '_extend', None) + if rule == 'zero': + return "0" + if rule == 'hold': + held = cells[0] if j < 0 else cells[-1] + return self._maybe_qualify(held, var_id, ctx.current_sheet) + return self._ship_value(node, ctx) + + def _frame(self, ctx: RenderCtx) -> _Frame: + """The value the walk is writing an element of: the intermediate + it entered (``_entered``), else the cell's own line.""" + frame = ctx.frame + if isinstance(frame, _Frame): + return frame + return _Frame(ctx.self_var, 0, _aligned_key(ctx.self_var)) + + def _sheet_key(self, var_id: str) -> Optional[tuple]: + """The window of the sheet a line is written on (its cells run on + it), or None where its columns are not dated.""" + book = self.time + sheet = self.var_to_sheet.get(var_id) + if book is None or sheet is None: + return None + st = book.sheets.get(sheet) + return None if st is None else st.key + + @staticmethod + def _index_in(var: Any, window: Optional[tuple], frame: _Frame, i: int) -> int: + """The element of ``var`` (its values on ``window``) Python read + for ``frame``'s element ``i``: of the same date where ``frame`` + was aligned by date, of the same key where the frame runs along + the same keys in another order, else of the same position.""" + if frame.window is not None: + if window is None or window[0] != frame.window[0]: + return i + return i - _periods_from(frame.window, window) + keys = getattr(frame.var, '_keys', None) + own = getattr(var, '_keys', None) + if (keys and own and list(own) != list(keys) and 0 <= i < len(keys) + and keys[i] in own): + return list(own).index(keys[i]) + return i + + def _read_index(self, var: Any, var_id: str, addr_obj: Any, + ctx: RenderCtx) -> Optional[tuple[int, bool]]: + """``(j, dated)``: the cell of ``var``'s row Python read for the + element being written, and whether by date (``j`` then may fall + outside the row: its ``extend=`` rule continues it); None where + the row's cells are not its own periods, read as before.""" + frame = self._frame(ctx) + i = ctx.period_idx + frame.offset + if frame.window is not None: + window = self._sheet_key(var_id) + if (window is not None and window[0] == frame.window[0] + and (window[2] is None or len(addr_obj.values) == window[2])): + return self._index_in(var, window, frame, i), True + if not self._is_native_row(var, addr_obj): + return None + return self._index_in(var, None, frame, i), False + + def _entered(self, var: Any, ctx: RenderCtx) -> contextlib.AbstractContextManager[None]: + """While an unnamed intermediate's expression is written inline, + the walk writes ``var``'s element — the one Python read for the + enclosing element (``_index_in``). Where neither it nor the cell + reads anything but the column's own cells, there is no frame to + enter.""" + if var is None or (ctx.frame is None and _by_position(ctx.self_var) + and _by_position(var)): + return contextlib.nullcontext() + return self._entered_frame(var, ctx) + + @contextlib.contextmanager + def _entered_frame(self, var: Any, ctx: RenderCtx) -> Iterator[None]: + parent = self._frame(ctx) + window = _aligned_key(var) + index = self._index_in(var, window, parent, ctx.period_idx + parent.offset) + previous = ctx.frame + ctx.frame = _Frame(var, index - ctx.period_idx, window) + try: + yield + finally: + ctx.frame = previous + def _resolve_var_addr( self, var_id: str, period_idx: int, current_sheet: Optional[str] = None ) -> str: @@ -1675,7 +2108,7 @@ def _maybe_qualify(self, addr: str, var_id: str, current_sheet: Optional[str]) - #: ``XIRR`` are not among them: their second argument is a guess / #: a date range, so a comma list would be read as another argument #: entirely. - _LIST_SAFE_FUNCS = {"SUM", "AVERAGE", "MIN", "MAX", "COUNT", "NPV"} + _LIST_SAFE_FUNCS = {"SUM", "AVERAGE", "MIN", "MAX", "COUNT", "COUNTA", "NPV"} def _resolve_var_range( self, diff --git a/src/modeleon/compile/excel/totals.py b/src/modeleon/compile/excel/totals.py index e73fd3c..cba09e4 100644 --- a/src/modeleon/compile/excel/totals.py +++ b/src/modeleon/compile/excel/totals.py @@ -27,10 +27,18 @@ from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Tuple -from ...core.time import bucket_ranges +from ...core.calendar import GREGORIAN +from ...core.time import article, bucket_ranges -#: One interleaved column: ``('month', i)`` / ``('quarter', bi)`` / +#: The kind of a sheet's OWN periods in a plan — its native grain's +#: columns, whatever that grain is. A bucket is keyed by its ruler's +#: name (``'month'``, ``'quarter'``, ``'half'``, ``'year'``), so a day +#: sheet may carry month buckets beside its days without the two keys +#: meeting (they used to: the native columns were spelled ``'month'``). +NATIVE = "period" + +#: One interleaved column: ``(NATIVE, i)`` / ``('quarter', bi)`` / #: ``('year', bi)`` — in on-sheet order. ColKey = Tuple[str, int] @@ -63,10 +71,17 @@ class TotalsPlan: #: (every book without a compare layout) collapses both to the #: historical single-column reading. stride: int = 1 + #: The sheet's own ruler — what its native captions shorten to under + #: a tier (an hour keeps its clock, "13:00", under its day). + grain: Optional[str] = None + + def native_col(self, i: int) -> int: + """The column of the sheet's own period ``i``.""" + return self.col_of[(NATIVE, i)] - def month_col(self, i: int) -> int: - return self.col_of[("month", i)] + #: The name this had when every native period was a month. + month_col = native_col def cell(self, key: ColKey, slot: int = 0) -> int: """The physical column of one SLOT inside ``key``'s group.""" @@ -99,14 +114,14 @@ def build_totals_plan( (label, lo, hi) for (label, lo, hi) in bucket_ranges(grain, kind, str(start), n) ] - # Walk the months; a bucket column follows its last month — + # Walk the periods; a bucket column follows its last period — # quarter before year where both close on the same month. ends: Dict[str, Dict[int, int]] = { kind: {hi: bi for bi, (_l, _lo, hi) in enumerate(plan.buckets[kind])} for kind in usable } for i in range(n): - plan.columns.append(("month", i)) + plan.columns.append((NATIVE, i)) for kind in usable: bi = ends[kind].get(i + 1) if bi is not None: @@ -115,25 +130,83 @@ def build_totals_plan( plan.col_of[key] = base_col + offset * stride plan.depth = 1 + len(usable) plan.stride = stride + plan.grain = grain return plan def _coarser(kind: str, grain: str) -> bool: - order = {"day": 0, "month": 1, "quarter": 2, "year": 3} - return order.get(kind, -1) > order.get(grain, 99) + """Whether ``kind`` is a bucket a ``grain`` sheet can total by — a + ruler ``grain`` nests in; an unknown word never is.""" + return GREGORIAN.coarser(kind, grain) + + +class TotalsWarning(UserWarning): + """A view declares a totals kind a sheet cannot draw — its periods do + not lie inside that kind's (weeks in months), or it straddles another + declared kind's (weeks beside months over days). The column is left + out; the warning says why.""" + + +def drawable_kinds(kinds: Tuple[str, ...], grain: str) -> Tuple[Tuple[str, ...], List[str]]: + """The declared totals ``kinds`` a ``grain`` sheet draws, in their + order, and a note for each it cannot. A kind finer than the sheet + (a month column on a quarterly sheet) is skipped without a word — + one view serves sheets of several grains. A kind the sheet's periods + do not lie inside (a month column over weeks) is a note; so is a + kind that does not nest with one already kept (weeks and months over + days — the tiers of a header must nest).""" + kept: List[str] = [] + notes: List[str] = [] + up = GREGORIAN.rolls_up_to(grain) if GREGORIAN.knows(grain) else () + for kind in kinds: + if kind in kept or not (GREGORIAN.knows(kind) and GREGORIAN.knows(grain)): + continue + if kind == grain or GREGORIAN.nests(kind, grain): + continue + if not GREGORIAN.nests(grain, kind): + notes.append( + f"'{kind}' totals are not drawn: a {grain} does not lie inside one " + f"{kind} ({grain}s straddle {kind}s). A {grain} sheet totals by " + f"{', '.join(up) if up else 'nothing coarser'}." + ) + continue + clash = next((k for k in kept if not GREGORIAN.comparable(k, kind)), None) + if clash is not None: + notes.append( + f"'{kind}' totals are not drawn beside '{clash}' on {article(grain)} {grain} sheet: " + f"{kind}s and {clash}s do not nest, so their header tiers cannot " + f"stand one over the other. Total by one of them." + ) + continue + kept.append(kind) + return tuple(kept), notes def display_label(bucket_label: str, kind: str, style: str) -> str: """``2026-Q1`` → ``Q1 2026`` (finance) — same voice as the month - labels beside it; iso keeps the native spelling.""" + labels beside it; iso keeps the native spelling. A month bucket (a + day sheet's month total) is written as the month ruler writes a + month: ``Jan 2026``, ``Jan-26``.""" if style == "iso": return bucket_label - if kind == "quarter" and "-Q" in bucket_label.upper(): - y, q = bucket_label.upper().split("-Q") - return f"Q{q} {y}" + if kind in ("day", "month", "week", "iso_year"): + ruler = GREGORIAN.ruler(kind) + try: + return ruler.label(ruler.parse(bucket_label), style) + except ValueError: + return bucket_label + tag = _TAG.get(kind) + if tag and f"-{tag}" in bucket_label.upper(): + y, n = bucket_label.upper().split(f"-{tag}") + return f"{tag}{n} {y}" return bucket_label +#: The letter a quarter's and a half's keys carry (``2026-Q1``, +#: ``2026-H2``) — what the finance spelling leads with (``Q1 2026``). +_TAG = {"quarter": "Q", "half": "H"} + + def regrain_rule_of(var: Any) -> Any: """The line's declared coarsening rule, when the writer can spell it as Excel arithmetic: a recipe name (``'sum'``, ``'mean'``, @@ -255,17 +328,27 @@ def rule_formula(rule: str, refs: List[str], return None if body is None else f"={body}" -def short_label(label: str, kind: str, style: str, hierarchical: bool) -> str: +def short_label(label: str, kind: str, style: str, hierarchical: bool, + grain: Optional[str] = None) -> str: """The label a HIERARCHICAL header wants: the year lives on its own row, so the rows below drop it — "Jan", "Q1" — while a flat header keeps the full form. Non-finance styles keep their native spelling - either way.""" + either way. ``grain`` is the sheet's own ruler: a clock ruler's + captions keep their clock ("13:00"), the day on the tier above.""" if not hierarchical or style != "finance": - return display_label(label, kind, style) if kind != "month" else label - if kind == "month": + return display_label(label, kind, style) if kind != NATIVE else label + if kind == NATIVE: + if grain is not None and GREGORIAN.knows(grain) and GREGORIAN.ruler(grain).sub_day: + return label.rsplit(" ", 1)[-1] if " " in label else label return label.split(" ")[0] if " " in label else label - if kind == "quarter" and "-Q" in label.upper(): - return f"Q{label.upper().split('-Q')[1]}" + if kind in ("day", "month", "week"): + # A month or week tier over days: "Jan", "W05" — the year on + # the tier above. + full = display_label(label, kind, style) + return full.split(" ")[0] if " " in full else full + tag = _TAG.get(kind) + if tag and f"-{tag}" in label.upper(): + return f"{tag}{label.upper().split(f'-{tag}')[1]}" return label @@ -287,15 +370,22 @@ def header_rows( paints and merges it, and any other renderer can draw the same header from it. """ - kinds_top_down = [ - k for k in ("year", "quarter") if k in plan.kinds - ] + # Coarsest tier on top — the calendar's order, whatever order the + # kinds were declared in; a month tier stands over a day sheet's + # days. + kinds_top_down = sorted(plan.kinds, key=GREGORIAN.rank, reverse=True) rows: List[dict] = [] hierarchical = True + + def tiered(kind: str) -> bool: + # A day tier drops its month and year only under a month tier + # that carries them; over an hour sheet alone it reads in full. + return hierarchical and (kind != "day" or "month" in plan.kinds) + for row_i, kind in enumerate(kinds_top_down, start=1): cells = [] for bi, (blabel, lo, hi) in enumerate(plan.buckets[kind]): - first = plan.col_of[("month", lo)] + first = plan.col_of[(NATIVE, lo)] # …up to but NOT including its own total column: that one # carries the bucket's "Total" label, starting on this very # row. Swallowing it left the label nowhere to go but the @@ -307,20 +397,20 @@ def header_rows( cells.append({ "col": first, "colspan": last - first + 1, - "label": short_label(blabel, kind, plan.style, hierarchical), + "label": short_label(blabel, kind, plan.style, tiered(kind)), "kind": kind, }) rows.append({"row": row_i, "cells": cells}) month_row = len(kinds_top_down) + 1 cells = [ { - "col": plan.month_col(i), + "col": plan.native_col(i), "colspan": plan.stride, "label": short_label( month_labels[i] if i < len(month_labels) else "", - "month", plan.style, hierarchical, + NATIVE, plan.style, hierarchical, plan.grain, ), - "kind": "month", + "kind": NATIVE, } for i in range(plan.n) ] @@ -344,7 +434,7 @@ def header_rows( "rowspan": rowspan, "label": ( f"{plan.totals_word} " - f"{short_label(blabel, kind, plan.style, hierarchical)}" + f"{short_label(blabel, kind, plan.style, tiered(kind))}" ), "kind": kind, }) diff --git a/src/modeleon/compile/excel/totals_writer.py b/src/modeleon/compile/excel/totals_writer.py index 0352ab8..8a6c2ec 100644 --- a/src/modeleon/compile/excel/totals_writer.py +++ b/src/modeleon/compile/excel/totals_writer.py @@ -25,9 +25,10 @@ from typing import Any, Dict from .addresses import VariableAddresses +from .cached_values import remember from .layout import line_keys from .translator import ExcelTranslator -from .totals import TotalsPlan, listed_or_ranged, regrain_rule_of, rule_formula +from .totals import NATIVE, TotalsPlan, listed_or_ranged, regrain_rule_of, rule_formula from .writer import resolve_number_format from ...core.multi_variable import MultiVariableBase from ...core.variable import Variable @@ -48,7 +49,7 @@ def _vars_by_place(mv: MultiVariableBase, prefix: str = "") -> Dict[str, Variabl component names from ``mv`` down (``tax``, ``income.revenue``).""" out: Dict[str, Variable] = {} for name in mv._component_order: - comp = mv._components[name] + comp = mv._component(name) if isinstance(comp, Variable): out.setdefault(f"{prefix}{name}", comp) elif isinstance(comp, MultiVariableBase): @@ -177,6 +178,10 @@ def bucket_cells( # ROUND, a product of two flows): its bucket is the SUM of its # own month cells, live as a rule line's. rule, tracked = "sum", None + elif rule is None and not authored and (picked := _picked_cell(var, pvar)): + # A time row over hours: the day's first start or last stop is + # this very row's hour cell, live as a rule line's. + rule, tracked = picked, None # A compare column (Var, Var %, Share) of a line whose track rows # have no fold of their own: its formula over those rows' bucket # cells, all on this sheet. @@ -184,6 +189,7 @@ def bucket_cells( and getattr(var, "_expr", None) is not None and all(getattr(r, "id", None) in bucket_refs[kind] for r in var._expr.iter_refs())) + own_time = pvar is not None and _reads_its_own_time(var, pvar) for bi, (_label, lo, hi) in enumerate(plan.buckets[kind]): formula: Any = None value: Any = None @@ -191,7 +197,7 @@ def bucket_cells( if rule is not None: _slot = getattr(var, '_col_slot', 0) or 0 month_refs = [ - f"{_col_letter(plan.cell(('month', i), _slot))}{month_row}" + f"{_col_letter(plan.cell((NATIVE, i), _slot))}{month_row}" for i in range(lo, min(hi, plan.n)) if i < len(addresses[vid].values) ] @@ -257,6 +263,10 @@ def bucket_cells( # True when a formula's text carries a computed value baked # in: that formula is correct for the current inputs only. "inlined": bool(inlined), + # What the engine computed for a time row's bucket on a + # clock sheet, kept beside its formula for a reader that does not recalculate. + "cached": (_bucket_value(pvar, bi, var) + if own_time and pvar is not None and formula is not None else None), } return out @@ -277,6 +287,8 @@ def write_totals( cell = ws[ref] if spec["formula"] is not None: cell.value = spec["formula"] + if spec.get("cached") is not None: + remember(ws, ref, spec["cached"]) elif spec["value"] is not None: cell.value = spec["value"] cell.font = Font(bold=True) @@ -313,6 +325,30 @@ def _shown_series(var: Variable) -> Any: return value[blend if blend in roles else roles[0]] +def _reads_its_own_time(var: Variable, pvar: Any) -> bool: + """A time row whose projection reads its own cells — a clock sheet's + start, end or days (``core.projection._project_time_row``).""" + from ...core.expr import Regrain + from ...core.time import time_field_of + + expr = getattr(pvar, "_expr", None) + return (isinstance(expr, Regrain) and time_field_of(var) is not None + and getattr(expr.source, "id", None) == getattr(var, "id", None)) + + +def _picked_cell(var: Variable, pvar: Any) -> str | None: + """``'first'`` or ``'last'`` when the projection reads each bucket off + this very row's own cells, as a clock sheet's time row does (a day's + first start, its last stop: ``core.projection._project_time_row``).""" + from ...core.expr import Regrain + + expr = getattr(pvar, "_expr", None) + if (isinstance(expr, Regrain) and expr.recipe in ("first", "last") + and getattr(expr.source, "id", None) == getattr(var, "id", None)): + return str(expr.recipe) + return None + + def _sums_its_months(var: Variable, pvar: Any, tracked: Any, buckets: Any, sheet_name: str) -> bool: """The engine's bucket is the sum of the months this row shows, held diff --git a/src/modeleon/compile/excel/view.py b/src/modeleon/compile/excel/view.py index 9ed1bdf..2e2df2b 100644 --- a/src/modeleon/compile/excel/view.py +++ b/src/modeleon/compile/excel/view.py @@ -26,7 +26,7 @@ from contextlib import contextmanager from contextvars import ContextVar -from dataclasses import dataclass +from dataclasses import dataclass, replace from dataclasses import fields as _dc_fields from html import escape from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple, TypeVar @@ -240,6 +240,26 @@ def _node_value(node: Any) -> Any: return node +#: The kinds a view's ``timeline={'totals': [...]}`` may name — the +#: subtotal columns the book knows how to draw. Anything else is refused +#: by name: a word the book cannot draw must not vanish into "no totals". +TOTALS_KINDS = ("day", "week", "month", "quarter", "half", "year", "iso_year") + + +def totals_kinds(declared: Any) -> tuple: + """The declared totals kinds, in their order, or a refusal naming + the words the book does not draw.""" + kinds = tuple(declared or ()) + unknown = [k for k in kinds if k not in TOTALS_KINDS] + if unknown: + raise ValueError( + f"ExcelView timeline totals may name {', '.join(TOTALS_KINDS)}; " + f"got {', '.join(repr(k) for k in unknown)}. Nothing else is a " + f"totals kind." + ) + return kinds + + class ExcelView(MultiVariable): """Presentation settings for the Excel + notebook renderers, as a model. @@ -257,6 +277,10 @@ class ExcelView(MultiVariable): _TRACK_MODES = ("blend", "rows", "compare") + #: Its children hold settings, not lines: a list in ``timeline`` or + #: ``sheet`` is a list of words, never a series placed on a window. + _settings_only = True + def __init__( self, display_name: Optional[str] = None, @@ -274,6 +298,9 @@ def __init__( sheet: Any = None, ) -> None: super().__init__(display_name or "ExcelView") + # A named view is one a caller applies; an unnamed one is + # meant as a model's own view (see :func:`unread_view_notes`). + self.__dict__['_named'] = display_name is not None if tracks is not None: # Two authoring forms: the mode string, or the dict form # {'mode': ..., 'shown': [...]} that also declares the @@ -316,6 +343,8 @@ def __init__( f"ExcelView grain= must be one of {', '.join(GRAINS)}; " f"got {grain!r}." ) + if isinstance(timeline, dict): + totals_kinds(timeline.get("totals")) # refuses by name for name, value in ( ("orient", orient), ("font", font), @@ -337,8 +366,10 @@ def _grouped(self) -> Dict[str, Any]: def __setattr__(self, name: str, value: Any) -> None: # Any mutation (field set, sub-view adoption) invalidates the - # cached snapshot below — the cache must never outlive an edit. + # cached snapshot below — the cache must never outlive an edit — + # and whatever an open :func:`view_pass` resolved through it. self.__dict__.pop("_snapshot_cache", None) + _VIEW_EDITS[0] += 1 super().__setattr__(name, value) def snapshot(self) -> ResolvedView: @@ -403,10 +434,7 @@ def snapshot(self) -> ResolvedView: type_styles=cell_types.get("styles"), time_header=timeline.get("header"), time_label_format=timeline.get("label_format"), - time_totals=tuple( - k for k in (timeline.get("totals") or ()) - if k in ("quarter", "year") - ), + time_totals=totals_kinds(timeline.get("totals")), time_totals_word=timeline.get("totals_word"), time_opening=timeline.get("opening"), time_repeat_on_anchor=timeline.get("repeat_on_anchor"), @@ -490,6 +518,16 @@ def _timeline(header: Any, label_format: Optional[str]) -> str: f"{body}" ) + def _clone(self) -> "ExcelView": + # A copy made on adoption (a projection carries the view over, a + # second parent takes it) is the same view: it keeps its name and + # remembers its source, so a slot pointing at the source reads + # the copy too (see :func:`unread_view_notes`). + view = super()._clone() + view.__dict__['_named'] = self.__dict__.get('_named', False) + view.__dict__['_base'] = self + return view # type: ignore[return-value] + def extend(self, **overrides: Any) -> "ExcelView": """Return a copy with ``overrides`` (whole sub-views) applied.""" unknown = set(overrides) - set(_GROUPS) @@ -500,7 +538,11 @@ def extend(self, **overrides: Any) -> "ExcelView": ) merged = self._grouped() merged.update(overrides) - return ExcelView(**{k: v for k, v in merged.items() if v is not None}) + view = ExcelView(**{k: v for k, v in merged.items() if v is not None}) + # The copy remembers what it extends: a view read through its + # extension is read (see :func:`unread_view_notes`). + view.__dict__['_base'] = self + return view @classmethod def engine_default(cls) -> "ExcelView": @@ -574,6 +616,22 @@ def _layered(mine: Any, under: Any) -> Any: #: Context-local, so two threads or asyncio tasks rendering under #: different house views at once never see each other's. _HOUSE: ContextVar[Optional[ResolvedView]] = ContextVar("modeleon_house_view", default=None) +#: The ExcelView objects the house layer was given, so a view a model keeps +#: and renders under is read (:func:`unread_view_notes`). +_HOUSE_VIEWS: ContextVar[Tuple[Any, ...]] = ContextVar("modeleon_house_views", default=()) + +#: The memo of the read-only pass in force (:func:`view_pass`): each +#: node's resolved view under the node's id, and each chain of views +#: already folded under the tuple of their ids. Every entry holds the +#: objects it is keyed by, so no id is reused while the pass lasts. +#: Context-local like the house view; it ends with the pass. +_PASS: ContextVar[Optional[Dict[Any, Any]]] = ContextVar("modeleon_view_pass", default=None) + +#: Edits of any :class:`ExcelView`, counted process-wide. A pass's memo +#: carries the count it was filled under and starts over when it moved — +#: an edit made in a nested pass or in another thread is seen, not only +#: one made in the pass's own context. +_VIEW_EDITS: List[int] = [0] def changes(view: ExcelView, base: Optional[ExcelView] = None) -> ResolvedView: @@ -606,9 +664,11 @@ def house_view(*views: "ExcelView | ResolvedView") -> Iterator[None]: context-local (a thread or an asyncio task sees only its own), and it ends with the block.""" snap: Optional[ResolvedView] = _HOUSE.get() + given: Tuple[Any, ...] = _HOUSE_VIEWS.get() for view in views: if isinstance(view, ExcelView): layer = view.snapshot() + given = (*given, view) elif isinstance(view, ResolvedView): layer = view else: @@ -617,12 +677,68 @@ def house_view(*views: "ExcelView | ResolvedView") -> Iterator[None]: ) snap = layer if snap is None else _merge(layer, snap) token = _HOUSE.set(snap) + views_token = _HOUSE_VIEWS.set(given) try: yield finally: + _HOUSE_VIEWS.reset(views_token) _HOUSE.reset(token) +#: The view a caller applied to the book being written — the layer above every +#: view the models declare. Context-local, like the house view. +_APPLIED: ContextVar[Optional[ExcelView]] = ContextVar("modeleon_applied_view", default=None) + + +@contextmanager +def applied_view(view: ExcelView) -> Iterator[None]: + """Render under one of a model's named views:: + + model.board = mo.ExcelView('Board pack', font={'name': 'Georgia'}) + with applied_view(model.board): + model.to_excel("board.xlsx") + + A named view changes nothing until a caller applies it. + Inside the block it is the layer above every view the model declares: + where it sets a field it wins, on every sheet, and whatever it leaves + unset comes from the model's own views, the house view and + ``mo.default_excel_view`` as before. A ``sheet={'order': [...]}`` it + declares orders the book. The period axis is not a look: ``orient`` + stays where the model states it. + One view at a time — an inner block's view replaces the outer one's. + Context-local; it ends with the block.""" + if not isinstance(view, ExcelView): + raise TypeError(f"applied_view takes an ExcelView; got {type(view).__name__}.") + token = _APPLIED.set(view) + try: + yield + finally: + _APPLIED.reset(token) + + +@contextmanager +def view_pass() -> Iterator[None]: + """Resolve each node's view once for the length of the block. + + A renderer asks for the view of every line several times over — for + its band, its number format, its cell type, its indent — and each ask + walks the line's whole owner chain and folds every view on it. Inside + this block a node's answer is kept and served again, and a chain of + views is folded once, whoever asks. + + Only for a block that READS a built model: nothing in it may assign a + ``default_excel_view`` or move a node under another owner, which the + memo cannot see. An edit of an :class:`ExcelView` drops the memo, as + it drops the view's snapshot, and a different :func:`house_view` or + process-wide default is a miss. Context-local; it ends with the + block.""" + token = _PASS.set({}) + try: + yield + finally: + _PASS.reset(token) + + def resolve_excel_view(node: Any) -> ResolvedView: """The effective :class:`ResolvedView` for ``node`` — resolved HIERARCHICALLY. @@ -631,8 +747,23 @@ def resolve_excel_view(node: Any) -> ResolvedView: cascade like CSS: the *closest* level that sets a field wins, falling back up the tree, with the process-wide ``mo.default_excel_view`` as the top-level default (the floor) and the :func:`house_view` in force, if any, right - above it. With no pointer anywhere, that floor is returned. + above it. With no pointer anywhere, that floor is returned. An + :func:`applied_view` in force lies over all of it. Inside a + :func:`view_pass` the answer for a node is kept for the pass. """ + g = _global_default() + floor = g.snapshot() if g is not None else None + house = _HOUSE.get() + applied = _applied_snapshot() + memo = _PASS.get() + if memo is not None: + if memo.get("edits") != _VIEW_EDITS[0]: + memo.clear() + memo["edits"] = _VIEW_EDITS[0] + hit = memo.get(id(node)) + if (hit is not None and hit[0] is node and hit[1] is floor and hit[2] is house + and hit[3] is applied): + return hit[4] seen: set[int] = set() n = node chain: list[ResolvedView] = [] # nearest-first @@ -642,18 +773,164 @@ def resolve_excel_view(node: Any) -> ResolvedView: if isinstance(v, ExcelView): chain.append(v.snapshot()) n = getattr(n, "_owner", None) or getattr(n, "_parent", None) - g = _global_default() - result = g.snapshot() if g is not None else ResolvedView() - house = _HOUSE.get() + if memo is None: + return _fold(floor, house, chain, applied) + key = (id(floor), id(house), id(applied), *map(id, chain)) + folded = memo.get(key) + if folded is None: + folded = memo[key] = ( + floor, house, applied, chain, _fold(floor, house, chain, applied)) + memo[id(node)] = (node, floor, house, applied, folded[4]) + return folded[4] + + +def _applied_snapshot() -> Optional[ResolvedView]: + """The snapshot of the :func:`applied_view` in force, its ``orient`` + dropped: the period axis is geometry a container states for itself + (the layout reads it off the container's own view), never a look a + caller lays over the book. The snapshot is cached on the view, so + the same object comes back while the view is unchanged.""" + view = _APPLIED.get() + if view is None: + return None + snap = view.snapshot() + if snap.orient is None: + return snap + cached = view.__dict__.get("_applied_cache") + if cached is not None and cached[0] is snap: + return cached[1] + stripped = replace(snap, orient=None) + view.__dict__["_applied_cache"] = (snap, stripped) + return stripped + + +def _fold( + floor: Optional[ResolvedView], house: Optional[ResolvedView], + chain: List[ResolvedView], applied: Optional[ResolvedView] = None, +) -> ResolvedView: + """The applied view over the views of ``chain`` (nearest-first) over the + house view over ``floor``, the default (``mo.default_excel_view``).""" + result = floor if floor is not None else ResolvedView() if house is not None: result = _merge(house, result) # Fold top-down (root-most first) so each lower level overrides its ancestors # and the nearest level wins; the global default stays the floor. for snap in reversed(chain): result = _merge(snap, result) + if applied is not None: + result = _merge(applied, result) return result +class UnreadViewWarning(UserWarning): + """A view hangs on a container under a name that says "view", and no + ``default_excel_view`` slot holds it: the workbook keeps its look.""" + + +def _spelled_node(node: Any) -> str: + """How a message names ``node``: its path from the model, or its name + when it floats.""" + path = getattr(node, "path", None) + if path is not None and not getattr(path, "is_floating", False): + return str(getattr(node, "id", "") or path) + return str( + getattr(node, "python_name", None) or getattr(node, "display_name", None) or "model" + ) + + +def _sources(view: Any) -> Iterator[Any]: + """``view`` and what it was made from — the base it extends, the view + a copy was cloned from — each once.""" + seen: set[int] = set() + while isinstance(view, ExcelView) and id(view) not in seen: + seen.add(id(view)) + yield view + view = view.__dict__.get("_base") + + +def _views_read(roots: Sequence[Any]) -> set[int]: + """The ids of every view some ``default_excel_view`` slot holds — under + ``roots`` and on the ancestors a root's lines resolve through — with + the views each was made from, the house layer's views and the + process-wide default.""" + read: set[int] = set() + + def note(view: Any) -> None: + read.update(id(v) for v in _sources(view)) + + note(_global_default()) + for view in _HOUSE_VIEWS.get(): + note(view) + # The look a caller applies for one book (``applied_view``) reads the + # view it was handed, named or not. + note(_APPLIED.get()) + for node in _containers(roots): + note(node.__dict__.get("default_excel_view")) + for root in roots: + # A container written on its own still resolves its lines through + # its ancestors' slots (``resolve_excel_view`` walks up). + seen: set[int] = set() + n = getattr(root, "__dict__", {}).get("_owner") or getattr(root, "__dict__", {}).get("_parent") + while n is not None and id(n) not in seen: + seen.add(id(n)) + note(getattr(n, "__dict__", {}).get("default_excel_view")) + n = getattr(n, "__dict__", {}).get("_owner") or getattr(n, "__dict__", {}).get("_parent") + return read + + +def _containers(roots: Sequence[Any]) -> Iterator[Any]: + """``roots`` and every container under them, each once; a view's own + children (its setting groups) are not entered.""" + seen: set[int] = set() + pending = list(roots) + while pending: + node = pending.pop() + if not isinstance(node, MultiVariableBase) or id(node) in seen: + continue + seen.add(id(node)) + yield node + if getattr(type(node), "_settings_only", False): + continue + pending.extend(node._components.values()) + + +def unread_view_notes( + roots: Sequence[Any], spell: Optional[Any] = None, +) -> List[Tuple[str, str]]: + """``(path, sentence)`` for every view hung on a container under + ``roots`` where nothing reads it — ``model.view = mo.ExcelView(...)``, + ``model.excel_view = …``, ``mo.Model(..., view=…)``: an unnamed view + under a name that says "view", held by no ``default_excel_view`` slot + (nor extended or copied into one). The workbook keeps its look and no + total column appears; the sentence says where a model's view goes, + the holder spelled by ``spell(node)`` when given (the author's own + name for it), else by its path. A named view is one a caller + applies, and a view kept under another name (a house style) is read + where it is kept: neither is noted.""" + read = _views_read(roots) + notes: List[Tuple[str, str]] = [] + for node in _containers(roots): + if getattr(type(node), "_settings_only", False): + continue + for name, child in node._components.items(): + if not isinstance(child, ExcelView) or child.__dict__.get("_named"): + continue + if "view" not in name.lower() or any(id(v) in read for v in _sources(child)): + continue + holder = (spell(node) if spell is not None else None) or _spelled_node(node) + notes.append(( + f"{holder}.{name}", + f"`{holder}.{name} = mo.ExcelView(...)` hangs the view on {holder} " + f"as a container, where nothing reads it: the workbook keeps its " + f"look and no total column appears. A model's view is its " + f"default_excel_view — `{holder}.default_excel_view = " + f"mo.ExcelView(...)`, or `mo.Model(..., " + f"default_excel_view=mo.ExcelView(...))`; a view kept for later " + f"is named, `mo.ExcelView('Quarterly', ...)`.", + )) + return notes + + class SheetOrderWarning(UserWarning): """``sheet={'order': [...]}`` names a sheet the model does not have; the order skips the name and places the rest.""" @@ -664,8 +941,16 @@ def declared_sheet_order(root: Any) -> Optional[Tuple[str, ...]]: [...]}``), each name once, or ``None``. Read from the view's nodes, so a model that declares none is laid out exactly as before, whatever else its view holds. A container written on its own does not read its - parent's order: the order describes the whole book.""" - view = getattr(root, "__dict__", {}).get("default_excel_view") + parent's order: the order describes the whole book. An + :func:`applied_view` that declares an order wins over the model's.""" + for view in (_APPLIED.get(), getattr(root, "__dict__", {}).get("default_excel_view")): + order = _order_of(view) + if order: + return order + return None + + +def _order_of(view: Any) -> Optional[Tuple[str, ...]]: if not isinstance(view, ExcelView): return None group = view._components.get("sheet") diff --git a/src/modeleon/compile/excel/writer.py b/src/modeleon/compile/excel/writer.py index dc3cd4c..9fbb55d 100644 --- a/src/modeleon/compile/excel/writer.py +++ b/src/modeleon/compile/excel/writer.py @@ -20,9 +20,11 @@ from __future__ import annotations +import functools import logging import numbers import re +import warnings from datetime import date, datetime, time, timedelta from pathlib import Path from typing import Any, Optional @@ -46,7 +48,7 @@ ) from .addresses import VariableAddresses from .view import resolve_excel_view, units_hidden -from ...core.expr import ListExpr, Literal +from ...core.expr import FuncCall, ListExpr, Literal from ...core.multi_variable import MultiVariableBase from ...core.variable import Variable @@ -64,6 +66,10 @@ def _refuse_pending_rows(root: "MultiVariableBase") -> None: raise ValueError(pending_message(var)) +#: The package's own frames: a warning raised to the author skips them. +_ENGINE_DIR = str(Path(__file__).resolve().parents[2]) + + def to_excel(path: str | Path, root: "MultiVariableBase") -> None: """Write a model to an ``.xlsx`` file. @@ -77,7 +83,15 @@ def to_excel(path: str | Path, root: "MultiVariableBase") -> None: """ path = Path(path) _refuse_pending_rows(root) - from .view import resolve_excel_view + from ...core.mv_context import author_frame, spelled + from .view import UnreadViewWarning, resolve_excel_view, unread_view_notes + frame = author_frame() + for _path, note in unread_view_notes( + [root], spell=lambda node: spelled(node, frame), + ): + # A view nothing reads: the book keeps its look, and a warning tells + # the author where the view goes — on the line that asked for the book. + warnings.warn(note, UnreadViewWarning, skip_file_prefixes=(_ENGINE_DIR,)) track_mode = resolve_excel_view(root).tracks or 'rows' root = expand_tracked_tree(root, mode=track_mode) # (no compare SELECTION is passed here: a declared tracks='compare' @@ -100,6 +114,10 @@ def to_excel(path: str | Path, root: "MultiVariableBase") -> None: # section row in t's sheet rather than dropping it. engine = LayoutEngine(roots, flatten_nested_sheets=True, header_rows=True) addresses = engine.compute_addresses() + if engine.refusals: + # The layout found a line it has no place for — no book while it + # stands; the message says where the line belongs. + raise ValueError(engine.refusals[0]) if not addresses: raise ValueError( "Tabs exist but contain no Variables. " @@ -248,13 +266,14 @@ def _refuse_lines_off_the_window( window = engine.window_by_sheet.get(id(sheet_mv)) if window is None or window.start is None or window.grain is None: continue # undated columns: no date to put a line under + clean: dict[int, ListExpr] = {} for var in engine._iter_sheet_variables(sheet_mv): if var.id not in addresses: continue source, loc = var, _off_window(var, window) if loc is None and var._expr is not None: read = _reads_unwritten_off_window( - var._expr, window, addresses, set() + var._expr, window, addresses, set(), clean ) if read is not None: source, loc = read, read.time @@ -264,6 +283,9 @@ def _refuse_lines_off_the_window( return def _name(v: Variable) -> str: + if not (v._display_name or getattr(v, '_python_name', None)): + from ...core.variable_ops import _line_name + return _line_name(v) return (f"'{v.display_name}'" if v.path.is_floating else f"'{v.display_name}' ({v.path})") @@ -306,10 +328,12 @@ def _periods_of(var: Variable) -> Optional[int]: return getattr(value, 'time_length', None) -def _off_window(var: Variable, window: Any) -> Any: +def _off_window(var: Variable, window: Any, periods: bool = True) -> Any: """``var``'s time location when it is a series that does not run on ``window``; ``None`` when it does, or carries no dates at all (a - constant, a coordinate row, a row of axis labels).""" + constant, a coordinate row, a row of axis labels). ``periods``: a + line placed by its own ``start=`` must also hold one value per period + of the window.""" from ...core.time import Time, _parse from .layout import _is_periodic @@ -335,7 +359,7 @@ def _off_window(var: Variable, window: Any) -> Any: # it was given — it is never stretched to the window the way a bare # list with extend= is. Shorter or longer than the window, a formula # reading it in the book would pair periods Python does not pair. - if var._grain is not None and window.periods is not None: + if periods and var._grain is not None and window.periods is not None: n = _periods_of(var) if n is not None and n != window.periods: return loc @@ -344,6 +368,7 @@ def _off_window(var: Variable, window: Any) -> Any: def _reads_unwritten_off_window( expr, window: Any, addresses: dict, seen: set, + clean: Optional[dict[int, ListExpr]] = None, ) -> Optional[Variable]: """The first input ``expr`` reads that the workbook does not write and that is declared on a window of its own, or ``None``. @@ -355,24 +380,72 @@ def _reads_unwritten_off_window( intermediates (``a + b`` inside a longer formula) have no cells either — the formula is spelled through them, so the walk goes through them too, on a stack of its own: ``sum(rows)`` is one such - step per row. + step per row. A list of lines many formulas share (a range read + once, ``RANK`` of each record among its peers) is walked once on + the window: ``clean`` keeps, by id, the lists walked through + without finding one — never one met again on its own walk's way + down (a line reading itself), which another walk could not trust. """ - pending = [iter(expr.iter_refs())] + pending: list[tuple[Any, Optional[int], bool]] = [ + (_refs_walked(expr, clean), None, _reduces(expr))] + walking: set[int] = set() + looped = False while pending: - ref = next(pending[-1], None) + refs, owner, reduced = pending[-1] + ref = next(refs, None) if ref is None: pending.pop() + if owner is not None: + walking.discard(owner) + continue + if type(ref) is ListExpr: + if clean is not None and not looped: + clean[id(ref)] = ref continue - if id(ref) in seen or ref.id in addresses: + if id(ref) in seen: + looped = looped or id(ref) in walking + continue + if ref.id in addresses: continue seen.add(id(ref)) if ref._expr is not None: - pending.append(iter(ref._expr.iter_refs())) + if (not reduced and ref._read_by_date and ref._grain is not None + and _off_window(ref, window, periods=False) is not None): + # Its operands were read by date onto a window of its own + # (two windows met in it); written from the sheet's first + # column, its periods would sit under other dates. A + # reduction over it (a total) reads no dates. + return ref + walking.add(id(ref)) + pending.append((_refs_walked(ref._expr, clean), id(ref), _reduces(ref._expr))) elif ref._grain is not None and _off_window(ref, window) is not None: return ref return None +def _reduces(expr: Any) -> bool: + """Does ``expr`` read its operands as ranges (a reduction, a call over + a range), never period by period?""" + if type(expr) is not FuncCall: + return False + from ...functions.registry import is_range_taking + return bool(is_range_taking(expr.func)) + + +# ``expr.iter_refs()``, a list of lines in ``clean`` left out, each other +# list a call holds followed by itself — walked through. +def _refs_walked(expr: Any, clean: Optional[dict[int, ListExpr]]) -> Any: + if clean is None or type(expr) is not FuncCall: + yield from expr.iter_refs() + return + for arg in expr.args: + if type(arg) is not ListExpr: + yield from arg.iter_refs() + elif id(arg) not in clean: + yield from arg.iter_refs() + yield arg + + def _has_tracked(mv: "MultiVariableBase") -> bool: from ...core.tracks import TrackValues for child in mv._components.values(): @@ -441,8 +514,8 @@ def _section_total_of(child): or getattr(node, '_owner', None)) while node is not None: if isinstance(node, MultiVariableBase): - total = next((node._components[n] for n in _TOTAL_NAMES - if n in node._components), None) + total = next((node._component(n) for n in _TOTAL_NAMES + if node._has_component(n)), None) if (total is not None and total is not child and isinstance(total, Variable)): return total @@ -541,6 +614,41 @@ def _compare_plan(decl, selection, roles, ) +def _growth_by_calendar(child, window, grain, n, shown_vals): + """``growth%`` on a grain with no fixed step a year back (a day): + each period over the period the calendar puts a year before it in + this window (:func:`~modeleon.core.time.year_back_index`), empty + where the window does not reach one. ``None`` when no period has.""" + from ...core.expr import BinOp, ListExpr, Literal, Subscript, VarRef + from ...core.time import year_back_index + try: + backs = year_back_index(str(window.start), grain, n) + except (ValueError, TypeError): + return None + if all(b is None for b in backs): + return None + items = [] + vals = [] + for t, b in enumerate(backs): + if b is None: + items.append(Literal(value=None)) + vals.append(None) + continue + items.append(BinOp( + op='/', + left=Subscript(base=VarRef(var=child), key=t), + right=Subscript(base=VarRef(var=child), key=b), + )) + a, d = shown_vals[t], shown_vals[b] + vals.append((a / d) if (a is not None and d not in (None, 0)) else None) + out = Variable(display_name='') + out._value = vals + out.var_type = 'list' + out._set_expr(ListExpr(items=items)) + out._metric_no_bucket = True + return out + + def _metric_variable(key, child, default_role, value, copies, plan, decl): """One metric COLUMN as a live Variable — one of the four in the registry, each an expression the file can hold: @@ -627,7 +735,15 @@ def _div(a, b): from ...core.time import resolve_default_window window = resolve_default_window(child) grain = getattr(window, 'grain', None) or 'month' - steps = {'month': 12, 'quarter': 4, 'year': 1}.get(grain) + # Periods in a year: a growth compares a period with the one a + # year before it. A grain of no whole months has no such step. + from ...core.calendar import GREGORIAN as _cal + months = _cal.ruler(grain).months if _cal.knows(grain) else None + steps = 12 // months if months else None + if steps is None and _cal.knows(grain) and getattr(window, 'start', None): + # No fixed step (a year of days is 365 or 366 of them): the + # calendar says which period a year back each one has. + return _growth_by_calendar(child, window, grain, n, shown_vals) if not steps or n <= steps: return None # A short first period is no base for a year's growth: the period @@ -711,6 +827,9 @@ def _flat_copy(var: Variable, series: object) -> Variable: flat._unit = var._unit flat._start = var._start flat._grain = var._grain + # A line read by date onto its window (a plain line beside a + # tracked one) is written so: each operand's cell of the same date. + flat._read_by_date = var._read_by_date # The coarsening rule MUST ride the emission copy: the totals # writer spells a ruled line's bucket as its own arithmetic # (=SUM over the month cells). Dropped, the bucket fell to the @@ -995,7 +1114,7 @@ def _collect_root_sheets(root: "MultiVariableBase") -> list[MultiVariableBase]: root, "_is_context", False ): label = root.display_name or root.python_name or "Sheet1" - items = [(n, root._components[n]) for n in root._component_order] + items = [(n, root._component(n)) for n in root._component_order] claimed = [ comp for _n, comp in items @@ -1034,7 +1153,7 @@ def _collect_root_sheets(root: "MultiVariableBase") -> list[MultiVariableBase]: # Promote a plain MV to a virtual Sheet without mutating the # user's model — new wrapper, same components by reference. label = mv.display_name or mv.python_name or mv.id - items = [(n, mv._components[n]) for n in mv._component_order] + items = [(n, mv._component(n)) for n in mv._component_order] sheets.append((name, _make_virtual_sheet(label, items, source=mv))) from .view import in_sheet_order @@ -1052,7 +1171,7 @@ def _root_parts( direct_vars: list[tuple[str, Variable]] = [] sub_mvs: list[tuple[str, MultiVariableBase]] = [] for name in root._component_order: - comp = root._components[name] + comp = root._component(name) if isinstance(comp, Variable): direct_vars.append((name, comp)) elif isinstance(comp, MultiVariableBase) and not isinstance(comp, ExcelView): @@ -1276,6 +1395,8 @@ def _nesting_level(comp: Variable) -> int: #: Number formats a period caption wears when it is the period's end #: date (``=Time!C3``): what the text caption said, as a date format. _LABEL_FORMATS = { + 'hour': {'iso': 'yyyy-mm-dd"T"hh', 'finance': 'dd mmm yyyy hh:mm', + 'compact': 'dd-mmm-yy hh:mm'}, 'day': {'iso': 'yyyy-mm-dd', 'finance': 'dd mmm yyyy', 'compact': 'dd-mmm-yy'}, 'month': {'iso': 'yyyy-mm', 'finance': 'mmm yyyy', 'compact': 'mmm-yy'}, 'year': {'iso': 'yyyy', 'finance': 'yyyy', 'compact': 'yyyy'}, @@ -1293,9 +1414,13 @@ def _label_source(translator: Any, sheet_name: str, grain: str) -> Any: own = book.sheets.get(sheet_name) if own is None: return None - vid = own.fields.get('end') + # The caption shows the period itself: its last day, or — a clock + # ruler's period ends where the next starts — its first instant. + from ...core.calendar import GREGORIAN + shown = 'start' if GREGORIAN.knows(grain) and GREGORIAN.ruler(grain).sub_day else 'end' + vid = own.fields.get(shown) if vid is None and own.anchor is not None and own.anchor.key == own.key: - vid = own.anchor.fields.get('end') + vid = own.anchor.fields.get(shown) if vid is None: return None resolve = translator.renderer._resolve_var_addr @@ -1387,6 +1512,33 @@ def resolve_number_format(comp: Variable) -> str | None: return quote_bare_literals(fmt) if fmt else fmt +#: Keys of ``ExcelView(formats=...)`` that name a kind of value, not a unit. +_NOT_UNIT_KEYS = frozenset({"number", "date"}) + + +@functools.lru_cache(maxsize=512) +def _unit_key_of(text: str) -> Optional[tuple]: + """A format key read as a unit, for :func:`_format_by_unit`.""" + from ...core.unit import Unit, unit_key + try: + return unit_key(Unit.parse(text)) + except (TypeError, ValueError): + return None + + +def _format_by_unit(fmts: dict, unit: Any) -> Optional[str]: + """The format keyed by a unit EQUAL to ``unit``: a derived line whose + unit prints canonically (``USD/(month * kW)``) finds the one keyed + ``'USD/kW/month'``. A pure number matches by its label too, so a + ``'%'`` format never dresses a ``'bps'`` line.""" + from ...core.unit import unit_key + want = unit_key(unit) + for key, fmt in fmts.items(): + if fmt and key not in _NOT_UNIT_KEYS and _unit_key_of(key) == want: + return fmt + return None + + def _resolve_number_format(comp: Variable) -> str | None: layout = getattr(comp, "_excel_layout", None) if layout is not None: @@ -1431,20 +1583,32 @@ def _resolve_number_format(comp: Variable) -> str | None: # the generic floor for unitless numerics. unit = getattr(comp, "_unit", None) if unit is not None: - by_unit = fmts.get(str(unit)) + by_unit = fmts.get(str(unit)) or _format_by_unit(fmts, unit) if by_unit: return by_unit number = fmts.get("number") if number: return number from ...core.time import time_field_of - if time_field_of(comp) in ("days", "months", "index"): + field = time_field_of(comp) + if field in ("days", "months", "index"): # A count of days or months spelled as date arithmetic - # (end - start + 1) would otherwise show as a date. + # (end - start + 1) would otherwise show as a date. An hour's + # days are its fraction of a day: shown, not rounded to 0. + if field == "days" and _on_clock(comp): + return "0.0000" return "0" return None +def _on_clock(comp: Variable) -> bool: + """Whether ``comp`` lives on a clock ruler's periods (hours).""" + from ...core.calendar import GREGORIAN + loc = getattr(comp, "time", None) + grain = getattr(loc, "grain", None) + return bool(grain) and GREGORIAN.knows(grain) and GREGORIAN.ruler(grain).sub_day + + def _date_format_of(value: object) -> str | None: """The date format a line's values call for, or ``None``. @@ -2132,9 +2296,9 @@ def _write_sheet( # the workbook itself. Summary RIGHT of detail — where # our totals stand. ws.sheet_properties.outlinePr.summaryRight = True - _kinds_depth = {"quarter": 0, "year": 0} - _enclosers = [k for k in ("year", "quarter") - if k in _plan.buckets] + from ...core.calendar import GREGORIAN as _cal + from .totals import NATIVE as _native + _enclosers = sorted(_plan.buckets, key=_cal.rank, reverse=True) # A period is a GROUP of ``stride`` columns and the # bracket must cover all of them — a level on the # first column alone drew Excel brackets over one @@ -2146,12 +2310,12 @@ def _write_sheet( _hidden_o = set(group_plan[2]) if group_plan else set() for _ci, (_kind, _bi) in enumerate(_plan.columns): _col_num = _plan.col_of[(_kind, _bi)] - if _kind == "month": + if _kind == _native: _level = len(_enclosers) - elif _kind == "quarter": - _level = 1 if "year" in _plan.buckets else 0 else: - _level = 0 + # A bucket's total column sits inside the buckets + # coarser than it: a quarter's inside the year. + _level = sum(1 for k in _enclosers if _cal.rank(k) > _cal.rank(_kind)) for _si in range(_stride_o): _lv = _level + (1 if _si in _hidden_o else 0) if _lv > 0: @@ -2919,8 +3083,11 @@ def opening_content(var: Variable, translator: ExcelTranslator, sheet_name: str) def opening_value(var: Variable) -> Any: """The value an opening cell holds (see :func:`opening_content`): the - day before the first period for an end row, 0 for a period number — - of the row itself, or of the row a header mirror repeats.""" + day before the first period for an end row (on a clock ruler the + instant the first period starts — "period 0" stops there), 0 for a + period number — of the row itself, or of the row a header mirror + repeats.""" + from ...core.calendar import GREGORIAN from ...core.time import _first_day, _parse, resolve_default_window, time_field_of mirrored = getattr(var, "_mirror_of", None) src = var if mirrored is None else mirrored @@ -2937,6 +3104,8 @@ def opening_value(var: Variable) -> Any: if not isinstance(first, date): first = (getattr(window, "first_day", None) or _first_day(_parse(window.start, window.grain), window.grain)) + if GREGORIAN.knows(window.grain) and GREGORIAN.ruler(window.grain).sub_day: + return first return first - timedelta(days=1) diff --git a/src/modeleon/compile/renderer.py b/src/modeleon/compile/renderer.py index f5d3f1e..42313ac 100644 --- a/src/modeleon/compile/renderer.py +++ b/src/modeleon/compile/renderer.py @@ -92,6 +92,13 @@ class RenderCtx: # cell"; a roll-forward inlined anywhere else would read the ENCLOSING # row's previous cell and compute another number. own_chain: Optional[Any] = None + # Which value the walk is writing an element of, and which element — + # renderer-specific: the Excel renderer sets it on entering an unnamed + # intermediate written inline, whose element may sit elsewhere than + # the cell's own column (aligned onto a window of its own, or along + # keys in another order). None = the cell's own Variable at + # ``period_idx``. + frame: Optional[Any] = None class Renderer(Protocol): diff --git a/src/modeleon/core/calendar.py b/src/modeleon/core/calendar.py new file mode 100644 index 0000000..86c4e4c --- /dev/null +++ b/src/modeleon/core/calendar.py @@ -0,0 +1,598 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The calendar — rulers, and the family they belong to. + +Time is a line of instants. A **ruler** is a grain laid on it: the rule +that cuts the line into periods — ``month`` cuts on the 1st of every +month, ``quarter`` on the 1st of January, April, July and October, +``day`` on every day, ``hour`` on every hour of the wall clock. A ruler answers four questions and nothing else: which +period holds a day (:meth:`Ruler.start_of`), where the next period +starts (:meth:`Ruler.next_start`), how a period is written +(:meth:`Ruler.label`) and read back (:meth:`Ruler.parse`). Everything +the engine does with time — a window's labels, the buckets a coarser +grain groups a window into, the day-count weights of a mean, the fields +of ``mo.time``, a short first period — is derived from those answers in +:mod:`modeleon.core.time`; nothing there asks what a grain is called. + +A **calendar** is a family of rulers that agree on how they nest: in +the gregorian calendar a day lies in one month, a month in one quarter, +a quarter in one year, so a coarser ruler's periods are unions of whole +finer ones and a roll-up is exact. :data:`GREGORIAN` is the calendar +every model runs on; its rulers — hour, day, ISO week, month, +quarter, half, year, ISO year — are the engine's grains. + +The rulers of whole days speak in ``datetime.date``. A **clock ruler** +(``sub_day``: the hour) speaks in naive ``datetime.datetime`` — the +wall clock of one fixed-offset zone, so no hour is ever skipped or +repeated; the zone itself is the window's (``default_zone``), kept with +the window for a caller that writes an instant out with its offset (a +workbook's dates carry no zone). +""" + +from __future__ import annotations + +import re +from datetime import date, datetime, time, timedelta +from typing import Dict, FrozenSet, Iterable, Optional, Tuple + +#: Three-letter month names, 1-based (``MONTH_ABBR[1] == 'Jan'``). +MONTH_ABBR = ('', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', + 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec') + +#: A day label is exactly ``YYYY-MM-DD``. ``date.fromisoformat`` also reads +#: ``2026-W01`` (an ISO week) and ``20260105``; a week label must not pass +#: as the Monday it names — a day window would then start on a date the +#: label never spells. +_DAY_LABEL = re.compile(r'^\d{4}-\d{2}-\d{2}$') + +#: An hour label is exactly ``YYYY-MM-DDTHH`` — the wall-clock hour of +#: the window's zone. +_HOUR_LABEL = re.compile(r'^(\d{4})-(\d{2})-(\d{2})T(\d{2})$') + + +def as_day(value: date) -> date: + """The day an instant falls on — a ``datetime`` cut to its date; a + ``date`` is returned as it is. A ruler of whole days answers about + days, whatever instant it is asked about.""" + return value.date() if isinstance(value, datetime) else value + + +def as_instant(value: date) -> datetime: + """``value`` as a wall-clock instant: a ``date`` is its midnight.""" + if isinstance(value, datetime): + return value + return datetime.combine(value, time()) + + +class Ruler: + """One grain of a calendar: the rule that cuts the line of days into + periods. A period is named by its first day; every answer below is + about first days. + + ``months`` is the whole calendar months one period holds (``None`` + for a ruler finer than a month — ``mo.time.months`` is refused + there). ``stub_unit`` names the finer ruler at whose boundaries a + window may open INSIDE a period — a quarter or year window starting + on the 1st of any month keeps the period's label and shortens it; + ``None`` means a window starts only on a period's first day. + """ + + name: str = '' + rank: int = 0 + months: Optional[int] = None + stub_unit: Optional[str] = None + grammar: str = '' + #: How the grain is named to a reader, and its short form ('Monthly', + #: 'M') — defined once here for any caller that names a grain. + word: str = '' + letter: str = '' + #: A ruler of a fixed number of days per period (a day, a week): + #: the workbook spells its time by adding days to the start cell. + stride_days: Optional[int] = None + #: A clock ruler: its periods are stretches of the wall clock shorter + #: than a day, named by their first instant (a ``datetime``); its + #: ``mo.time.end`` is where a period STOPS (13:00 → 14:00) — a period + #: of hours has no last day. + sub_day: bool = False + #: Hours in one period of a clock ruler (an hour: 1): the workbook + #: spells its time as the start cell plus i/24, one closed form. + stride_hours: Optional[int] = None + #: An ISO week-numbering year: 52 or 53 weeks from the Monday of the + #: week holding 4 January — the workbook spells its bounds with + #: ``DATE(y,1,4) - WEEKDAY(DATE(y,1,4),3)``. + week_years: bool = False + + def refuse_start(self, value: date) -> str: + """Why a window of this ruler cannot start on ``value`` — a day + inside its first period that no stub may open on.""" + return ( + f"A {self.name} window starts on the 1st of a month; {value.isoformat()} " + f"falls inside a month. Start the window on the 1st and keep the " + f"exact day as an input — read mo.time.days for the part of a period " + f"after it." + ) + + def parse(self, label: str) -> date: + """The first day of the period ``label`` names, or a refusal that + spells the grammar.""" + raise NotImplementedError + + def label(self, first: date, style: str = 'iso') -> str: + """How the period starting on ``first`` is written: ``iso`` (the + engine's own key), ``finance`` or ``compact``.""" + raise NotImplementedError + + def start_of(self, day: date) -> date: + """The first day of the period that holds ``day``.""" + raise NotImplementedError + + def next_start(self, first: date) -> date: + """The first day of the period after the one starting on ``first``.""" + raise NotImplementedError + + def measure(self, first: date) -> float: + """The calendar day count of the period starting on ``first`` — + the weight a time-weighted recipe uses: a whole number of days + (an ``int``) for every ruler but a clock ruler's (1/24 an hour).""" + return (self.next_start(first) - first).days + + def count(self, first: date, last: date) -> int: + """Inclusive period count from the period starting on ``first`` + to the one starting on ``last``.""" + n, cur = 1, first + while cur < last: + cur = self.next_start(cur) + n += 1 + return n + + def year_back(self, first: date) -> Optional[date]: + """The first day of the period a year before the one starting on + ``first`` — what a growth on last year compares against: the + period holding the same date a year earlier (29 February asks + for the 28th).""" + try: + back = first.replace(year=first.year - 1) + except ValueError: + back = first.replace(year=first.year - 1, day=28) + return self.start_of(back) + + def __repr__(self) -> str: + return f"Ruler({self.name!r})" + + +class _Hour(Ruler): + """The hours of the wall clock: a period is ``[13:00, 14:00)``, keyed + ``2026-10-21T13``. Its instants are naive ``datetime`` values — the + clock of the window's fixed-offset zone, where every day has 24 hours + (a zone that moves its clocks is refused where it is declared). + ``mo.time.end`` is the hour's stop, ``mo.time.days`` its 1/24 of a + day. An hour nests in its day, and so in everything a day nests in.""" + + name, rank, grammar, word, letter = 'hour', 0, "'YYYY-MM-DDTHH'", 'Hourly', 'Hr' + sub_day = True + stride_hours = 1 + + def parse(self, label: str) -> date: + m = _HOUR_LABEL.match(label) if isinstance(label, str) else None + if m is None: + raise ValueError(f"Time hour start must be 'YYYY-MM-DDTHH'; got {label!r}.") + year, month, day, hour = (int(g) for g in m.groups()) + if not 0 <= hour <= 23: + raise ValueError(f"An hour is 00-23 in {label!r}.") + try: + return datetime(year, month, day, hour) + except ValueError: + raise ValueError(f"Time hour start must be 'YYYY-MM-DDTHH'; got {label!r}.") from None + + def label(self, first: date, style: str = 'iso') -> str: + first = as_instant(first) + clock = f"{first.hour:02d}:00" + if style == 'finance': + return f"{first.day:02d} {MONTH_ABBR[first.month]} {first.year:04d} {clock}" + if style == 'compact': + return f"{first.day:02d}-{MONTH_ABBR[first.month]}-{first.year % 100:02d} {clock}" + return f"{first.year:04d}-{first.month:02d}-{first.day:02d}T{first.hour:02d}" + + def start_of(self, day: date) -> date: + return as_instant(day).replace(minute=0, second=0, microsecond=0) + + def next_start(self, first: date) -> date: + return as_instant(first) + timedelta(hours=1) + + def measure(self, first: date) -> float: + return 1 / 24 + + def count(self, first: date, last: date) -> int: + return int((as_instant(last) - as_instant(first)) // timedelta(hours=1)) + 1 + + def refuse_start(self, value: date) -> str: + hour = self.start_of(value) + return ( + f"An hour window starts on the hour; {as_instant(value).isoformat()} is " + f"inside the hour from {hour.strftime('%H:%M')}. Start it at " + f"{hour.isoformat()} and keep the exact moment as an input." + ) + + +class _Day(Ruler): + name, rank, grammar, word, letter = 'day', 1, "'YYYY-MM-DD'", 'Daily', 'D' + stride_days = 1 + + def parse(self, label: str) -> date: + if not isinstance(label, str) or not _DAY_LABEL.match(label): + raise ValueError(f"Time day start must be 'YYYY-MM-DD'; got {label!r}.") + try: + return date.fromisoformat(label) + except ValueError: + raise ValueError(f"Time day start must be 'YYYY-MM-DD'; got {label!r}.") from None + + def label(self, first: date, style: str = 'iso') -> str: + first = as_day(first) + if style == 'finance': + return f"{first.day:02d} {MONTH_ABBR[first.month]} {first.year:04d}" + if style == 'compact': + return f"{first.day:02d}-{MONTH_ABBR[first.month]}-{first.year % 100:02d}" + return first.isoformat() + + def start_of(self, day: date) -> date: + return as_day(day) + + def next_start(self, first: date) -> date: + return first + timedelta(days=1) + + def count(self, first: date, last: date) -> int: + return (last - first).days + 1 + + +#: An ISO week label: ``2026-W05`` — the ISO year, then the week. +_WEEK_LABEL = re.compile(r'^(\d{4})-W(\d{2})$') + +_WEEKDAYS = ('Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday') + + +class _Week(Ruler): + """ISO 8601 weeks: Monday to Sunday, numbered within their ISO year + (week 1 holds the year's first Thursday), keyed ``2026-W05``. A week + lies inside one ISO year and in no month, quarter or calendar year.""" + + name, rank, grammar, word, letter = 'week', 2, "'YYYY-Www'", 'Weekly', 'W' + stride_days = 7 + + def parse(self, label: str) -> date: + m = _WEEK_LABEL.match(label) if isinstance(label, str) else None + if m is None: + raise ValueError(f"Time week start must be 'YYYY-Www' (an ISO week); got {label!r}.") + try: + return date.fromisocalendar(int(m.group(1)), int(m.group(2)), 1) + except ValueError: + raise ValueError( + f"{label!r} names no ISO week: the year {m.group(1)} has " + f"{_iso_weeks_in(int(m.group(1)))} weeks." + ) from None + + def label(self, first: date, style: str = 'iso') -> str: + year, week, _ = first.isocalendar() + if style == 'finance': + return f"W{week:02d} {year:04d}" + if style == 'compact': + return f"W{week:02d}-{year % 100:02d}" + return f"{year:04d}-W{week:02d}" + + def start_of(self, day: date) -> date: + day = as_day(day) + return day - timedelta(days=day.weekday()) + + def next_start(self, first: date) -> date: + return first + timedelta(days=7) + + def count(self, first: date, last: date) -> int: + return (last - first).days // 7 + 1 + + def year_back(self, first: date) -> Optional[date]: + """The same week number of the ISO year before (W01 2026 against + W01 2025); a week 53 whose year before had 52 has none.""" + year, week, _ = first.isocalendar() + try: + return date.fromisocalendar(year - 1, week, 1) + except ValueError: + return None + + def refuse_start(self, value: date) -> str: + monday = self.start_of(value) + return ( + f"A week window starts on a Monday; {value.isoformat()} is a " + f"{_WEEKDAYS[value.weekday()]}. Start it on {monday.isoformat()}, the " + f"Monday of that week, and keep the exact day as an input." + ) + + +def _iso_weeks_in(year: int) -> int: + """52 or 53 — the weeks of an ISO year (28 December is always in its last).""" + return date(year, 12, 28).isocalendar()[1] + + +class _IsoYear(Ruler): + """The ISO week-numbering year: 52 or 53 whole weeks from the Monday + of its week 1 — what a weekly sheet totals by. Keyed by its number + (``2026``); it starts on 29 December 2025 and is not the calendar + year, so it nests weeks and days and nothing coarser than a day.""" + + name, rank, grammar, word, letter = 'iso_year', 7, "'YYYY'", 'Yearly (ISO)', 'IY' + stub_unit = 'week' + week_years = True + + def parse(self, label: str) -> date: + try: + year = int(label) + except (ValueError, TypeError): + raise ValueError(f"Time iso_year start must be 'YYYY'; got {label!r}.") from None + return date.fromisocalendar(year, 1, 1) + + def label(self, first: date, style: str = 'iso') -> str: + year = first.isocalendar()[0] + # The ISO year of the period's first day is the period's: a stub + # starts on a Monday inside it. + if style == 'compact': + return f"{year % 100:02d}" + return f"{year:04d}" + + def start_of(self, day: date) -> date: + return date.fromisocalendar(day.isocalendar()[0], 1, 1) + + def next_start(self, first: date) -> date: + return date.fromisocalendar(first.isocalendar()[0] + 1, 1, 1) + + def year_back(self, first: date) -> Optional[date]: + return date.fromisocalendar(first.isocalendar()[0] - 1, 1, 1) + + def refuse_start(self, value: date) -> str: + return ( + f"An iso_year window starts on a Monday; {value.isoformat()} is a " + f"{_WEEKDAYS[value.weekday()]}. Start it on the Monday of that week, " + f"{(value - timedelta(days=value.weekday())).isoformat()}." + ) + + +class _Months(Ruler): + """A ruler of ``months`` whole months, its periods opening on the + 1st of a month in step from January.""" + + def start_of(self, day: date) -> date: + k = self.months or 1 + return date(day.year, ((day.month - 1) // k) * k + 1, 1) + + def next_start(self, first: date) -> date: + year, month = first.year, first.month + (self.months or 1) + while month > 12: + month, year = month - 12, year + 1 + return date(year, month, 1) + + def count(self, first: date, last: date) -> int: + k = self.months or 1 + return ((last.year - first.year) * 12 + (last.month - first.month)) // k + 1 + + +class _Month(_Months): + name, rank, months, grammar, word, letter = 'month', 3, 1, "'YYYY-MM'", 'Monthly', 'M' + + def parse(self, label: str) -> date: + parts = label.split('-') if isinstance(label, str) else [] + if len(parts) != 2: + raise ValueError(f"Time month start must be 'YYYY-MM'; got {label!r}.") + try: + year, month = int(parts[0]), int(parts[1]) + except ValueError: + raise ValueError(f"Time month start must be 'YYYY-MM'; got {label!r}.") from None + if not 1 <= month <= 12: + raise ValueError(f"Month must be 01-12 in {label!r}.") + return date(year, month, 1) + + def label(self, first: date, style: str = 'iso') -> str: + year, month = first.year, first.month + if style == 'finance': + return f"{MONTH_ABBR[month]} {year:04d}" + if style == 'compact': + return f"{MONTH_ABBR[month]}-{year % 100:02d}" + return f"{year:04d}-{month:02d}" + + +class _Quarter(_Months): + name, rank, months, stub_unit, grammar = 'quarter', 4, 3, 'month', "'YYYY-Qn'" + word, letter = 'Quarterly', 'Q' + + def parse(self, label: str) -> date: + parts = label.upper().split('-Q') if isinstance(label, str) else [] + if len(parts) != 2: + raise ValueError(f"Time quarter start must be 'YYYY-Qn'; got {label!r}.") + try: + year, quarter = int(parts[0]), int(parts[1]) + except ValueError: + raise ValueError(f"Time quarter start must be 'YYYY-Qn'; got {label!r}.") from None + if not 1 <= quarter <= 4: + raise ValueError(f"Quarter must be Q1-Q4 in {label!r}.") + return date(year, (quarter - 1) * 3 + 1, 1) + + def label(self, first: date, style: str = 'iso') -> str: + year, quarter = first.year, (first.month - 1) // 3 + 1 + if style == 'finance': + return f"Q{quarter} {year:04d}" + if style == 'compact': + return f"Q{quarter}-{year % 100:02d}" + return f"{year:04d}-Q{quarter}" + + +class _Half(_Months): + name, rank, months, stub_unit, grammar = 'half', 5, 6, 'month', "'YYYY-Hn'" + word, letter = 'Half-yearly', 'H' + + def parse(self, label: str) -> date: + parts = label.upper().split('-H') if isinstance(label, str) else [] + if len(parts) != 2: + raise ValueError(f"Time half start must be 'YYYY-Hn'; got {label!r}.") + try: + year, half = int(parts[0]), int(parts[1]) + except ValueError: + raise ValueError(f"Time half start must be 'YYYY-Hn'; got {label!r}.") from None + if not 1 <= half <= 2: + raise ValueError(f"Half must be H1-H2 in {label!r}.") + return date(year, (half - 1) * 6 + 1, 1) + + def label(self, first: date, style: str = 'iso') -> str: + year, half = first.year, (first.month - 1) // 6 + 1 + if style == 'finance': + return f"H{half} {year:04d}" + if style == 'compact': + return f"H{half}-{year % 100:02d}" + return f"{year:04d}-H{half}" + + +class _Year(_Months): + name, rank, months, stub_unit, grammar = 'year', 6, 12, 'month', "'YYYY'" + word, letter = 'Yearly', 'Y' + + def parse(self, label: str) -> date: + try: + year = int(label) + except (ValueError, TypeError): + raise ValueError(f"Time year start must be 'YYYY'; got {label!r}.") from None + return date(year, 1, 1) + + def label(self, first: date, style: str = 'iso') -> str: + if style == 'compact': + return f"FY{first.year % 100:02d}" + return f"{first.year:04d}" + + +class Calendar: + """A named family of rulers, finest first, and how they nest: a + pair ``(fine, coarse)`` is in ``nesting`` when every period of + ``coarse`` is a union of whole periods of ``fine`` — then a roll-up + from ``fine`` to ``coarse`` is exact. Rulers that do not nest either + way (a week and a month) never meet in a roll-up: the engine refuses + rather than splitting a week between two months.""" + + def __init__(self, name: str, rulers: Tuple[Ruler, ...], + nesting: Iterable[Tuple[str, str]] = ()) -> None: + self.name = name + self.rulers = tuple(rulers) + self._by_name: Dict[str, Ruler] = {r.name: r for r in self.rulers} + self._nesting: FrozenSet[Tuple[str, str]] = frozenset(nesting) + + @property + def grains(self) -> Tuple[str, ...]: + """The rulers' names, finest to coarsest.""" + return tuple(r.name for r in self.rulers) + + def ruler(self, name: str) -> Ruler: + """The ruler called ``name``, or a refusal naming the family.""" + try: + return self._by_name[name] + except KeyError: + raise ValueError( + f"Unknown time grain {name!r}; supported: {', '.join(self.grains)}." + ) from None + + def knows(self, name: object) -> bool: + return isinstance(name, str) and name in self._by_name + + def rank(self, name: str) -> int: + """The ruler's place in the family's order, finest first — an + order to list them in, not a statement that they nest.""" + return self.ruler(name).rank + + def nests(self, fine: str, coarse: str) -> bool: + """Whether every period of ``coarse`` is a union of whole + periods of ``fine``.""" + return (fine, coarse) in self._nesting + + def coarser(self, kind: object, grain: object) -> bool: + """Whether ``kind`` names a ruler that ``grain`` nests in — a + bucket a ``grain`` sheet can total by; an unknown name on + either side never is.""" + if not (self.knows(kind) and self.knows(grain)): + return False + return self.nests(grain, kind) # type: ignore[arg-type] + + def comparable(self, a: str, b: str) -> bool: + """Whether ``a`` and ``b`` are one ruler or one nests in the other.""" + return a == b or self.nests(a, b) or self.nests(b, a) + + def rolls_up_to(self, fine: str) -> Tuple[str, ...]: + """The rulers ``fine`` nests in, finest first.""" + return tuple(r.name for r in self.rulers if self.nests(fine, r.name)) + + def verify(self, span: int = 420) -> None: + """Check every ruler and every nesting claim on a probe span of + ``span`` periods from 2023: labels read back as the period they + name, periods tile the line with no gap or overlap, and each + claimed nesting holds for every fine period. A clock ruler is + probed with its own tick (an hour) and over ``span`` days of its + periods, so its nestings cross month and year ends. Raises naming + the first bad period.""" + anchor = date(2023, 1, 2) # a Monday, inside every ruler's first period + + def tick(r: Ruler) -> timedelta: + return timedelta(hours=1) if r.sub_day else timedelta(days=1) + + def start(r: Ruler) -> date: + return r.start_of(as_instant(anchor) if r.sub_day else anchor) + + for r in self.rulers: + first = start(r) + for _ in range(span): + nxt = r.next_start(first) + if not first < nxt: + raise AssertionError(f"{r.name}: the period of {first} does not end after it") + if r.start_of(nxt - tick(r)) != first or r.start_of(nxt) != nxt: + raise AssertionError(f"{r.name}: the period of {first} overlaps or leaves a gap") + if r.parse(r.label(first)) != first: + raise AssertionError(f"{r.name}: {r.label(first)!r} does not read back as {first}") + first = nxt + for fine, coarse in self._nesting: + f, c = self.ruler(fine), self.ruler(coarse) + first = start(f) + for _ in range(span * 24 if f.sub_day else span): + last = f.next_start(first) - tick(f) + if c.start_of(first) != c.start_of(last): + raise AssertionError( + f"{fine} does not nest in {coarse}: the {fine} from {first} " + f"to {last} straddles two" + ) + first = f.next_start(first) + + def __repr__(self) -> str: + return f"Calendar({self.name!r}, {self.grains})" + + +HOUR = _Hour() +DAY = _Day() +WEEK = _Week() +MONTH = _Month() +QUARTER = _Quarter() +HALF = _Half() +YEAR = _Year() +ISO_YEAR = _IsoYear() + + +def _closure(pairs: Iterable[Tuple[str, str]]) -> FrozenSet[Tuple[str, str]]: + """The transitive closure of a nesting relation.""" + out = set(pairs) + while True: + more = {(a, d) for (a, b) in out for (c, d) in out if b == c} - out + if not more: + return frozenset(out) + out |= more + + +#: The gregorian calendar — the one every model runs on. Hours nest in +#: their day, days in everything; months in quarters, halves and years; +#: ISO weeks only in their ISO year (a week straddles months, quarters +#: and calendar years). +GREGORIAN = Calendar( + 'gregorian', + (HOUR, DAY, WEEK, MONTH, QUARTER, HALF, YEAR, ISO_YEAR), + _closure([ + ('hour', 'day'), + ('day', 'week'), ('day', 'month'), ('month', 'quarter'), ('quarter', 'half'), + ('half', 'year'), ('week', 'iso_year'), + ]), +) diff --git a/src/modeleon/core/entries.py b/src/modeleon/core/entries.py index f36e899..1132b14 100644 --- a/src/modeleon/core/entries.py +++ b/src/modeleon/core/entries.py @@ -22,11 +22,15 @@ through it at its offset. """ -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Set, Tuple #: How many named formulas deep the walk reads. MAX_DEPTH = 40 +#: Readings one shared ``readings`` dict keeps per track before it starts +#: afresh - room for every line of a large model, read either way. +READINGS_LIMIT = 1 << 16 + #: Functions that work month by month, whose arguments' entries are the #: result's. An aggregate (``SUM`` of a row is one number for every #: month) or a shift in time (``lag``, ``cumsum``) is not among them. @@ -42,12 +46,13 @@ _Shifted = Tuple[Any, Optional[str], int] -def entry_lines(expr: Any, track: str, *, own: bool = False) -> List[Entry]: +def entry_lines(expr: Any, track: str, *, own: bool = False, + readings: Optional[dict] = None) -> List[Entry]: """The rows of entries ``expr`` reads on ``track``, in reading order, each once. ``own``: ``expr`` was written for this track alone (``actual=s.raw * 1000``), so the lines without tracks it reads - loaded - data - are its entries too.""" - found = _Walk(track, shifts=False).formula(expr, 0, own) + data - are its entries too. ``readings``: see :func:`readings_under`.""" + found = _found(expr, track, False, own, readings) seen: Dict[Tuple[int, Optional[str]], Entry] = {} for line, role, _offset in found: seen.setdefault((id(line), role), (line, role)) @@ -55,37 +60,91 @@ def entry_lines(expr: Any, track: str, *, own: bool = False) -> List[Entry]: def shown_entered_mask(expr: Any, track: str, periods: int, *, - own: bool = False) -> Optional[List[bool]]: + own: bool = False, + readings: Optional[dict] = None) -> Optional[List[bool]]: """Per period, was this formula's month entered? A record of which cells of a formula line were entered, never used for its value. The entries of the same month decide; a formula that reads only earlier months (salaries paid a month late: ``lag(accrued)``) is asked of them, at the period each reads, and a lag's first periods - its fill, - a known number - count as entered. None when it reads no entries.""" - found = _Walk(track, shifts=True).formula(expr, 0, own) + a known number - count as entered. None when it reads no entries. + ``readings``: see :func:`readings_under`.""" + found = _found(expr, track, True, own, readings) if not found: return None from .tracks import TrackValues same_month = [entry for entry in found if entry[2] == 0] - reads = [] + reads: List[Tuple[list, int]] = [] + read: Set[Tuple[int, int]] = set() # a row reached twice is read once for line, role, offset in (same_month or found): value = getattr(line, "_value", None) if isinstance(value, TrackValues): - value = value[role] if role is not None and role in value.roles else None - if isinstance(value, list): + value = value[role] if role is not None and role in value else None + if isinstance(value, list) and (id(value), offset) not in read: + read.add((id(value), offset)) reads.append((value, offset)) if not reads: return None - def entered(t: int) -> bool: - for series, offset in reads: + # A period is entered once any read says so: each read is asked only + # about the periods none before it has answered. + mask = [False] * periods + open_periods: Any = range(periods) + for series, offset in reads: + length = len(series) + still: List[int] = [] + for t in open_periods: i = t - offset - if i < 0 or i >= len(series) or series[i] is not None: - return True - return False + if i < 0 or i >= length or series[i] is not None: + mask[t] = True + else: + still.append(t) + if not still: + break + open_periods = still + return mask - return [entered(t) for t in range(periods)] + +def readings_under(node: Any) -> Optional[dict]: + """The readings shared by every line under ``node``'s topmost + container, kept on that container (None when it keeps no attributes). + + Every line of a model is asked what it reads as it joins, and a total + reads its subtotals, which read the lines below: walked afresh for + each line, the same subtotals are read over and over. A reading is + kept per line, track and way of reading, and reused only while the + line - and every line it reads through - still holds the very value, + formula and track bindings it was read from (they are replaced, never + changed in place, when a line is rebound, re-materialized or given a + new definition). So whatever the model does between two questions, + the answer is the one a fresh walk gives; the dict only lives as long + as the model does.""" + top = node + seen: Set[int] = set() + while id(top) not in seen: + seen.add(id(top)) + above = getattr(top, '_owner', None) or getattr(top, '_parent', None) + if above is None: + break + top = above + slots = getattr(top, '__dict__', None) + if not isinstance(slots, dict): + return None + return slots.setdefault('_entry_readings', {}) + + +def _found(expr: Any, track: str, shifts: bool, own: bool, + readings: Optional[dict]) -> List[_Shifted]: + """What ``expr`` reads on ``track``: through the shared ``readings`` + when given; afresh when there are none, or when the walk would stop at + ``MAX_DEPTH`` (what it finds there depends on the way it came).""" + if readings is not None: + try: + return _Walk(track, shifts, readings).formula(expr, 0, own) + except _TooDeep: + pass + return _Walk(track, shifts).formula(expr, 0, own) def entered_mask(entries: List[Entry], periods: int) -> List[bool]: @@ -103,85 +162,228 @@ def entered_mask(entries: List[Entry], periods: int) -> List[bool]: for t in range(periods)] +class _TooDeep(Exception): + """A walk that keeps readings would stop at ``MAX_DEPTH`` here - a chain + of formulas that long, or lines that read each other.""" + + +class _Reading: + """What one line reads on one track, with what it was read from: the + line's value, formula and track bindings, the track's operand (its + owner and formula decide whether the track is a reference), each line + its formula names - with the formula read in place when it was one + (an unnamed step) - and the readings of the lines it read through.""" + + __slots__ = ('line', 'value', 'expr', 'kwargs', 'operand', 'operand_state', + 'refs', 'parts', 'found', 'height', 'too_deep') + + def __init__(self, line: Any, value: Any, expr: Any, kwargs: Any) -> None: + self.line = line + self.value = value + self.expr = expr + self.kwargs = kwargs + self.operand: Any = None + self.operand_state: Optional[Tuple[bool, Any]] = None + self.refs: List[Tuple[Any, Any]] = [] + self.parts: List['_Reading'] = [] + self.found: List[_Shifted] = [] + #: Named formulas read through, down the longest way (0: none). + self.height = 0 + #: Met too deep to read through: ``height`` is then how far down + #: the line reads at least, and the reading answers nothing. + self.too_deep = False + + @classmethod + def deeper_than(cls, line: Any, height: int) -> '_Reading': + """A line a walk gave up on: it reads at least ``height`` named + formulas down. A later walk meeting it as deep gives up at once; + one from higher up reads it again.""" + reading = cls(line, getattr(line, "_value", None), getattr(line, "_expr", None), + getattr(line, "_role_kwargs", None)) + reading.height = height + reading.too_deep = True + return reading + + class _Walk: - def __init__(self, track: str, shifts: bool) -> None: + def __init__(self, track: str, shifts: bool, + readings: Optional[dict] = None) -> None: self.track = track self.shifts = shifts # read through a lag, at its offset - self.memo: Dict[Tuple[int, bool], List[_Shifted]] = {} + if readings is None: + self.memo: Dict[Tuple[int, bool], _Reading] = {} + self.kept: Optional[Set[int]] = None + else: + self.memo = readings.setdefault((track, shifts), {}) + if len(self.memo) > READINGS_LIMIT: + self.memo.clear() + self.kept = set() # readings found still true this walk + self.open: Set[Tuple[int, bool]] = set() - def formula(self, expr: Any, depth: int, own: bool = False) -> List[_Shifted]: - from .expr import (BinOp, Compare, FuncCall, MethodCall, Paren, Restrict, - UnaryOp, VarRef) + def formula(self, expr: Any, depth: int, own: bool = False, + reading: Optional[_Reading] = None) -> List[_Shifted]: + from .expr import (BinOp, Compare, FuncCall, Literal, MethodCall, Paren, + Restrict, UnaryOp, VarRef) out: List[_Shifted] = [] stack = [(expr, 0)] seen: set = set() while stack: node, offset = stack.pop() - if isinstance(node, Restrict): - # ``line.at(track=...)`` reads that one track: this track's - # entries when it names this track, none when another. - if node.label == self.track: - stack.append((node.base, offset)) - elif isinstance(node, Paren): - stack.append((node.inner, offset)) - elif isinstance(node, UnaryOp): - stack.append((node.operand, offset)) + # Each kind of node is its own class (none is another's + # subclass), so the order of the tests is free: the common + # ones - a reference, a literal, arithmetic - come first. + if isinstance(node, VarRef): + var = node.var + if (id(var), offset) in seen: + continue + seen.add((id(var), offset)) + inner = getattr(var, "_expr", None) + inline = getattr(var, "_owner", None) is None and inner is not None + if reading is not None: + reading.refs.append((var, inner if inline else None)) + if inline: + stack.append((inner, offset)) # unnamed - part of the formula + else: + part = self.line(var, depth, own) + if reading is not None: + reading.parts.append(part) + out.extend((line, role, offset + o) + for line, role, o in part.found) + elif isinstance(node, Literal): + pass # a constant: no entry elif isinstance(node, (BinOp, Compare)): stack.extend(((node.right, offset), (node.left, offset))) + elif isinstance(node, Paren): + stack.append((node.inner, offset)) elif isinstance(node, FuncCall): if node.func.upper() in SAME_MONTH_FUNCTIONS: stack.extend((a, offset) for a in reversed(node.args)) + elif isinstance(node, UnaryOp): + stack.append((node.operand, offset)) elif isinstance(node, MethodCall): if node.method == 'copy': stack.append((node.base, offset)) elif (node.method == 'shift' and self.shifts and node.args and isinstance(node.args[0], int)): stack.append((node.base, offset + node.args[0])) - elif isinstance(node, VarRef): - var = node.var - if (id(var), offset) in seen: - continue - seen.add((id(var), offset)) - if (getattr(var, "_owner", None) is None - and getattr(var, "_expr", None) is not None): - stack.append((var._expr, offset)) # unnamed - part of the formula - else: - out.extend((line, role, offset + o) - for line, role, o in self.line(var, depth, own)) + elif isinstance(node, Restrict): + # ``line.at(track=...)`` reads that one track: this track's + # entries when it names this track, none when another. + if node.label == self.track: + stack.append((node.base, offset)) return out - def line(self, var: Any, depth: int, own: bool) -> List[_Shifted]: + def line(self, var: Any, depth: int, own: bool) -> _Reading: key = (id(var), own) - if key not in self.memo: - self.memo[key] = self._line(var, depth, own) - return self.memo[key] + reading = self.memo.get(key) + if self.kept is None: + if reading is None: + reading = self.memo[key] = self._line(var, depth, own) + return reading + if reading is not None and reading.too_deep: + if (reading.line is var and depth + reading.height > MAX_DEPTH + and self._holds(reading)): + raise _TooDeep + reading = None + if reading is None or reading.line is not var or not self._holds(reading): + if key in self.open: + raise _TooDeep # a line that reads itself + self.open.add(key) + try: + reading = self._line(var, depth, own) + except _TooDeep: + # Something below stops the walk MAX_DEPTH down: the line + # reads at least that far from here. Remember it, so the + # next walk meeting it as deep does not read down again. + self.memo[key] = _Reading.deeper_than(var, MAX_DEPTH - depth + 1) + raise + finally: + self.open.discard(key) + self.memo[key] = reading + self.kept.add(id(reading)) + if depth + reading.height > MAX_DEPTH: + raise _TooDeep + return reading + + def _holds(self, reading: _Reading) -> bool: + """Is ``reading`` still what its line reads? Its line, the lines + its formula names and the lines below hold the same state.""" + kept = self.kept + assert kept is not None + if id(reading) in kept: + return True + var = reading.line + if (getattr(var, "_value", None) is not reading.value + or getattr(var, "_expr", None) is not reading.expr + or getattr(var, "_role_kwargs", None) is not reading.kwargs): + return False + if reading.operand_state is not None: + operand = reading.operand + if ((getattr(operand, "_owner", None) is None) is not reading.operand_state[0] + or getattr(operand, "_expr", None) is not reading.operand_state[1]): + return False + for ref, inlined in reading.refs: + inner = getattr(ref, "_expr", None) + inline = getattr(ref, "_owner", None) is None and inner is not None + if (inner if inline else None) is not inlined: + return False + for part in reading.parts: + if not self._holds(part): + return False + kept.add(id(reading)) + return True - def _line(self, var: Any, depth: int, own: bool) -> List[_Shifted]: + def _line(self, var: Any, depth: int, own: bool) -> _Reading: from .expr import ListExpr from .tracks import TrackValues value = getattr(var, "_value", None) + expr = getattr(var, "_expr", None) + kwargs = getattr(var, "_role_kwargs", None) + reading = _Reading(var, value, expr, kwargs) if not isinstance(value, TrackValues): # A line without tracks is the same on every track: no actual # of its own - unless the expression was written for this # track, where it is what the actual is made of. if not own or not isinstance(value, list): - return [] - expr = getattr(var, "_expr", None) + return reading if expr is None: - return [(var, None, 0)] - return self.formula(expr, depth + 1, True) if depth < MAX_DEPTH else [] - role = self.track - if role not in value.roles or not isinstance(value[role], list): - return [] # a constant is no month's entry - authored = var.track_expr(role) - if isinstance(authored, ListExpr): - return [(var, role, 0)] # cell by cell: blank where nothing was - expr = authored if authored is not None else getattr(var, "_expr", None) - if expr is None or (authored is None and role in ( - getattr(var, "_role_kwargs", None) or {})): - return [(var, role, 0)] # entered: blank where nothing was + reading.found = [(var, None, 0)] + return reading + through, through_own = expr, True + else: + role = self.track + if role not in value.roles or not isinstance(value[role], list): + return reading # a constant is no month's entry + operand = (kwargs or {}).get(role) + if isinstance(operand, list): + # Entered as a list - of numbers, or a row of references + # (``track_expr`` is None or a list expression): cell by + # cell, blank where nothing was. Asking ``track_expr`` would + # only look the cells over. + reading.found = [(var, role, 0)] + return reading + if operand is not None: + # A line as the operand is a reference unless it is a typed + # value (no owner, no formula or a typed date). + reading.operand = operand + reading.operand_state = (getattr(operand, "_owner", None) is None, + getattr(operand, "_expr", None)) + authored = var.track_expr(role) + if isinstance(authored, ListExpr): + reading.found = [(var, role, 0)] + return reading # cell by cell: blank where nothing was + through = authored if authored is not None else expr + if through is None or (authored is None and role in (kwargs or {})): + reading.found = [(var, role, 0)] + return reading # entered: blank where nothing was + through_own = authored is not None if depth >= MAX_DEPTH: - return [] - return self.formula(expr, depth + 1, authored is not None) + if self.kept is not None: + raise _TooDeep + return reading + reading.found = self.formula(through, depth + 1, through_own, + reading if self.kept is not None else None) + reading.height = 1 + max((part.height for part in reading.parts), default=0) + return reading diff --git a/src/modeleon/core/expr.py b/src/modeleon/core/expr.py index a6bfd43..342fbcd 100644 --- a/src/modeleon/core/expr.py +++ b/src/modeleon/core/expr.py @@ -31,7 +31,7 @@ from contextvars import ContextVar from dataclasses import dataclass, field -from typing import Any, Dict, Iterator, List, Optional, TYPE_CHECKING, Union +from typing import Any, Dict, Iterator, List, Optional, Set, TYPE_CHECKING, Union if TYPE_CHECKING: from .variable import Variable @@ -40,9 +40,11 @@ # ─── Base ──────────────────────────────────────────────────────────── -#: Variables whose expression is being inlined into a formula string -#: right now — see :meth:`VarRef.to_string`. -_INLINING: ContextVar[frozenset] = ContextVar('modeleon_inlining', default=frozenset()) +#: Ids of the Variables whose expressions are being inlined into a +#: formula string right now — see :meth:`VarRef.to_string`. ``None`` +#: outside a render; the outermost inlining frame of a context owns the +#: set and clears the variable when it returns. +_INLINING: ContextVar[Optional[Set[int]]] = ContextVar('modeleon_inlining', default=None) class Expr: @@ -114,12 +116,16 @@ class VarRef(Expr): var: "Variable" def to_string(self) -> str: + var = self.var # Prefer python_name (set by ``parent.child = ...``). For unattached # Variables with an explicit display_name, snake-case the label so - # formulas read ``revenue * 0.6`` instead of ``v3 * 0.6``. - if self.var.python_name: - return self.var.python_name - explicit_label = getattr(self.var, '_display_name', None) + # formulas read ``revenue * 0.6`` instead of ``v3 * 0.6``. The + # field is read directly: the property is a plain accessor and + # this is the most visited line of a formula render. + name = var._python_name + if name: + return name + explicit_label = getattr(var, '_display_name', None) if explicit_label: from .humanize import identifier_from_label return identifier_from_label(explicit_label) @@ -131,22 +137,36 @@ def to_string(self) -> str: # ``(revenue * cogs_pct) * units`` render as their composite # algebra rather than an opaque floating id. Wrap in parens for # precedence safety — the inlined expression may itself be a BinOp. - inner_expr = getattr(self.var, '_expr', None) + inner_expr = getattr(var, '_expr', None) if isinstance(inner_expr, TimeRef): return inner_expr.to_string() if inner_expr is not None: # A row of a loop across periods reads itself through ``lag``; # once detached from its container it has no name to stop the - # inlining, so a second visit prints its id instead. - inlining = _INLINING.get() - if id(self.var) in inlining: - return self.var.path.leaf - token = _INLINING.set(inlining | {id(self.var)}) + # inlining, so a second visit prints its id instead. The ids + # being inlined live in one mutable set per context: the + # outermost frame creates it, every frame below adds its id on + # the way in and removes it on the way out, so at any moment + # the set holds exactly the frames on the stack — a chain of + # intermediates costs one ContextVar round trip instead of a + # fresh set and two ContextVar operations per level. The + # ContextVar keeps renders on other threads and tasks apart. + key = id(var) + active = _INLINING.get() + if active is None: + token = _INLINING.set({key}) + try: + return f"({inner_expr.to_string()})" + finally: + _INLINING.reset(token) + if key in active: + return var.path.leaf + active.add(key) try: return f"({inner_expr.to_string()})" finally: - _INLINING.reset(token) - return self.var.path.leaf + active.discard(key) + return var.path.leaf def iter_refs(self) -> Iterator["Variable"]: yield self.var @@ -265,7 +285,7 @@ class ListExpr(Expr): items: List[Expr] = field(default_factory=list) def to_string(self) -> str: - return f"[{', '.join(item.to_string() for item in self.items)}]" + return f"[{', '.join([item.to_string() for item in self.items])}]" def iter_refs(self) -> Iterator["Variable"]: for item in self.items: @@ -301,7 +321,7 @@ class FuncCall(Expr): compute_backends: Optional[frozenset] = None def to_string(self) -> str: - return f"{self.func}({', '.join(a.to_string() for a in self.args)})" + return f"{self.func}({', '.join([a.to_string() for a in self.args])})" def iter_refs(self) -> Iterator["Variable"]: for arg in self.args: diff --git a/src/modeleon/core/loops.py b/src/modeleon/core/loops.py index 28b1221..4d7a198 100644 --- a/src/modeleon/core/loops.py +++ b/src/modeleon/core/loops.py @@ -57,6 +57,7 @@ BinOp, Compare, Expr, FuncCall, ListExpr, Literal, MethodCall, Paren, SelfRef, Subscript, UnaryOp, VarRef, ) +from .unit import UNSETTLED class Dependents: @@ -89,17 +90,13 @@ def __reduce__(self) -> Any: return (Dependents, (list(self._items.values()),)) -#: Functions that map one period to one period, so the loop can call -#: them with this period's numbers. Everything else — ``SUM``, ``NPV``, -#: ``MAX`` over one operand, … — reads a whole series and cannot sit -#: inside the loop. ``MAX`` / ``MIN`` over two or more operands (a floor, -#: a cap) are per period as well; :func:`_check_elementwise` admits them -#: by their arity. -_ELEMENTWISE = frozenset({ - 'IF', 'AND', 'OR', 'NOT', 'ABS', 'ROUND', 'INT', 'MOD', 'CHOOSE', - 'ISBLANK', 'EDATE', 'EOMONTH', 'YEAR', 'MONTH', 'DAY', 'DATE', - 'DAYS360', 'TODAY', 'LEN', 'UPPER', 'LOWER', 'CONCAT', -}) +# Which functions the loop may call with this period's numbers is the +# function registry's to say (``modeleon.functions.registry.admits_in_loop``): +# the element-wise ones, and a call over two or more values that is one +# call per period — ``MAX`` / ``MIN`` over two or more operands (a floor, a +# cap), ``SUM(a, b)``, ``MEDIAN(a, b)``, ``PERCENTILE_INC([a, b, c], k)``. +# Everything else — ``NPV``, ``SUM`` or ``MAX`` over one operand, … — reads +# a whole series and cannot sit inside the loop. _BINOPS = { '+': operator.add, '-': operator.sub, '*': operator.mul, @@ -380,16 +377,26 @@ def _resolve(nodes: List[Any], horizon: Optional[int]) -> None: ) if periodic: - order = _order(periodic, same) + order = _order([v for v in nodes if v in periodic], same) n = horizon or _horizon_of(order) _evaluate_periodic(order, n) for v in order: - v._awaits = None v._loop_error = None # The loop's own units: its rows were built on placeholders, # which carry none, so they are derived again from the formulas — - # to a fixed point, as the rows depend on each other. + # to a fixed point, as the rows depend on each other. The rows + # wait until their units settle: a loop refused here leaves them + # without values, its refusal on each (``_settle_nodes``). _rederive_units(order) + # The check + makes of a count added to a share waited while the + # loop was open; it is made on the settled lines. + for v in order: + try: + _check_counts(v, order) + except ValueError as exc: + raise ValueError(f"{_label(v)}: {exc}") from exc + for v in order: + v._awaits = None if downstream: deps = {v: same[v] + lagged[v] + led[v] for v in downstream} # A downstream temporary that no row holds (an expression built and @@ -406,9 +413,11 @@ def _resolve(nodes: List[Any], horizon: Optional[int]) -> None: if r not in held: held.add(r) stack.append(r) - for v in _order(set(downstream), deps): + for v in _order(downstream, deps): try: - value, unit = _eval_row(v._expr) + declared = v._unit if getattr(v, '_unit_declared', False) else None + value, unit = _eval_row(v._expr, declared) + _check_counts(v) except (ValueError, TypeError) as exc: if v in held: raise ValueError(f"{_label(v)}: {exc}") from exc @@ -424,69 +433,290 @@ def _resolve(nodes: List[Any], horizon: Optional[int]) -> None: # ─── Units ───────────────────────────────────────────────────────── -def _unit_of(node: Any) -> Any: +#: No row is being derived: every unit read is the row's own. +_SETTLED: frozenset = frozenset() + + +def _unit_of(node: Any, unsettled: Any = _SETTLED) -> Any: """The unit a formula tree produces, by the rules the operators use (``_unit_add`` / ``_unit_mul`` / …) — so a mix that the operators - refuse is refused here with the same message.""" + refuse is refused here with the same message. While a loop settles, + a row in ``unsettled`` (ids) reads as :data:`UNSETTLED`: ``+`` takes + the other side's unit, a product or a power of it stays unsettled, + and a comparison with it waits.""" from .variable_ops import _VariableArithmetic as arith + def unit_of(n: Any) -> Any: + return _unit_of(n, unsettled) + if isinstance(node, Literal): return None if isinstance(node, VarRef): - return getattr(node.var, '_unit', None) + if id(node.var) in unsettled: + return UNSETTLED + return getattr(_settled(node.var), '_unit', None) if isinstance(node, Paren): - return _unit_of(node.inner) + return unit_of(node.inner) if isinstance(node, UnaryOp): - return _unit_of(node.operand) + return unit_of(node.operand) if isinstance(node, BinOp): - left, right = _unit_of(node.left), _unit_of(node.right) - if node.op in ('+', '-'): - return arith._unit_add(left, right) + left, right = unit_of(node.left), unit_of(node.right) + if node.op in ('+', '-', '%'): + if left is UNSETTLED or right is UNSETTLED: + known = right if left is UNSETTLED else left + return UNSETTLED if known is None or known is UNSETTLED else known + return (arith._unit_mod if node.op == '%' else arith._unit_add)(left, right) + if left is UNSETTLED: + return UNSETTLED + if node.op == '**': + exponent = node.right.value if isinstance(node.right, Literal) else None + return arith._unit_pow(left, exponent) + if right is UNSETTLED: + return UNSETTLED if node.op == '*': return arith._unit_mul(left, right) if node.op in ('/', '//'): return arith._unit_div(left, right) - # ``%`` and ``**`` track no unit on the operators either. return None if isinstance(node, Compare): + left, right = unit_of(node.left), unit_of(node.right) + if (left is not None and right is not None and UNSETTLED not in (left, right) + and left != right): + from .variable_ops import _incompatible_units + raise ValueError(_incompatible_units(left, right).replace( + "Cannot add/subtract", "Cannot compare", 1)) return None if isinstance(node, FuncCall): - from ..functions._helpers import _UNITLESS_RESULTS, _shared_arg_unit - if node.func in _UNITLESS_RESULTS: + # The function's own law, the one its result was built by; an + # operand read here is often last period's value (a shift), not a + # line, so its unit comes from its formula, as for ``+``. + from ..functions._helpers import function_unit + return function_unit(node.func, node.args, unit_of, + own_law=node.render_backends != frozenset()) + if isinstance(node, MethodCall) and node.method == 'shift': + # A lag keeps its source's unit; its first cells hold the fill, so a + # fill in a unit is the source's (a pick, as ``mo.lag`` builds it), + # and a source not derived yet takes the fill's. + if unit_of(node.base) is None: return None - return _shared_arg_unit(node.args) - if isinstance(node, MethodCall) and node.method in ('shift', 'copy'): - return _unit_of(node.base) + from ..functions._helpers import same_unit + fill = node.kwargs.get('fill_value') + return same_unit('lag', [node.base, *([fill] if isinstance(fill, Expr) else [])], + unit_of) + if isinstance(node, MethodCall) and node.method == 'copy': + return unit_of(node.base) + if isinstance(node, Subscript): + # A period, a key or a slice of a line is in the line's unit, as + # ``line[0]`` builds it. + return unit_of(node.base) + if isinstance(node, SelfRef): + # A recurrence is in the unit its template gives from its start, + # by the one rule ``mo.recurrence`` builds it by - read here with + # the rows not settled yet; a declared unit= is the row's own, and + # the row's dimension is checked where that unit is known. + from ..functions.recurrence import template_unit + return template_unit(node, unit_of(node.start), None, + unsettled=unsettled, check_dimension=False) return None def _rederive_units(nodes: List[Any]) -> None: """Units were derived while the loop's rows were placeholders, which - carry none. Re-derive them from the final formulas — to a fixed - point, because a loop's units depend on each other the way its - values do. A unit the author declared stays as declared: a label on - a pure number does not survive a product, so it cannot be derived.""" + carry none. Derive them again from the final formulas, to a fixed + point, as the rows depend on each other: every row starts unsettled, + so none takes a unit from a placeholder — ``+`` gives the other + side's unit, a product waits. Rows nothing in the loop settles (a + flag, an index, a discount factor) are pure numbers, as their + placeholders were: read so, the derivation goes on, and a product + with one takes its other factor's unit. Then every row's formula is + checked with the settled units, and must give the unit its row + holds. A unit the author declared stays as declared: a label on a + pure number does not survive a product, so it cannot be derived.""" + rows = [v for v in nodes if getattr(v, '_expr', None) is not None] + declared = {id(v) for v in rows if getattr(v, '_unit_declared', False)} + unsettled = {id(v) for v in rows if id(v) not in declared} + _settle(rows, declared, unsettled) + if unsettled: + seeded = set(unsettled) + for v in rows: + if id(v) in unsettled: + v._unit = None + unsettled.clear() + _settle(rows, declared, unsettled, seeded) + # Every row's formula is checked - a declared unit decides what the + # row prints, never what its formula may add. + units: Dict[int, Any] = {} + failed: List[Tuple[Any, ValueError]] = [] + for v in rows: + try: + units[id(v)] = _unit_of(v._expr) + except ValueError as exc: + failed.append((v, exc)) + if failed: + v, mix = _the_mix(rows, failed) + raise ValueError(f"{_label(v)}: {mix}") from mix + for v in rows: + unit = units[id(v)] + if id(v) in declared or _dimension(unit) == _dimension(v._unit): + continue + raise ValueError( + f"{_label(v)}: its unit does not settle - read through mo.lag, its " + f"formula gives {unit or 'no unit'} from a row in " + f"{v._unit or 'no unit'}. A row multiplied by a dimension every " + f"period grows a power of it: give the factor no unit, or " + f"declare the row's unit (.set_unit(...))." + ) + + +def _the_mix(rows: List[Any], failed: List[Tuple[Any, ValueError]]) -> Tuple[Any, ValueError]: + """The failing row whose own formula mixes units. A row that fails + keeps the unit an earlier pass gave it, and a row that reads it - + through mo.lag it may be checked first - fails on that unit: read + every failing row, and what takes its unit from one, as not settled, + and the mix is the row that still fails.""" + unsettled = {id(v) for v, _exc in failed} + grew = True + while grew: + grew = False + for v in rows: + if id(v) in unsettled: + continue + try: + waits = _unit_of(v._expr, unsettled) is UNSETTLED + except ValueError: + waits = False + if waits: + unsettled.add(id(v)) + grew = True + for v, _exc in failed: + try: + _unit_of(v._expr, unsettled - {id(v)}) + except ValueError as exc: + return v, exc + return failed[0] + + +def _check_counts(v: Any, loop: Optional[List[Any]] = None) -> None: + """The check ``+`` makes of a bps or pp count added to a % fraction + (:func:`~modeleon.core.variable_ops._refuse_a_count_added_to_a_share`), + which waits while an operand's loop is open, made on the settled + lines: over a row's ``+`` / ``-``, and over the lines a function adds, + compares or picks (``SUM``, ``MAX``, ``IF``, ``CHOOSE``, …). A side the + row itself feeds - its row read through mo.lag - holds the row's + numbers from the second period on, so it is judged by its first, the + one the row has not written into.""" + from .variable_ops import (_numbers, _refuse_a_count_added_to_a_share, + _refuse_counts_among_shares) + expr = v._expr + where = None + pairs: List[Tuple[Any, Any]] = [] + if isinstance(expr, BinOp) and expr.op in ('+', '-'): + operands = [expr.left, expr.right] + elif isinstance(expr, FuncCall): + from ..functions._helpers import _SAME_UNIT_FROM + start = _SAME_UNIT_FROM.get(expr.func) + if start is None: + return + operands, where = list(expr.args[start:]), f"mo.{expr.func}" + elif isinstance(expr, MethodCall) and expr.method == 'shift': + # A lag's fill is its series' first cell: a count beside a fraction. + fill = expr.kwargs.get('fill_value') + operands, where = [expr.base, *([fill] if isinstance(fill, VarRef) else [])], "mo.lag" + elif isinstance(expr, SelfRef): + # The template's own + and -, {prev} read as the start. + from ..functions.recurrence import template_count_pairs + start = _settled(expr.start.var) if isinstance(expr.start, VarRef) else None + pairs = [(_settled(a), _settled(b)) for a, b in template_count_pairs(expr, start)] + operands = [] + else: + return + sides = [] + for node in operands: + while isinstance(node, Paren): + node = node.inner + if isinstance(node, VarRef): + sides.append(_settled(node.var)) + elif where is None: + return + fed: List[Set[int]] = [] + + def numbers(side: Any) -> List[Any]: + value = side._value + if loop is not None and isinstance(value, list): + if not fed: + fed.append({id(r) for r in _dependents_of(v, loop)}) + if id(side) in fed[0]: + value = value[:1] + return _numbers(value) + + for a, b in pairs: + _refuse_a_count_added_to_a_share(a, b, numbers, where=where) + _refuse_counts_among_shares(sides, numbers, where=where) + + +def _filled_source(v: Any, by_id: Dict[int, Any], unsettled: Set[int], + seeded: Any, declared: Set[int]) -> Optional[Tuple[Any, Any]]: + """``(row, unit)`` when ``v`` lags a row of this loop that no formula has + settled yet (or only as a pure number for want of anything else) with + a fill whose unit is known: the row is in the fill's unit.""" + expr = v._expr + if not (isinstance(expr, MethodCall) and expr.method == 'shift' + and isinstance(expr.base, VarRef)): + return None + row = by_id.get(id(expr.base.var)) + if row is None or id(row) in declared: + return None + if id(row) not in unsettled and not (id(row) in seeded and row._unit is None): + return None + fill = expr.kwargs.get('fill_value') + if not isinstance(fill, VarRef) or id(fill.var) in unsettled: + return None + unit = getattr(_settled(fill.var), '_unit', None) + return None if unit is None else (row, unit) + + +def _dimension(unit: Any) -> Dict[str, int]: + """What a unit measures, its label aside: none is a pure number.""" + return {} if unit is None else unit._unit_components + + +def _settle(rows: List[Any], declared: Set[int], unsettled: Set[int], + seeded: Set[int] | frozenset = _SETTLED) -> None: + """Derive the rows' units to a fixed point, reading a row in + ``unsettled`` as :data:`UNSETTLED` until its formula settles it; a + mismatch on the way is left for the check on settled units. A row + read through ``mo.lag`` with a fill in a unit is in that unit — the + lag's cells are the fill and the row — so the fill settles it, as + it does a row ``seeded`` as a pure number for want of anything else.""" from .unit import unit_key def key(u: Any) -> Any: return None if u is None else unit_key(u) - rows = [v for v in nodes if getattr(v, '_expr', None) is not None] - declared = {id(v) for v in rows if getattr(v, '_unit_declared', False)} + by_id = {id(v): v for v in rows} for _pass in range(len(rows) + 1): changed = False for v in rows: - # Every row's formula is checked - a declared unit decides what - # the row prints, never what its formula may add. - try: - unit = _unit_of(v._expr) - except ValueError as exc: - raise ValueError(f"{_label(v)}: {exc}") from exc + source = _filled_source(v, by_id, unsettled, seeded, declared) + if source is not None: + row, unit = source + unsettled.discard(id(row)) + if isinstance(seeded, set): + seeded.discard(id(row)) + row._unit = unit + changed = True if id(v) in declared: continue + try: + unit = _unit_of(v._expr, unsettled) + except ValueError: + continue + if unit is UNSETTLED: + continue # By key, not equality: a pure number's label is part of what # the row prints, though every pure number equals every other. - if key(unit) != key(v._unit): + if id(v) in unsettled or key(unit) != key(v._unit): + unsettled.discard(id(v)) v._unit = unit changed = True if not changed: @@ -554,11 +784,14 @@ def strongconnect(root: Any) -> None: return cyclic -def _order(nodes: Set[Any], deps: Dict[Any, List[Any]]) -> List[Any]: - """Topological order over ``deps`` (edges restricted to ``nodes``); - a cycle that remains is refused by name.""" +def _order(nodes: List[Any], deps: Dict[Any, List[Any]]) -> List[Any]: + """Topological order over ``deps`` (edges restricted to ``nodes``), + rows that do not wait on each other kept in the order given — the + order they were built in, never their addresses; a cycle that + remains is refused by name.""" + members: Set[Any] = set(nodes) pending: Dict[Any, Set[Any]] = { - v: {d for d in deps[v] if d in nodes and d is not v} for v in nodes + v: {d for d in deps[v] if d in members and d is not v} for v in nodes } self_reads = [v for v in nodes if any(d is v for d in deps[v])] if self_reads: @@ -618,6 +851,7 @@ def _evaluate_periodic(order: List[Any], n: int) -> None: for v in order: _check_elementwise(v) values: Dict[Any, List[Any]] = {v: [None] * n for v in order} + _read_outside_by_date(order, values, n) for t in range(n): for v in order: values[v][t] = _eval_at(v._expr, t, values, n) @@ -625,6 +859,39 @@ def _evaluate_periodic(order: List[Any], n: int) -> None: _write_back(v, values[v]) +def _read_outside_by_date(order: List[Any], values: Dict[Any, List[Any]], n: int) -> None: + """The lines outside the loop its rows read that run on windows of + their own, each materialized onto the loop's window by date + (:func:`_on_loop_window`) — as the operators read them when the rows + were built — so every period reads the line's value of its date; a + line read as it is (on the loop's window, a constant) is left to + :func:`_outside_at`.""" + home = None + for v in order: + home = v.__dict__.get('_owner') or v.__dict__.get('_parent') + if home is not None: + break + from .time import resolve_default_window + window = resolve_default_window(home) if home is not None else None + if window is None or window.grain is None or window.start is None: + return + from .variable import TimeLoc + from .variable_ops import _on_loop_window + loop = TimeLoc(window.start, window.grain, window.first_day) + seen: Set[int] = set() + for v in order: + for ref in v._expr.iter_refs(): + if ref in values or id(ref) in seen: + continue + seen.add(id(ref)) + x = _settled(ref)._value + if not isinstance(x, list): + continue + aligned = _on_loop_window(ref, x, v, loop, n) + if aligned is not None: + values[ref] = aligned + + def _check_elementwise(v: Any) -> None: expr = v._expr @@ -639,8 +906,8 @@ def walk(node: Any) -> None: elif isinstance(node, UnaryOp): walk(node.operand) elif isinstance(node, FuncCall): - per_period_pair = node.func in ('MAX', 'MIN') and len(node.args) >= 2 - if node.func not in _ELEMENTWISE and not per_period_pair: + from ..functions.registry import admits_in_loop + if not admits_in_loop(node.func, _values_per_period(node)): from .. import functions as fns if not hasattr(fns, node.func): raise ValueError( @@ -648,17 +915,16 @@ def walk(node: Any) -> None: f"and {node.func} cannot be run one period at a time. " f"Build that row after the loop closes." ) - hint = "" - if node.func in ('MAX', 'MIN'): - hint = (" A per-period floor or cap is mo.MAX(x, 0) / " - "mo.MIN(x, cap): two or more operands.") raise ValueError( f"{_label(v)} applies {node.func} to a row of the loop. " f"{node.func} reads a whole series, and inside the loop " - f"the series is still being computed.{hint}" + f"the series is still being computed.{_per_period_hint(node.func)}" ) - for arg in node.args: - if isinstance(arg, Expr): + for i, arg in enumerate(node.args): + if isinstance(arg, ListExpr) and _reads_range(node.func, i): + for item in arg.items: + walk(item) + elif isinstance(arg, Expr): walk(arg) elif isinstance(node, MethodCall): if node.method not in ('shift', 'copy'): @@ -679,9 +945,55 @@ def walk(node: Any) -> None: walk(expr) +def _reads_range(func: str, i: int) -> bool: + """Is argument ``i`` of ``func`` read as a range — every argument of a + reduction, the positions the registry names of a range function?""" + from ..functions.registry import is_range_taking, range_positions + positions = range_positions(func) + return is_range_taking(func) and (positions is None or i in positions) + + +def _values_per_period(call: FuncCall) -> int: + """The values a call combines, the count the registry admits it by: + the arguments it reads as ranges, a list of rows there counting each + row — or, reading none, its operands.""" + ranged = [a for i, a in enumerate(call.args) if _reads_range(call.func, i)] + if not ranged: + return len(call.args) + return sum(len(a.items) if isinstance(a, ListExpr) else 1 for a in ranged) + + +def _per_period_hint(func: str) -> str: + """How ``func`` is one call per period, when the registry says it can + be.""" + from ..functions.registry import per_period_arity, range_positions, spec + s = spec(func) + if s is None or per_period_arity(func) is None: + return "" + if s.shape == 'reduction': + hint = f" Over two or more operands mo.{func} is one call per period, mo.{func}(a, b)" + if func in ('MAX', 'MIN'): + hint += "; a per-period floor or cap is mo.MAX(x, 0) / mo.MIN(x, cap)" + return hint + "." + rows = "[a, b]" + shown = f"{rows}, …" if (range_positions(func) or (0,))[0] == 0 else f"…, {rows}" + return (f" Over a list of two or more rows mo.{func} is one call per period, " + f"over each period's cells: mo.{func}({shown}).") + + +def _settled(var: Any) -> Any: + """A Variable the loop reads as settled. One whose own recompute + failed (a temporary no row held when its loop closed) keeps its + error, never its placeholder numbers: its reader is refused with it.""" + error = getattr(var, '_loop_error', None) + if error: + raise ValueError(error) + return var + + def _outside_at(var: Any, t: int, n: int) -> Any: """A settled Variable's value at period ``t``.""" - x = var._value + x = _settled(var)._value from .tracks import TrackValues if isinstance(x, TrackValues): raise ValueError( @@ -751,11 +1063,13 @@ def _eval_at(node: Any, t: int, values: Dict[Any, List[Any]], n: int) -> Any: left, right, _COMPARES[node.op], op_symbol=node.op, ) if isinstance(node, FuncCall): - args = [ - _eval_at(a, t, values, n) if isinstance(a, Expr) else a - for a in node.args - ] - out = _function(node.func)(*args) + from ..functions.registry import is_range_taking + if is_range_taking(node.func): + args = [_cell_at(a, t, values, n) for a in node.args] + else: + args = [_eval_at(a, t, values, n) if isinstance(a, Expr) else a + for a in node.args] + out = _function(node)(*args) return out._value if isinstance(out, Variable) else out if isinstance(node, MethodCall): if node.method == 'copy': @@ -775,7 +1089,7 @@ def _eval_at(node: Any, t: int, values: Dict[Any, List[Any]], n: int) -> Any: if var in values: got = values[var][idx] if 0 <= idx < n else fill else: - x = var._value + x = _settled(var)._value if isinstance(x, list): got = x[idx] if 0 <= idx < len(x) else fill else: @@ -789,17 +1103,33 @@ def _eval_at(node: Any, t: int, values: Dict[Any, List[Any]], n: int) -> Any: raise ValueError(f"{type(node).__name__} expressions are not supported inside a loop yet.") +def _cell_at(node: Any, t: int, values: Dict[Any, List[Any]], n: int) -> Any: + """What a call that reads ranges reads of an operand at period ``t``: a + row as its cell, a list of rows as the range of their cells. A number + or a date reads the same typed into the call or in a cell; a blank, + text, a boolean or an error is handed over as a line of that one value, + so it is read as the workbook reads the cell — never as a value typed + in (a blank as 0, an error as ``#VALUE!``).""" + from .variable import Variable + if isinstance(node, VarRef): + value = _eval_at(node, t, values, n) + return Variable(value) if value is None or isinstance(value, (str, bool)) else value + if isinstance(node, ListExpr): + return tuple(_cell_at(item, t, values, n) for item in node.items) + return _eval_at(node, t, values, n) if isinstance(node, Expr) else node + + # ─── Whole-series evaluation, for the rows downstream of a loop ───── -def _eval_whole(node: Any) -> Any: +def _eval_whole(node: Any, declared_unit: Any = None) -> Any: from .variable import Variable from .variable_ops import _VariableArithmetic as arith if isinstance(node, Literal): return node.value if isinstance(node, VarRef): - return node.var._value + return _settled(node.var)._value if isinstance(node, Subscript): base = _eval_whole(node.base) key = node.key @@ -831,8 +1161,7 @@ def _eval_whole(node: Any) -> Any: items = [_eval_whole(item) for item in node.items] return [x[0] if isinstance(x, list) and len(x) == 1 else x for x in items] if isinstance(node, FuncCall): - args = [_as_argument(a) for a in node.args] - out = _function(node.func)(*args) + out = _function(node)(*_call_arguments(node)) return out._value if isinstance(out, Variable) else out if isinstance(node, MethodCall): if node.method == 'copy': @@ -860,8 +1189,11 @@ def _eval_whole(node: Any) -> Any: periods = periods.var elif isinstance(periods, Expr): periods = _eval_whole(periods) + # Run again as the row was written: a declared unit= rides along, + # so the template's dimension check reads it as it did at first. + line = {'unit': declared_unit} if declared_unit is not None else {} return recurrence( - _as_argument(node.start), node.template, periods=periods, **variables, + _as_argument(node.start), node.template, periods=periods, **variables, **line, )._value raise ValueError( f"{type(node).__name__} expressions over a row of a loop are not " @@ -869,10 +1201,13 @@ def _eval_whole(node: Any) -> Any: ) -def _function(name: str) -> Any: - """The ``mo.*`` function behind a formula's function name.""" +def _function(call: FuncCall) -> Any: + """The ``mo.*`` function behind a formula's call. A call native to no + renderer (a ``pyformula``) is not the engine's function of its name, + whatever the name, and cannot be run again.""" from .. import functions as fns - fn = getattr(fns, name, None) + name = call.func + fn = None if call.render_backends == frozenset() else getattr(fns, name, None) if fn is None: raise ValueError( f"{name}(...) cannot be computed again when the loop closes — " @@ -882,20 +1217,34 @@ def _function(name: str) -> Any: return fn -def _eval_row(expr: Any) -> Tuple[Any, Any]: - """A downstream row's values and unit. A function or a recurrence at - the top is run again as itself, so its unit is the one it always - gives; arithmetic follows the operators' unit rules.""" +def _call_arguments(call: FuncCall) -> List[Any]: + """A formula call's arguments as its ``mo.*`` function takes them + (:func:`_as_argument`). An operator joined to a cell - the + ``CONCAT('>=', cap)`` a ``('>=', cap)`` criteria is written as - goes + back as that pair, so the function reads the criteria as it was given: + the cell's unit is compared with its range's and an error the cell + holds is the result, never the text the join gives.""" + from ..functions.criteria import joined_criteria + out: List[Any] = [] + for arg in call.args: + pair = joined_criteria(arg) + out.append(_as_argument(arg) if pair is None + else (pair[0], _as_argument(pair[1]))) + return out + + +def _eval_row(expr: Any, declared_unit: Any = None) -> Tuple[Any, Any]: + """A downstream row's values and unit. A function at the top is run + again as itself, so its unit is the one it always gives; a recurrence, + a subscript and arithmetic follow the rules :func:`_unit_of` reads + them by.""" from .variable import Variable if isinstance(expr, FuncCall): - out = _function(expr.func)(*[_as_argument(a) for a in expr.args]) + out = _function(expr)(*_call_arguments(expr)) if isinstance(out, Variable): return out._value, out._unit return out, None - if isinstance(expr, SelfRef): - value = _eval_whole(expr) - return value, None - return _eval_whole(expr), _unit_of(expr) + return _eval_whole(expr, declared_unit), _unit_of(expr) def _as_argument(node: Any) -> Any: @@ -903,9 +1252,13 @@ def _as_argument(node: Any) -> Any: Variable itself for a reference (so the function sees its unit and shape), the constant for a literal.""" if isinstance(node, VarRef): - return node.var + return _settled(node.var) if isinstance(node, Literal): return node.value + if isinstance(node, ListExpr) and all(isinstance(i, (VarRef, Literal)) for i in node.items): + # A list of lines and values, as the call was given it: the lines + # themselves (their units, their cells), never their values. + return [_as_argument(item) for item in node.items] if isinstance(node, Expr): return _eval_whole(node) return node @@ -913,12 +1266,16 @@ def _as_argument(node: Any) -> Any: def _write_back(v: Any, value: Any) -> None: v._value = value + # Typed by the first cell that holds a value: the placeholder build + # may have typed the row by an error marker it no longer holds. + cells = value if isinstance(value, list) else [value] if isinstance(value, list): v.var_type = 'list' - sample = value[0] if value else None - else: - sample = value + sample = next((c for c in cells if c is not None + and not (isinstance(c, str) and c.startswith('#'))), None) if isinstance(sample, bool): v.value_type = 'bool' elif isinstance(sample, (date, datetime)): v.value_type = 'datetime' + elif isinstance(sample, (int, float)) and v.value_type not in ('int', 'float'): + v.value_type = 'float' diff --git a/src/modeleon/core/multi_variable.py b/src/modeleon/core/multi_variable.py index 602e661..a76853c 100644 --- a/src/modeleon/core/multi_variable.py +++ b/src/modeleon/core/multi_variable.py @@ -53,7 +53,7 @@ def compute(self, start_users=100, churn=0.05): """ from datetime import date -from typing import Dict, List, Optional, Any, TYPE_CHECKING +from typing import Dict, Iterable, List, Optional, Any, Set, Tuple, TYPE_CHECKING import contextvars import inspect import weakref @@ -138,6 +138,76 @@ def _signature_of(compute_func: Any) -> "inspect.Signature": return inspect.signature(compute_func) +#: The slots that declare a container's window. +_WINDOW_SLOTS = frozenset(('default_grain', 'default_start', 'default_periods', 'default_zone')) + +#: Below this many names a list is scanned; from it on, looked up. +_SCANNED_NAMES = 16 + + +class _ComponentNames(list[str]): + """A container's component names in order: a list whose ``in`` is a + set lookup once it is long. Every child adopted asks whether its name + is taken, and a scan of the list made filling a container quadratic + in its size. Every change to the list brings the set up to date — + ``append`` adds its name, any other change builds the set again — so + ``in`` only reads it.""" + + __slots__ = ('_lookup',) + + def __init__(self, names: Iterable[str] = ()) -> None: + super().__init__(names) + self._lookup: Optional[Set[str]] = None + self._index() + + def _index(self) -> None: + try: + self._lookup = set(self) if len(self) >= _SCANNED_NAMES else None + except TypeError: + # An unhashable name: the list is scanned, as a plain list is. + self._lookup = None + + def __reduce__(self) -> Tuple[Any, ...]: + # A copy (or a pickle) is the same names in a new list. + return (type(self), (list(self),)) + + def __contains__(self, name: object) -> bool: + lookup = self._lookup + if lookup is not None: + try: + return name in lookup + except TypeError: + pass + return list.__contains__(self, name) + + def append(self, name: str) -> None: + super().append(name) + lookup = self._lookup + if lookup is None: + if len(self) == _SCANNED_NAMES: + self._index() + else: + try: + lookup.add(name) + except TypeError: + self._lookup = None + + +def _indexes_after(change: Any) -> Any: + def changed(self: _ComponentNames, *args: Any) -> Any: + result = change(self, *args) + self._index() + return result + changed.__name__ = change.__name__ + return changed + + +for _change in ('extend', 'insert', 'remove', 'pop', 'clear', + '__setitem__', '__delitem__', '__iadd__', '__imul__'): + setattr(_ComponentNames, _change, _indexes_after(getattr(list, _change))) +del _change + + class MultiVariableBase(_MVLifecycle, _MVInspect, Component): """ Base class with internal machinery for grouped Variables and nested MultiVariables. @@ -258,7 +328,7 @@ def __init__( # disambiguating from private state attrs (``_parent``, # ``_qualified_id``, etc.) and from Variable IDs whose binding # names start with underscore (floating qpaths). - self._component_names: List[str] = [] + self._component_names: List[str] = _ComponentNames() # Layout role is a derived internal marker. Users set # ``excel_props={'tab': True}`` to mark an Excel tab; the writer @@ -277,21 +347,25 @@ def _set_default_window( default_grain: Optional[str] = None, default_start: Optional[str] = None, default_periods: Optional[Any] = None, + default_zone: Optional[str] = None, ) -> None: """Validate and set the ambient time window (``default_grain`` / - ``default_start`` / ``default_periods``) declared in a constructor. + ``default_start`` / ``default_periods`` / ``default_zone``) + declared in a constructor. The window is where a model's time lives: list-valued lines inherit their native ``(start, grain)`` from the nearest ancestor window (``Variable.time``), and every renderer's timeline header is derived from it (``resolve_default_window``). Plain attribute - assignment (``m.default_grain = 'month'``) remains valid and - unvalidated; this constructor path fails loudly on a bad window. + assignment (``m.default_grain = 'month'``) remains valid; the grain + is checked there too (``__setattr__``), the rest of the window only + on this constructor path, which fails loudly on a bad window. """ - from .time import GRAINS, _parse + from .time import GRAINS, WINDOW_MAX_PERIODS, SheetWidthError, _parse, width_refusal from .variable import Variable if default_grain is None: - given = 'default_start' if default_start is not None else 'default_periods' + given = ('default_start' if default_start is not None + else 'default_periods' if default_periods is not None else 'default_zone') raise ValueError( f"{given}= needs default_grain= beside it — the window's grain " f"is what anchors the start label and the period count " @@ -315,7 +389,11 @@ def _set_default_window( raise ValueError( f"default_periods must be positive; got {default_periods}." ) + if default_periods > WINDOW_MAX_PERIODS: + raise SheetWidthError(width_refusal(default_grain, default_periods)) self.default_grain = default_grain + if default_zone is not None: + self.default_zone = default_zone if default_start is not None: self.default_start = default_start if default_periods is not None: @@ -326,7 +404,8 @@ def _set_default_window( @property def _components(self) -> Dict[str, Any]: """Dict view of components — derived from ``__dict__`` indexed - by ``_component_names``. Read returns a fresh dict snapshot.""" + by ``_component_names``. Read returns a fresh dict snapshot, a + copy of every child: one child is read with ``_component(name)``.""" return {k: self.__dict__[k] for k in self._component_names if k in self.__dict__} @_components.setter @@ -337,7 +416,7 @@ def _components(self, mapping: Dict[str, Any]) -> None: current = list(self.__dict__.get('_component_names', [])) for name in current: self.__dict__.pop(name, None) - self.__dict__['_component_names'] = list(mapping.keys()) + self.__dict__['_component_names'] = _ComponentNames(mapping.keys()) for name, value in mapping.items(): self.__dict__[name] = value @@ -350,7 +429,7 @@ def _component_order(self) -> List[str]: def _component_order(self, order: List[str]) -> None: """Replace the component name order. Components themselves stay in ``__dict__``; only the order list is updated.""" - self.__dict__['_component_names'] = list(order) + self.__dict__['_component_names'] = _ComponentNames(order) @property def _is_sheet(self) -> bool: @@ -409,7 +488,29 @@ def __getattr__(self, name: str): if not is_engine_frame(frame): return _forward_placeholder( self, name, (frame.f_code.co_filename, frame.f_lineno)) - raise AttributeError(f"'{type(self).__name__}' has no component '{name}'") + missing = f"'{type(self).__name__}' has no component '{name}'" + if name in _SETTING_SLOTS: + # A setting of the container, never a row — and the engine's + # own probes (``.tracks``, ``.default_grain``, tens of thousands + # per workbook), which pay no frame walk. + raise AttributeError(missing) + caller = sys._getframe(1) + from .mv_context import is_engine_frame, spelled + if is_engine_frame(caller): + # The engine's own probe (``getattr(mv, name, None)``): no hint + # to build — the author never reads this message. + raise AttributeError(missing) + # The author's own name for the container (``model``, ``income``, + # ``model.pnl``) — never its declared or display name, which + # need not be Python. + holder = spelled(self, caller) or "" + raise AttributeError( + f"{missing}. Assign it before the line that reads it — " + f"`{holder}.{name} = mo.Variable(...)` for a row, " + f"`{holder}.{name} = mo.MultiVariable('{name}')` for a container; " + f"a row read before its own line belongs in a loop container " + f"(loop=True)." + ) def __setattr__(self, name: str, value: Any) -> None: """Auto-register Variables or nested MVs assigned as attributes. @@ -423,6 +524,25 @@ def __setattr__(self, name: str, value: Any) -> None: through normal attribute assignment. """ from .variable import Variable + if name == 'default_grain' and value is not None: + # The same gate as the constructor's: a grain the engine does + # not know must not sit on a model until a writer dies on it. + from .time import GRAINS + if value not in GRAINS: + raise ValueError( + f"Unknown default_grain {value!r}; supported: " + f"{', '.join(GRAINS)}." + ) + if name == 'default_zone' and value is not None: + # A fixed offset, spelled one way ('+01:00'); a named zone is + # refused with the offset to write instead. + from .time import zone_offset + value = zone_offset(value) + if name == 'default_periods' and isinstance(value, int) and not isinstance(value, bool): + from .time import WINDOW_MAX_PERIODS, SheetWidthError, width_refusal + if value > WINDOW_MAX_PERIODS: + raise SheetWidthError(width_refusal( + self.__dict__.get('default_grain') or 'period', value)) if name == 'default_grain' and '_default_start_source' in self.__dict__ \ and isinstance(value, str): from .time import window_start_label @@ -434,6 +554,28 @@ def __setattr__(self, name: str, value: Any) -> None: '_default_start_source'))): from .mv_context import refuse_window_change refuse_window_change(self, name) + if name == 'default_periods' and isinstance(value, int) and not isinstance(value, bool): + # A model given its horizon by assignment (``m.default_periods = + # 36``) is the model last given a horizon, as the constructor's + # is: a line reading ``mo.time`` outside its ``with`` block (a + # model written without one) finds the window there. Noted once the + # assignment is allowed (a refused change moves nothing), and + # not for the engine's own assignments (a projection builds + # models too). + from .mv_context import _note_window, is_engine_frame + if not is_engine_frame(sys._getframe(1)): + _note_window(self) + if name in _WINDOW_SLOTS and (self.__dict__.get('_component_names') + or self.__dict__.get('_forward') + or self.__dict__.get('_sliced')): + # The lines under this window are placed by it — a row read + # ahead of its line and a track slice too: a formula placed + # through them before the change is placed again. A container + # that holds none of them yet places nothing — a line adopted + # into it later is placed again on its own adoption — so the + # window every constructor sets moves nothing kept. + from .variable_ops import _windows_changed + _windows_changed() if name == 'default_start' and isinstance(value, (date, Variable)): self._set_default_start(value) return @@ -516,6 +658,13 @@ def __delattr__(self, name: str) -> None: self._detach_component(name) return super().__delattr__(name) + if name in _WINDOW_SLOTS and (self.__dict__.get('_component_names') + or self.__dict__.get('_forward') + or self.__dict__.get('_sliced')): + # Deleting a window slot changes the window as setting one + # does: a formula placed through these lines is placed again. + from .variable_ops import _windows_changed + _windows_changed() # ``python_name`` property + setter are inherited from :class:`Base`. @@ -544,7 +693,7 @@ def path(self) -> QPath: self._qualified_id = QPath.ROOT.child(self._python_name) from .mv_context import _crystallize_subtree for comp_name in self._component_order: - child = self._components[comp_name] + child = self._component(comp_name) _crystallize_subtree(child, self._qualified_id.child(comp_name)) return self._qualified_id return QPath.floating(id(self), kind="m") @@ -621,7 +770,7 @@ def remove(self, name: str) -> "Component": MV's components, returning the detached instance. Raises :class:`KeyError` if there is no such component. """ - if not isinstance(name, str) or name not in self._components: + if not isinstance(name, str) or not self._has_component(name): raise KeyError( f"{name!r} is not a component of this " f"{type(self).__name__}." @@ -883,7 +1032,7 @@ def __init__(self, display_name=None, **components): # by every list-valued line under it (``Variable.time``) and by the # Excel timeline header (``resolve_default_window``). window_kwargs: Dict[str, Any] = {} - for window_key in ("default_grain", "default_start", "default_periods"): + for window_key in ("default_grain", "default_start", "default_periods", "default_zone"): if window_key in components: window_kwargs[window_key] = components.pop(window_key) diff --git a/src/modeleon/core/mv_context.py b/src/modeleon/core/mv_context.py index f07b015..a8912e6 100644 --- a/src/modeleon/core/mv_context.py +++ b/src/modeleon/core/mv_context.py @@ -32,6 +32,7 @@ from __future__ import annotations import os +import sys import warnings import weakref from contextvars import ContextVar @@ -95,7 +96,7 @@ def _refuse_read_only_name(cls: type, name: str) -> None: #: Names that are settings of a container, never rows. _SETTING_SLOTS = frozenset({ - 'default_grain', 'default_start', 'default_periods', + 'default_grain', 'default_start', 'default_periods', 'default_zone', 'default_excel_view', 'default_header', 'tracks', }) @@ -266,6 +267,9 @@ def _fit_time(name: str, var: Any, window: Any, home: Any) -> None: int(window.periods), first_day) t._start, t._grain = window.start, window.grain t._first_day = first_day + # Its window moved: a formula placed over it is placed again. + from .variable_ops import _windows_changed + _windows_changed() continue def _from(start: Any, first: Any) -> str: return first.isoformat() if first is not None else str(start) @@ -354,6 +358,105 @@ def check_borrowed_horizon(parent: Any, component: Any) -> None: del component.__dict__['_borrowed_horizon'] +def author_frame() -> Any: + """The innermost frame that runs neither the engine's code nor a + trusted extension's: the line the author wrote.""" + frame: Any = sys._getframe(1) + while frame is not None and is_engine_frame(frame): + frame = frame.f_back + return frame + + +#: How many of the author's frames a hint may draw names from, and how +#: large a namespace it reads: a probe that throws the message away +#: (``getattr(mv, name, None)`` in a loop) must not pay for a scan of +#: tens of thousands of names. +_NAME_FRAMES = 8 +_NAME_SCAN_MAX = 2000 + + +def _names_by_object(frame: Any) -> dict: + """``id(object) -> the name it is bound to`` where the author's line + runs: the frame's locals, then its globals less the names its locals + shadow (inside ``def compare(model)`` the global ``model`` is not + reachable by that name), then the same for the frames that called it + while they are the author's own — a lambda or a nested function sees + the names of the line that called it. One pass over each namespace, + however many ancestors are looked up afterwards. Private names are + not spelled.""" + out: dict = {} + depth = 0 + while frame is not None and depth < _NAME_FRAMES and not is_engine_frame(frame): + local = frame.f_locals + if len(local) <= _NAME_SCAN_MAX: + for key, value in local.items(): + if key.isidentifier() and not key.startswith('_'): + out.setdefault(id(value), key) + globs = frame.f_globals + if globs is not local and len(globs) <= _NAME_SCAN_MAX: + for key, value in globs.items(): + if key in local or not key.isidentifier() or key.startswith('_'): + continue + out.setdefault(id(value), key) + frame = frame.f_back + depth += 1 + return out + + +def spelled(node: Any, frame: Any) -> Optional[str]: + """How the author's line reaches ``node``, spelled from the HIGHEST + bound ancestor and the path down: ``model.b`` for a container of + ``model`` even while a nearer name is bound to it, because the + nearer name may be a comprehension's or a lambda's variable + (``[seg.revenue for seg in (model.a, model.b)]``), gone before the + line the hint says to write; a name bound to the node itself when no + ancestor is bound (``income`` after ``with … as income:``); ``None`` + when nothing is. A hint spells the author's own names, never a + container's declared or display name — ``mo.Model('m')`` bound as + ``model`` is ``model`` — so following the hint runs.""" + if frame is None: + return None + names = _names_by_object(frame) + best: Optional[str] = None + path: list = [] + while node is not None: + name = names.get(id(node)) + if name is not None: + best = '.'.join([name, *reversed(path)]) + child = node.__dict__.get('_name_in_parent') + if not child: + break + path.append(child) + node = node.__dict__.get('_parent') + return best + + +def _refuse_self_adoption(parent: Any, name: str, component: Any) -> None: + """``parent. = component`` where ``component`` is ``parent`` + itself or one of its ancestors: refused with the alias to write.""" + node = parent + while node is not None: + if node is component: + break + node = node.__dict__.get('_parent') + else: + return + frame = author_frame() + holder = spelled(parent, frame) + what = spelled(component, frame) + holds = "itself" if component is parent else "one of its ancestors" + where = f"`{holder}.{name} = {what}`" if holder and what else f"Assigning `{name}`" + if what: + alias = f"with {what}:" if name == what else f"with {what} as {name}:" + hint = f"To write its rows under the name `{name}`, open a block: `{alias}`." + else: + hint = ( + f"To write a container's rows under the name `{name}`, open a block " + f"on it: `with as {name}:`." + ) + raise ValueError(f"{where}: a container cannot hold {holds}. {hint}") + + def _forward_placeholder(mv: Any, name: str, site: Any = None) -> Any: """The placeholder standing in for ``mv.`` until it is assigned — the same object on every read.""" @@ -455,6 +558,9 @@ def _transplant(target: Any, component: Any) -> None: keep = {k: target.__dict__[k] for k in ('_owner', '_component_name', '_python_name', '_qualified_id') if k in target.__dict__} + # The formulas placed through ``target`` while it had no window: they + # are placed again when the row it becomes is adopted. + readers = target.__dict__.get('_placed_through') state = dict(component.__dict__) if isinstance(state.get('_value'), list): # Its own list: an alias must not share the source row's values. @@ -464,6 +570,11 @@ def _transplant(target: Any, component: Any) -> None: target.__class__ = type(component) if keep.get('_owner') is not None: target.__dict__.update(keep) + if readers: + own = target.__dict__.get('_placed_through') + if own: + readers.extend(own) + target.__dict__['_placed_through'] = readers def _become_forward(placeholder: Any, component: Any) -> Any: @@ -544,11 +655,14 @@ def _crystallize_subtree(item: Any, path: QPath) -> None: the root adopts its subtree. """ from .multi_variable import MultiVariableBase + from .variable import Variable + if not isinstance(item, (MultiVariableBase, Variable)): + return # a plain value written over a line: no identity item._qualified_id = path if isinstance(item, MultiVariableBase): for comp_name in item._component_order: - child = item._components[comp_name] + child = item._component(comp_name) _crystallize_subtree(child, path.child(comp_name)) @@ -566,7 +680,12 @@ def _run_adoption_hooks(item: Any) -> None: if isinstance(item, MultiVariableBase): for comp_name in item._component_order: - _run_adoption_hooks(item._components[comp_name]) + _run_adoption_hooks(item._component(comp_name)) + # A row read ahead of its line runs on this window now as well. + for placeholder in (item.__dict__.get('_forward') or {}).values(): + if '_placed_through' in placeholder.__dict__: + from .variable_ops import _forget_placements + _forget_placements(placeholder) elif isinstance(item, Variable): item._on_adopted() @@ -580,11 +699,14 @@ def _invalidate_subtree_paths(item: Any) -> None: lets the lazy ``path`` property walk the owner chain on access. """ from .multi_variable import MultiVariableBase + from .variable import Variable + if not isinstance(item, (MultiVariableBase, Variable)): + return # a plain value written over a line: no identity item._qualified_id = None if isinstance(item, MultiVariableBase): for comp_name in item._component_order: - child = item._components[comp_name] + child = item._component(comp_name) _invalidate_subtree_paths(child) @@ -599,11 +721,17 @@ class _MVLifecycle: # Attributes provided by MultiVariableBase. _components: dict _component_order: list + _component_names: list _display_name: Optional[str] _python_name: Optional[str] _qualified_id: Any _role: str + if TYPE_CHECKING: + # Read helpers of :class:`~.mv_inspect._MVInspect`. + def _has_component(self, name: str) -> bool: ... + def _component(self, name: str) -> Any: ... + # ─── Context-manager protocol ─────────────────────────────── def __enter__(self) -> "MultiVariableBase": @@ -661,6 +789,15 @@ def _register_component(self, name: str, component: Any) -> None: # …) is refused before anything below changes state. _refuse_read_only_name(type(self), name) + # A container assigned under a second name inside its own tree — + # ``model.m = model``, ``model.sub.m = model`` — would adopt + # itself or an ancestor: the demotion below would make the model + # a MultiVariable and the path walk recurse until Python gives up + # (a RecursionError). A short name for a container is a + # ``with … as`` alias. + if isinstance(component, MultiVariableBase): + _refuse_self_adoption(self, name, component) + # Demote a Model to a plain MultiVariable on adoption — Models # are top-level roots by contract; once adopted they are sub- # trees. The class swap is layout-safe because Model adds no @@ -673,8 +810,8 @@ def _register_component(self, name: str, component: Any) -> None: if isinstance(component, Model): component.__class__ = MultiVariable # type: ignore[assignment] - if name in self._components: - existing = self._components[name] + if self._has_component(name): + existing = self._component(name) if existing is component: return # idempotent self-assignment @@ -792,17 +929,23 @@ def _detach_component(self, name: str) -> Any: from .multi_variable import MultiVariableBase from .variable import Variable - if name not in self._components: + if not self._has_component(name): return None - existing = self._components[name] + existing = self._component(name) if isinstance(existing, MultiVariableBase): existing._parent = None existing._name_in_parent = None elif isinstance(existing, Variable): existing._owner = None existing._component_name = None - existing._qualified_id = None - existing._python_name = None + if isinstance(existing, (MultiVariableBase, Variable)): + # A plain value written over a line has no identity to clear. + existing._qualified_id = None + existing._python_name = None + # Its lines leave the window they ran on: a formula placed + # through them is placed again (``variable_ops``). + from .variable_ops import _windows_changed + _windows_changed() self.__dict__.pop(name, None) if name in self._component_names: self._component_names.remove(name) @@ -834,7 +977,7 @@ def _clone(self) -> "MultiVariableBase": if '_loop' in self.__dict__: clone._loop = self._loop for comp_name in self._component_order: - comp = self._components[comp_name] + comp = self._component(comp_name) if isinstance(comp, Variable) and not isinstance(comp, MultiVariableBase): clone._register_component(comp_name, comp.copy()) elif isinstance(comp, MultiVariableBase): diff --git a/src/modeleon/core/mv_inspect.py b/src/modeleon/core/mv_inspect.py index c297630..8f2a999 100644 --- a/src/modeleon/core/mv_inspect.py +++ b/src/modeleon/core/mv_inspect.py @@ -35,6 +35,7 @@ class _MVInspect: # Attributes provided by MultiVariableBase. _components: Dict[str, Any] _component_order: List[str] + _component_names: List[str] _parent: Optional["MultiVariableBase"] _excel_props: Dict[str, Any] _qualified_id: Any @@ -57,6 +58,18 @@ def component_names(self) -> List[str]: """List of all component names in creation order.""" return self._component_order.copy() + def _has_component(self, name: str) -> bool: + """``name in self._components``, without building the dict.""" + return name in self._component_names and name in self.__dict__ + + def _component(self, name: str) -> Any: + """``self._components[name]``, without building the dict — a + copy of every child, once per child read, made each walk over a + container quadratic in its size.""" + if self._has_component(name): + return self.__dict__[name] + raise KeyError(name) + @property def sheet_name(self) -> Optional[str]: """Resolved sheet (Excel tab) name for sheet-role MVs. @@ -74,7 +87,7 @@ def walk(self): instances.""" from .multi_variable import MultiVariableBase for name in self._component_order: - component = self._components[name] + component = self._component(name) yield name, component if isinstance(component, MultiVariableBase): yield from component.walk() diff --git a/src/modeleon/core/projection.py b/src/modeleon/core/projection.py index e4dda5c..fc1ec0c 100644 --- a/src/modeleon/core/projection.py +++ b/src/modeleon/core/projection.py @@ -24,6 +24,7 @@ from __future__ import annotations +import math import operator from typing import Any, Dict, Optional @@ -41,10 +42,18 @@ '<': operator.lt, '<=': operator.le, '>': operator.gt, '>=': operator.ge} #: mo.* functions safe to re-run over re-grained inputs — Variable-aware, -#: element-wise or scalar-broadcasting, no self-reference. Reductions -#: (SUM/NPV/IRR/…) never reach recompute: scalar-valued formulas copy. +#: element-wise or scalar-broadcasting, no self-reference. Reductions over +#: one series (SUM/NPV/IRR/…) never reach recompute: scalar-valued formulas +#: copy. _RECOMPUTE_FUNCS = ('IF', 'AND', 'OR', 'NOT', 'CHOOSE', 'MAX', 'MIN', 'ABS', 'ROUND', 'INT', 'MOD') +#: Aggregates whose call over two or more operands with a series among +#: them is one per period, re-grained as their operators are: ``SUM`` and +#: ``AVERAGE`` as ``a + b`` — the call over the re-grained operands, each +#: by its own rule (a balance its last month, a flow its sum) — and +#: ``PRODUCT`` as ``a * b``, a flow evaluated at native grain when two of +#: its operands vary in time. +_PER_PERIOD_AGGREGATES = ('SUM', 'AVERAGE', 'PRODUCT') def _unsupported(what: str) -> str: @@ -77,6 +86,12 @@ def _recompute(expr: Any, grain: str, cache: Dict[int, Any]) -> Any: raise ValueError(_unsupported(f"operator {expr.op!r}")) return fn(_recompute(expr.left, grain, cache), _recompute(expr.right, grain, cache)) + if (isinstance(expr, FuncCall) and expr.func in _PER_PERIOD_AGGREGATES + and _per_period_call(expr)): + if expr.func == 'PRODUCT' and sum(map(_is_time_varying, expr.args)) >= 2: + return _flow_regrain(expr, grain) + from .. import functions as fns + return getattr(fns, expr.func)(*[_recompute(a, grain, cache) for a in expr.args]) if isinstance(expr, (Compare, FuncCall)) and ( not isinstance(expr, FuncCall) or expr.func in _RECOMPUTE_FUNCS): # NON-LINEAR forms (mo.IF — the tax line — comparisons, @@ -133,6 +148,16 @@ def _recompute(expr: Any, grain: str, cache: Dict[int, Any]) -> Any: raise ValueError(_unsupported(f"{type(expr).__name__} expressions")) +def _per_period_call(call: FuncCall) -> bool: + """A call over as many operands as its ``per_period_arity`` or more, a + series among them — one call per period.""" + from ..functions._helpers import series_node + from ..functions.registry import per_period_arity + arity = per_period_arity(call.func) + return (arity is not None and len(call.args) >= arity + and any(series_node(arg) for arg in call.args)) + + def _is_time_varying(expr: Any, _seen: Optional[set] = None) -> bool: """Does this subexpression reference any time-located Variable? @@ -219,7 +244,7 @@ def _native_eval(expr: Any) -> Any: if fn is None: raise ValueError(_unsupported(f"comparison {expr.op!r}")) return fn(_native_eval(expr.left), _native_eval(expr.right)) - if isinstance(expr, FuncCall) and expr.func in _RECOMPUTE_FUNCS: + if isinstance(expr, FuncCall) and expr.func in _RECOMPUTE_FUNCS + _PER_PERIOD_AGGREGATES: from .. import functions as _fns fn = getattr(_fns, expr.func) args = [ @@ -756,7 +781,16 @@ def _project_time_row(var: Any, loc: Any, grain: str) -> Variable: coarse_first = coarse_first_day(loc.start, loc.grain, first_day, grain) full = time_field_series('days', buckets[0][0], grain, len(buckets), coarse_first) native_days = time_field_series('days', loc.start, loc.grain, n, first_day) - if all(sum(native_days[lo:hi]) == full[k] for k, (_, lo, hi) in enumerate(buckets)): + def whole(k: int, lo: int, hi: int) -> bool: + got = sum(native_days[lo:hi]) + # An hour's 1/24 of a day sums in floating point. + return got == full[k] if isinstance(got, int) else math.isclose(got, full[k]) + # A clock ruler's periods end where they stop; a day's end is its last + # day. Over hours, a day's time is its hours' — first start, last + # stop, their days — never re-spelled at the day's own reading. + from .calendar import GREGORIAN + clock = GREGORIAN.knows(loc.grain) and GREGORIAN.ruler(loc.grain).sub_day + if not clock and all(whole(k, lo, hi) for k, (_, lo, hi) in enumerate(buckets)): return time_variable(field, DefaultWindow(buckets[0][0], grain, len(buckets), None, coarse_first)) # A window that starts or ends inside a quarter: its first or last @@ -770,9 +804,18 @@ def _project_time_row(var: Any, loc: Any, grain: str) -> Variable: values.append({'start': native['start'][lo], 'end': native['end'][hi - 1], 'days': sum(native['days'][lo:hi]), 'months': sum(native['months'][lo:hi]), 'index': k + 1}[field]) - out = Variable(values, value_type='datetime' if field in ('start', 'end') else 'int', + kind = 'datetime' if field in ('start', 'end') else ( + 'int' if all(isinstance(v, int) for v in values) else 'float') + out = Variable(values, value_type=kind, var_type='list', start=buckets[0][0], grain=grain) out._first_day = coarse_first + recipe = {'start': 'first', 'end': 'last', 'days': 'sum'}.get(field) + if clock and recipe is not None: + # Over hours a day's time is read off its hours' own cells — the + # first start, the last stop, the sum of the days — a live formula + # wherever the row is laid out, its values where it is not. + out._set_expr(Regrain(source=var, buckets=[(lo, hi) for _, lo, hi in buckets], + recipe=recipe, fill_values=list(values))) return out diff --git a/src/modeleon/core/regrain.py b/src/modeleon/core/regrain.py index 6defdc4..230f835 100644 --- a/src/modeleon/core/regrain.py +++ b/src/modeleon/core/regrain.py @@ -13,7 +13,7 @@ regrain=mo.up('sum') # a flow regrain=mo.up('last') # a stock (period-end) regrain=mo.up(lambda g: mo.SUM(g.price * g.volume) / mo.SUM(g.volume)) - regrain=mo.up({('day', 'week'): 'last', ('week', 'month'): 'mean'}) + regrain=mo.up({('day', 'month'): 'last', ('month', 'year'): 'mean'}) regrain=mo.ratio('rev', 'users') # sum(num)/sum(den) per bucket regrain=mo.frozen() # grain-frozen (recurrence) diff --git a/src/modeleon/core/shape.py b/src/modeleon/core/shape.py index 0f59088..ca5bc4c 100644 --- a/src/modeleon/core/shape.py +++ b/src/modeleon/core/shape.py @@ -19,7 +19,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Tuple +from typing import TYPE_CHECKING, Any, List, Optional, Tuple if TYPE_CHECKING: from .variable import Variable @@ -47,6 +47,37 @@ def reject_axised(value, where: str) -> None: ) +def keys_along(var: Any) -> Optional[List[Any]]: + """The keys a line runs ALONG instead of periods — a keyed list + (``mo.Variable({'FY2023': 400, 'FY2024': 450})``) or a line on one + finite axis that is not time (``indexed_by=[regions]``) — one per + cell, in cell order, as given; ``None`` for a series, a constant and + a tracked line. The one predicate the layout, the operators and the + functions share for "this line is not a time series".""" + keys = getattr(var, '_keys', None) + if keys and isinstance(getattr(var, '_value', None), list): + return list(keys) + axes: Tuple[Any, ...] = tuple(getattr(var, '_indexed_by', None) or ()) + if len(axes) == 1 and getattr(axes[0], '_axis_kind', None) != 'time': + labels = getattr(axes[0], '_value', None) + if isinstance(labels, list): + return list(labels) + return None + + +def line_keys(var: "Variable") -> Optional[List[str]]: + """The keys a line runs ALONG instead of periods — a keyed list + (``mo.Variable({'FY2023': 400, 'FY2024': 450})``) or a line on one + finite axis that is not time (``indexed_by=[regions]``) — one per + cell, in cell order; ``None`` for a series, a constant and a + tracked line. Such a line stands under a caption row of these keys + (``key_captions_by_sheet``) and counts for no period column. Public + so a renderer captions the line by the keys the layout placed it + under.""" + labels = keys_along(var) + return None if labels is None else [str(label) for label in labels] + + @dataclass(frozen=True, slots=True) class Shape: """Tuple of axes a Variable is laid out along. diff --git a/src/modeleon/core/time.py b/src/modeleon/core/time.py index f298149..aa2cfae 100644 --- a/src/modeleon/core/time.py +++ b/src/modeleon/core/time.py @@ -8,8 +8,10 @@ time = mo.Time('2024-01', periods=3) # monthly revenue = mo.Variable([100, 110, 121], indexed_by=[time]) -Grains: ``day`` (``YYYY-MM-DD``), ``month`` (``YYYY-MM``), ``quarter`` -(``YYYY-Qn``), ``year`` (``YYYY``). Extent is ``start`` plus exactly one of +Grains: ``hour`` (``YYYY-MM-DDTHH``), ``day`` (``YYYY-MM-DD``), ``week`` +(``YYYY-Www``), ``month`` (``YYYY-MM``), ``quarter`` (``YYYY-Qn``), +``half`` (``YYYY-Hn``), ``year`` (``YYYY``), ``iso_year``. Extent is +``start`` plus exactly one of ``end`` or ``periods``. The per-grain calendar helpers (parse / step / format / count) are the foundation the re-grain projection reuses. """ @@ -18,17 +20,99 @@ import functools from collections import namedtuple -from datetime import date, timedelta +from datetime import date, datetime, time as clock_time, timedelta from typing import Any, List, Optional +from .calendar import GREGORIAN, MONTH_ABBR, as_day from .regrain import LINE_UNKNOWN, apply_recipe, hole_value from .variable import Variable -#: Supported grains, finest to coarsest. ``week`` is not supported. -GRAINS = ('day', 'month', 'quarter', 'year') +#: Supported grains, finest to coarsest — the gregorian calendar's rulers +#: (:mod:`modeleon.core.calendar`). +GRAINS = GREGORIAN.grains + +#: The columns one sheet holds (Excel's 16,384, ``XFD``). A period is a +#: column, so no window may be wider than a sheet; the label column takes +#: one, so a window refuses past :data:`WINDOW_MAX_PERIODS` where it is +#: declared, and the layout counts the rest (units, buckets). +SHEET_COLUMNS = 16384 +WINDOW_MAX_PERIODS = SHEET_COLUMNS - 1 + + +def article(grain: str) -> str: + """``'an'`` before ``hour``, ``'a'`` before every other ruler's name — + in the engine's messages.""" + return 'an' if grain == 'hour' else 'a' + + +class SheetWidthError(ValueError): + """A window wider than a sheet: refused by name, never truncated.""" + + +def width_refusal(grain: str, periods: int, lead: int = 1, + columns: Optional[int] = None) -> str: + """Why ``periods`` periods of ``grain`` — ``columns`` columns with + their totals, ``lead`` label columns before them — do not fit one + sheet: the count, the sheet's width, what this grain fits, and the + two ways out — a coarser grain, or the data kept as rows.""" + columns = periods if columns is None else columns + fits = SHEET_COLUMNS - lead + span = '' + if GREGORIAN.knows(grain): + ruler = GREGORIAN.ruler(grain) + days = fits * ruler.measure(ruler.start_of(date(2026, 1, 1))) + span = (f"; {grain}s fit about {round(days / 30.44)} months" if days < 3650 + else f"; {grain}s fit about {round(days / 365.25)} years") + coarser = next((g for g in GREGORIAN.rolls_up_to(grain)), None) if GREGORIAN.knows(grain) else None + by = f"View it by {coarser} (default_grain='{coarser}', or .at('{coarser}') on its lines)" \ + if coarser else "View it by a coarser grain" + totals = " with its totals" if columns != periods else "" + return ( + f"A window of {periods} {grain}s needs {columns} columns{totals} beside {lead} " + f"label column{'s' if lead != 1 else ''}, and a sheet holds {SHEET_COLUMNS}{span}. " + f"{by}, or keep the {grain}s as the rows of a table and chart them." + ) + + +#: A window's zone: a fixed offset from UTC, spelled ``+01:00`` — a zone +#: that moves its clocks would skip and repeat hours. +_ZONE_NAMES = {'UTC': '+00:00', 'Z': '+00:00', 'GMT': '+00:00'} + + +def zone_offset(value: Any) -> str: + """``default_zone`` as ``+HH:MM``: a fixed offset (``'+01:00'``, + ``'-03:30'``, ``'UTC'`` / ``'Z'``). A named zone (``'Europe/Paris'``) is + refused with the offset to write instead: an hour is the wall clock of + one zone whose clock never moves.""" + import re + if isinstance(value, str): + text = value.strip() + if text.upper() in _ZONE_NAMES: + return _ZONE_NAMES[text.upper()] + m = re.fullmatch(r'([+-])(\d{2}):?(\d{2})', text) + if m and int(m.group(2)) <= 14 and m.group(3) in ('00', '15', '30', '45'): + return f"{m.group(1)}{m.group(2)}:{m.group(3)}" + hint = '' + if isinstance(value, str) and '/' in value: + try: + from zoneinfo import ZoneInfo + now = datetime.now(ZoneInfo(value.strip())) + off = now.utcoffset() + if off is not None: + minutes = int(off.total_seconds() // 60) + sign = '+' if minutes >= 0 else '-' + hint = (f" {value.strip()} is {sign}{abs(minutes) // 60:02d}:" + f"{abs(minutes) % 60:02d} today; write that, if its clocks never move.") + except Exception: # noqa: BLE001 — a hint only + hint = '' + raise ValueError( + f"default_zone is a fixed offset from UTC - '+01:00', '-03:30' or 'UTC'; " + f"got {value!r}. A named zone can move its clocks, and an hour window " + f"never skips or repeats an hour.{hint}" + ) #: Coarsening order — re-grain only goes left-to-right. -_ORDER = {'day': 0, 'month': 1, 'quarter': 2, 'year': 3} +_ORDER = {name: GREGORIAN.rank(name) for name in GRAINS} #: Period-label styles for the Excel timeline header. ``iso`` is the engine's #: native form (matches ``mo.Time`` axis labels); ``finance`` / ``compact`` are @@ -38,8 +122,7 @@ #: Default header style when an ``ExcelView`` declares none. DEFAULT_TIME_LABEL_STYLE = 'finance' -_MONTH_ABBR = ('', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', - 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec') +_MONTH_ABBR = MONTH_ABBR #: A model's default *projection* window — ``start`` + ``grain`` + number of #: ``periods``. Resolved up the MV tree by :func:`resolve_default_window`. @@ -54,9 +137,12 @@ #: first period (on the 1st of a month): that period — a stub — runs from #: this day to its calendar end and keeps its label; every later one is #: whole. +#: ``zone`` is the window's fixed offset (``'+01:00'``) when it declares +#: one — read only where an instant leaves the engine; the engine itself +#: computes on the wall clock. DefaultWindow = namedtuple('DefaultWindow', - ['start', 'grain', 'periods', 'start_source', 'first_day'], - defaults=(None, None)) + ['start', 'grain', 'periods', 'start_source', 'first_day', 'zone'], + defaults=(None, None, None)) def resolve_default_window(node: Any) -> Optional['DefaultWindow']: @@ -79,6 +165,7 @@ def resolve_default_window(node: Any) -> Optional['DefaultWindow']: getattr(n, 'default_start', None), grain, getattr(n, 'default_periods', None), slots.get('_default_start_source'), slots.get('_default_first_day'), + slots.get('default_zone'), ) n = getattr(n, '_owner', None) or getattr(n, '_parent', None) return None @@ -138,12 +225,26 @@ def window_start_label(value: Any, grain: str) -> str: grain it may fall inside the period — a valuation on 1 May: the first period is then the rest of that quarter or year (a stub, :func:`stub_first_day`) and keeps its label. A day inside a month is - refused: a period's months are whole. + refused: a period's months are whole. A clock ruler (an hour) starts + at an instant on the hour — a ``datetime`` on the wall clock of the + window's zone, or a date (its midnight). """ - from datetime import datetime if isinstance(value, str): _parse(value, grain) return value + ruler = GREGORIAN.ruler(grain) if GREGORIAN.knows(grain) else None + if ruler is not None and ruler.sub_day and isinstance(value, date): + if isinstance(value, datetime) and value.tzinfo is not None: + raise ValueError( + f"An hour window starts at a wall-clock time of its zone, not at " + f"{value.isoformat()}: write the time without its offset and " + f"declare the zone once - default_zone='+01:00'." + ) + first = ruler.start_of(value) + if first != (value if isinstance(value, datetime) + else datetime.combine(value, clock_time())): + raise ValueError(ruler.refuse_start(value)) + return ruler.label(first) if isinstance(value, datetime): if value.time() != datetime.min.time(): raise ValueError( @@ -156,34 +257,43 @@ def window_start_label(value: Any, grain: str) -> str: f"A window starts at a date or a period label like '2026-01'; " f"got {value!r}." ) - if grain == 'day': - return value.isoformat() - if grain == 'month' and value.day == 1: - return f"{value.year:04d}-{value.month:02d}" - if grain == 'quarter' and value.day == 1 and value.month in (1, 4, 7, 10): - return f"{value.year:04d}-Q{(value.month - 1) // 3 + 1}" - if grain == 'year' and (value.month, value.day) == (1, 1): - return f"{value.year:04d}" - if grain in ('quarter', 'year') and value.day == 1: + ruler = GREGORIAN.ruler(grain) + first = ruler.start_of(value) + if first == value: + return ruler.label(first) + if _stub_unit_start(ruler, value): # A stub: the first period is the rest of this quarter / year. - return _containing_label(value, grain) - raise ValueError( - f"A {grain} window starts on the 1st of a month; {value.isoformat()} " - f"falls inside a month. Start the window on the 1st and keep the " - f"exact day as an input — read mo.time.days for the part of a period " - f"after it." - ) + return ruler.label(first) + raise ValueError(ruler.refuse_start(value)) def _containing_label(day: date, grain: str) -> str: - """The label of the ``grain`` period that holds ``day``.""" - if grain == 'quarter': - return f"{day.year:04d}-Q{(day.month - 1) // 3 + 1}" - if grain == 'year': - return f"{day.year:04d}" - if grain == 'month': - return f"{day.year:04d}-{day.month:02d}" - return day.isoformat() + """The label of the ``grain`` period that holds ``day`` (an instant + on a clock ruler).""" + if not GREGORIAN.knows(grain): + return day.isoformat() + ruler = GREGORIAN.ruler(grain) + return ruler.label(ruler.start_of(day)) + + +def _has_stub(grain: Any) -> bool: + """Whether a ``grain`` window may open inside its first period (on a + boundary of the ruler's ``stub_unit``) — a quarter or year window + starting on the 1st of any month.""" + return GREGORIAN.knows(grain) and GREGORIAN.ruler(grain).stub_unit is not None + + +def _stub_unit_start(ruler: Any, day: date) -> bool: + """Whether ``day`` opens a period of ``ruler``'s stub unit — the 1st + of a month under a quarter or year ruler. An instant opens one only + at its midnight.""" + if ruler.stub_unit is None: + return False + if isinstance(day, datetime): + if day.time() != clock_time(): + return False + day = day.date() + return GREGORIAN.ruler(ruler.stub_unit).start_of(day) == day def spelled_first_day(window: Any) -> Optional[date]: @@ -192,7 +302,7 @@ def spelled_first_day(window: Any) -> Optional[date]: spelling: the stub's day, or — the window starts at a date cell, which may be moved inside a period in the book — its first period's first day. A start written into the formulas cannot move: whole spelling.""" - if window is None or window.grain not in ('quarter', 'year') or window.start is None: + if window is None or not _has_stub(window.grain) or window.start is None: return None if getattr(window, 'first_day', None) is not None: return window.first_day @@ -208,12 +318,13 @@ def _coarse_first_day(native_start: str, native_grain: str, native_first_day: Op coarse first period is then short, as every line and time row of the re-grained view must agree. ``None`` when it starts on the coarse period's first day, or the target is finer than a quarter.""" - if target_grain not in ('quarter', 'year') or native_start is None: + if not _has_stub(target_grain) or native_start is None: return None anchor = _parse(native_start, native_grain) first = native_first_day or _first_day(anchor, native_grain) - if first.day != 1: + if not _stub_unit_start(GREGORIAN.ruler(target_grain), first): return None + first = as_day(first) label = _target_label(anchor, native_grain, target_grain) if _first_day(_parse(label, target_grain), target_grain) == first: return None @@ -226,7 +337,7 @@ def stub_first_day(value: Any, grain: str) -> Optional[date]: inside its first period (always the 1st of a month — :func:`window_start_label` refuses any other day).""" from datetime import datetime - if grain not in ('quarter', 'year'): + if not _has_stub(grain): return None if isinstance(value, datetime): value = value.date() @@ -238,107 +349,52 @@ def stub_first_day(value: Any, grain: str) -> Optional[date]: def _parse(start: str, grain: str) -> Any: - """Parse a ``start`` label into a per-grain anchor (the first period).""" - if grain == 'day': - try: - return date.fromisoformat(start) - except (ValueError, TypeError): - raise ValueError(f"Time day start must be 'YYYY-MM-DD'; got {start!r}.") - if grain == 'month': - return _parse_year_month(start) - if grain == 'quarter': - return _parse_year_quarter(start) - if grain == 'year': - try: - return int(start) - except (ValueError, TypeError): - raise ValueError(f"Time year start must be 'YYYY'; got {start!r}.") - raise ValueError(f"Unknown time grain {grain!r}; supported: {', '.join(GRAINS)}.") + """Parse a ``start`` label into a per-grain anchor (the first period). + + The anchor is the period's first day spelled the way this module + has always carried it — ``date`` for a day, ``(year, month)`` for a + month, ``(year, quarter)`` for a quarter, ``year`` for a year; the + ruler itself speaks in first days (:func:`_first_day` / + :func:`_anchor_of` convert).""" + return _anchor_of(GREGORIAN.ruler(grain).parse(start), grain) def _parse_year_month(start: str) -> tuple: - parts = start.split('-') if isinstance(start, str) else [] - if len(parts) != 2: - raise ValueError(f"Time month start must be 'YYYY-MM'; got {start!r}.") - try: - year, month = int(parts[0]), int(parts[1]) - except ValueError: - raise ValueError(f"Time month start must be 'YYYY-MM'; got {start!r}.") - if not 1 <= month <= 12: - raise ValueError(f"Month must be 01-12 in {start!r}.") - return (year, month) + return _anchor_of(GREGORIAN.ruler('month').parse(start), 'month') def _parse_year_quarter(start: str) -> tuple: - parts = start.upper().split('-Q') if isinstance(start, str) else [] - if len(parts) != 2: - raise ValueError(f"Time quarter start must be 'YYYY-Qn'; got {start!r}.") - try: - year, quarter = int(parts[0]), int(parts[1]) - except ValueError: - raise ValueError(f"Time quarter start must be 'YYYY-Qn'; got {start!r}.") - if not 1 <= quarter <= 4: - raise ValueError(f"Quarter must be Q1-Q4 in {start!r}.") - return (year, quarter) + return _anchor_of(GREGORIAN.ruler('quarter').parse(start), 'quarter') -def _step(anchor: Any, grain: str) -> Any: - """Advance an anchor by one period of ``grain``.""" - if grain == 'day': - return anchor + timedelta(days=1) +def _anchor_of(first: date, grain: str) -> Any: + """The per-grain anchor of the period starting on ``first`` — the + inverse of :func:`_first_day`.""" if grain == 'month': - year, month = anchor - month += 1 - if month > 12: - month, year = 1, year + 1 - return (year, month) + return (first.year, first.month) if grain == 'quarter': - year, quarter = anchor - quarter += 1 - if quarter > 4: - quarter, year = 1, year + 1 - return (year, quarter) - return anchor + 1 # year + return (first.year, (first.month - 1) // 3 + 1) + if grain == 'half': + return (first.year, (first.month - 1) // 6 + 1) + if grain == 'year': + return first.year + return first + + +def _step(anchor: Any, grain: str) -> Any: + """Advance an anchor by one period of ``grain``.""" + ruler = GREGORIAN.ruler(grain) + return _anchor_of(ruler.next_start(_first_day(anchor, grain)), grain) def _format(anchor: Any, grain: str, style: str = 'iso') -> str: - if grain == 'day': - if style == 'finance': - return f"{anchor.day:02d} {_MONTH_ABBR[anchor.month]} {anchor.year:04d}" - if style == 'compact': - return f"{anchor.day:02d}-{_MONTH_ABBR[anchor.month]}-{anchor.year % 100:02d}" - return anchor.isoformat() - if grain == 'month': - year, month = anchor - if style == 'finance': - return f"{_MONTH_ABBR[month]} {year:04d}" - if style == 'compact': - return f"{_MONTH_ABBR[month]}-{year % 100:02d}" - return f"{year:04d}-{month:02d}" - if grain == 'quarter': - year, quarter = anchor - if style == 'finance': - return f"Q{quarter} {year:04d}" - if style == 'compact': - return f"Q{quarter}-{year % 100:02d}" - return f"{year:04d}-Q{quarter}" - # year - if style == 'compact': - return f"FY{anchor % 100:02d}" - return f"{anchor:04d}" + return GREGORIAN.ruler(grain).label(_first_day(anchor, grain), style) def _count(start_anchor: Any, end_anchor: Any, grain: str) -> int: """Inclusive period count from ``start_anchor`` to ``end_anchor``.""" - if grain == 'day': - return (end_anchor - start_anchor).days + 1 - if grain == 'month': - (sy, sm), (ey, em) = start_anchor, end_anchor - return (ey - sy) * 12 + (em - sm) + 1 - if grain == 'quarter': - (sy, sq), (ey, eq) = start_anchor, end_anchor - return (ey - sy) * 4 + (eq - sq) + 1 - return end_anchor - start_anchor + 1 # year + return GREGORIAN.ruler(grain).count(_first_day(start_anchor, grain), + _first_day(end_anchor, grain)) def _period_labels(start: str, end: Optional[str], periods: Optional[int], @@ -350,11 +406,14 @@ def _period_labels(start: str, end: Optional[str], periods: Optional[int], count = _count(anchor, _parse(end, grain), grain) # type: ignore[arg-type] if count <= 0: raise ValueError(f"Time end {end!r} is before start {start!r}.") + # Step between labels, never past the last: the period after a window + # that ends in December 9999 is one no date can hold. labels: List[str] = [] current = anchor - for _ in range(count): + for i in range(count): + if i: + current = _step(current, grain) labels.append(_format(current, grain, style)) - current = _step(current, grain) return labels @@ -392,35 +451,25 @@ def __init__(self, start: str, end: Optional[str] = None, *, # --- coarsening / bucketing (re-grain projection foundation) ---------------- -def _days_in_period(anchor: Any, grain: str) -> int: - """Calendar day-count of one period at ``grain`` — the re-grain weight. +def _days_in_period(anchor: Any, grain: str) -> float: + """Calendar day-count of one period at ``grain`` — the re-grain weight + (whole days, an ``int``; 1/24 for an hour). Periods are not equal (Feb vs Jul; leap years), so any averaging recipe must weight by real days, not by period count. """ - if grain == 'day': - return 1 - if grain == 'month': - year, month = anchor - nxt = date(year + 1, 1, 1) if month == 12 else date(year, month + 1, 1) - return (nxt - date(year, month, 1)).days - if grain == 'quarter': - year, quarter = anchor - first_month = (quarter - 1) * 3 + 1 - end_year, end_month = (year + 1, 1) if quarter == 4 else (year, first_month + 3) - return (date(end_year, end_month, 1) - date(year, first_month, 1)).days - return (date(anchor + 1, 1, 1) - date(anchor, 1, 1)).days # year + return GREGORIAN.ruler(grain).measure(_first_day(anchor, grain)) def _period_weights(grain: str, start: str, n: int, - first_day: Optional[date] = None) -> List[int]: + first_day: Optional[date] = None) -> List[float]: """Day-count weights for ``n`` consecutive periods of ``grain`` from ``start``. A pure function of the calendar — every series over the same window shares one weight vector, which is what keeps weighted recipes aligned across lines. A short first period (``first_day``) weighs only its own days.""" anchor = _parse(start, grain) - weights: List[int] = [] + weights: List[float] = [] for _ in range(n): weights.append(_days_in_period(anchor, grain)) anchor = _step(anchor, grain) @@ -428,32 +477,37 @@ def _period_weights(grain: str, start: str, n: int, weights[0] = time_field_series('days', start, grain, 1, first_day)[0] return weights -def _to_year_month(anchor: Any, grain: str) -> tuple: - """Normalize a native-grain anchor to ``(year, month)`` (month = the - period's first month) for coarse-label computation.""" - if grain == 'day': - return (anchor.year, anchor.month) - if grain == 'month': - return anchor - if grain == 'quarter': - year, quarter = anchor - return (year, (quarter - 1) * 3 + 1) - return (anchor, 1) # year - - def _target_label(anchor: Any, native_grain: str, target_grain: str) -> str: - """The coarse-grain label a native period falls into.""" - year, month = _to_year_month(anchor, native_grain) - if target_grain == 'month': - return f"{year:04d}-{month:02d}" - if target_grain == 'quarter': - return f"{year:04d}-Q{(month - 1) // 3 + 1}" - if target_grain == 'year': - return f"{year:04d}" - raise ValueError( - f"Cannot coarsen to {target_grain!r}; re-grain only goes to a coarser " - f"grain (day -> month -> quarter -> year)." - ) + """The coarse-grain label a native period falls into — the period of + the target ruler that holds the native period's first day.""" + if not GREGORIAN.knows(target_grain) or GREGORIAN.rank(target_grain) == 0: + # The finest ruler is no coarser grain of anything. + raise ValueError( + f"Cannot coarsen to {target_grain!r}; re-grain only goes to a coarser " + f"grain ({' -> '.join(GRAINS)})." + ) + target = GREGORIAN.ruler(target_grain) + return target.label(target.start_of(_first_day(anchor, native_grain))) + + +def regrain_refusal(native_grain: str, target_grain: str) -> str: + """Why a ``native_grain`` line cannot be rolled up to ``target_grain``: + the target is not coarser, or the two rulers do not nest (a week + straddles two months) — never a guessed split.""" + up = GREGORIAN.rolls_up_to(native_grain) if GREGORIAN.knows(native_grain) else () + a = article(native_grain) + to = (f"{a} {native_grain} rolls up to {', '.join(up)}" if up + else f"{a} {native_grain} rolls up to nothing coarser") + if (native_grain == target_grain or not GREGORIAN.knows(target_grain) + or not GREGORIAN.knows(native_grain) + or GREGORIAN.nests(target_grain, native_grain)): + return (f"Re-grain coarsens only: {native_grain} -> {target_grain} is not a " + f"coarser grain ({to}).") + return (f"{a.capitalize()} {native_grain} does not lie inside one {target_grain}: " + f"{native_grain}s straddle {target_grain}s, so {a} {native_grain} line cannot " + f"be rolled up to {target_grain}s without splitting {a} {native_grain} ({to}). " + f"Model the line in a grain both share (days), or total it by a grain it " + f"rolls up to.") def _bucket_ranges(native_grain: str, target_grain: str, native_start: str, @@ -466,11 +520,8 @@ def _bucket_ranges(native_grain: str, target_grain: str, native_start: str, several series share one bucketing (the alignment that makes a ratio-of-sums correct). """ - if _ORDER[target_grain] <= _ORDER[native_grain]: - raise ValueError( - f"Re-grain coarsens only: {native_grain} -> {target_grain} is not a " - f"coarser grain (day -> month -> quarter -> year)." - ) + if not GREGORIAN.nests(native_grain, target_grain): + raise ValueError(regrain_refusal(native_grain, target_grain)) anchor = _parse(native_start, native_grain) labels: List[str] = [] current = anchor @@ -551,7 +602,7 @@ def _period_weights_cached(grain: str, start: Any, n: int, first_day: Optional[d def period_weights(grain: str, start: str, n: int, - first_day: Optional[date] = None) -> List[int]: + first_day: Optional[date] = None) -> List[float]: """Day-count weights for ``n`` consecutive periods of ``grain`` from ``start`` — see :func:`_period_weights`; remembered per argument tuple, the list is the caller's own.""" @@ -581,24 +632,28 @@ def coarse_first_day(native_start: str, native_grain: str, native_first_day: Opt # --- the time of a period (``mo.time``) ------------------------------------ -#: The fields of ``mo.time``: the period's first and last day, its calendar -#: day and month counts and its 1-based number in the window. +#: The fields of ``mo.time``: the period's first and last day (on a clock +#: ruler: the instants it starts and stops at), its calendar day and month +#: counts and its 1-based number in the window. TIME_FIELDS = ('start', 'end', 'days', 'months', 'index') #: The calendar months one period of a grain holds. A day holds none: a #: day window refuses ``mo.time.months`` (``mo.time.days`` is its measure). -MONTHS_IN_PERIOD = {'day': 0, 'month': 1, 'quarter': 3, 'year': 12} +MONTHS_IN_PERIOD = {r.name: (r.months or 0) for r in GREGORIAN.rulers} def _first_day(anchor: Any, grain: str) -> date: - """The first calendar day of the period ``anchor`` names.""" - if grain == 'day': - return anchor + """The first calendar day of the period ``anchor`` names — the + inverse of :func:`_anchor_of`.""" if grain == 'month': return date(anchor[0], anchor[1], 1) if grain == 'quarter': return date(anchor[0], (anchor[1] - 1) * 3 + 1, 1) - return date(anchor, 1, 1) # year + if grain == 'half': + return date(anchor[0], (anchor[1] - 1) * 6 + 1, 1) + if grain == 'year': + return date(anchor, 1, 1) + return anchor # a day's anchor is the day def time_field_series(field: str, start: str, grain: str, n: int, @@ -608,13 +663,25 @@ def time_field_series(field: str, start: str, grain: str, n: int, A short first period (``first_day``, the 1st of a month inside the first period) starts on that day: its ``days`` and ``months`` are only the part of the period from it; its ``end`` and every later period are - the calendar's.""" + the calendar's. A clock ruler's period ends where it stops (13:00 → + 14:00) and lasts its fraction of a day.""" if field not in TIME_FIELDS: raise AttributeError( f"mo.time has no field {field!r}; it has {', '.join(TIME_FIELDS)}." ) anchor = _parse(start, grain) + ruler = GREGORIAN.ruler(grain) out: List[Any] = [] + if ruler.sub_day: + for i in range(n): + first = _first_day(anchor, grain) + out.append({'start': first, + 'end': ruler.next_start(first), + 'days': ruler.measure(first), + 'months': MONTHS_IN_PERIOD[grain], + 'index': i + 1}[field]) + anchor = _step(anchor, grain) + return out for i in range(n): first = _first_day(anchor, grain) days = _days_in_period(anchor, grain) @@ -633,6 +700,31 @@ def time_field_series(field: str, start: str, grain: str, n: int, return out +def period_bounds(start: str, grain: str, n: int, + first_day: Optional[date] = None) -> tuple: + """``(starts, ends)`` — the first and last day of each of ``n`` + periods of ``grain`` from ``start``; a short first period starts on + ``first_day``. A caller that needs each period's dates reads them here + and does no calendar arithmetic of its own.""" + return (time_field_series('start', start, grain, n, first_day), + time_field_series('end', start, grain, n, first_day)) + + +def year_back_index(start: str, grain: str, n: int) -> List[Optional[int]]: + """For each of ``n`` periods of ``grain`` from ``start``, the index + of the period one calendar year earlier in the same window, or + ``None`` when the window does not reach it — what a growth-on-last- + year compares against, asked of the calendar rather than counted.""" + ruler = GREGORIAN.ruler(grain) + labels = _period_labels(start, None, n, grain, 'iso') + index = {label: i for i, label in enumerate(labels)} + out: List[Optional[int]] = [] + for label in labels: + back = ruler.year_back(ruler.parse(label)) + out.append(None if back is None else index.get(ruler.label(back))) + return out + + def time_variable(field: str, window: 'DefaultWindow') -> Variable: """A fresh Variable holding ``mo.time.`` over ``window``. @@ -648,9 +740,10 @@ def time_variable(field: str, window: 'DefaultWindow') -> Variable: first_day = getattr(window, 'first_day', None) values = time_field_series(field, window.start, window.grain, int(window.periods), first_day) + whole = not (field == 'days' and GREGORIAN.ruler(window.grain).sub_day) var = Variable( values, - value_type='datetime' if field in ('start', 'end') else 'int', + value_type='datetime' if field in ('start', 'end') else ('int' if whole else 'float'), var_type='list', start=window.start, grain=window.grain, ) var._first_day = first_day @@ -710,7 +803,8 @@ def end(self) -> Variable: @property def days(self) -> Variable: - """The calendar day count of each period (31, 28, …; 90 a quarter).""" + """The calendar day count of each period (31, 28, …; 90 a quarter; + 1/24 an hour).""" return self._read('days') @property @@ -743,11 +837,11 @@ def _read(self, field: str) -> Variable: f"set {'them' if len(missing) > 1 else 'it'} before the first " f"line that reads time." ) - if field == 'months' and window.grain == 'day': + if field == 'months' and GREGORIAN.ruler(window.grain).months is None: raise ValueError( "mo.time.months counts the calendar months of a period: 1 a " - "month, 3 a quarter, 12 a year. A day window has none - read " - "mo.time.days." + f"month, 3 a quarter, 12 a year. {article(window.grain).capitalize()} " + f"{window.grain} window has none - read mo.time.days." ) return time_variable(field, window) diff --git a/src/modeleon/core/unit.py b/src/modeleon/core/unit.py index 281158c..ac8ecfd 100644 --- a/src/modeleon/core/unit.py +++ b/src/modeleon/core/unit.py @@ -33,13 +33,22 @@ pv = cash * df # cash's unit: the label is inert Four words are labelled pure numbers by convention, as strings: -``'%'``, ``'x'``, ``'bps'`` and ``'pp'``. +``'%'``, ``'x'``, ``'bps'`` and ``'pp'``. Each is written as the number +it is — 5% is 0.05, 8x is 8, 50 bps is 0.005, 2 pp is 0.02 — because a +label prints and never rescales (a label the arithmetic drops must not +change a magnitude). A bps or pp line holding a count (50) added to a % +line holding fractions is refused: that sum is wrong under any reading. A unit is a plain value: assigned to a model it is an attribute, never a row or a section. """ -from typing import Dict, Optional +import re +from typing import Dict, Optional, Tuple + + +#: A factor raised to a whole power, as ``__str__`` writes it: ``m^2``. +_POWER = re.compile(r'(.+?)\s*\^\s*([+-]?\d+)') class Unit: @@ -61,6 +70,13 @@ class Unit: #: dimension — by convention, the same for every author. _DIMENSIONLESS_SYMBOLS = frozenset({'%', 'x', 'bps', 'pp'}) + #: The text a unit prints when its canonical form would not read back + #: as the author wrote it: a chain ('USD/kW/month' rather than + #: 'USD/(kW * month)') and a string with unbalanced parentheses. A unit + #: that arithmetic builds has none. A class default, so a Unit restored + #: from an older pickle prints too. + _written: Optional[str] = None + def __init__(self, symbol: Optional[str] = None, *, _components: Optional[Dict[str, int]] = None): self._unit_components: Dict[str, int] = {} @@ -79,6 +95,8 @@ def __init__(self, symbol: Optional[str] = None, *, self._display_symbol = symbol else: self._unit_components = self._parse(symbol) + if self._unit_components and self._prints_as_written(symbol.strip()): + self._written = symbol.strip() @staticmethod def _parse(text: str) -> Dict[str, int]: @@ -91,11 +109,24 @@ def _parse(text: str) -> Dict[str, int]: 'hr * Gb' → {'hr': 1, 'Gb': 1} '$/(hr * Gb)' → {'$': 1, 'hr': -1, 'Gb': -1} '$ * hr / Gb' → {'$': 1, 'hr': 1, 'Gb': -1} + '$/kW/month' → {'$': 1, 'kW': -1, 'month': -1} + '$/(kW/month)' → {'$': 1, 'kW': -1, 'month': 1} + 'USD/m^2' → {'USD': 1, 'm': -2} + + Everything after the first ``/`` is the denominator (``'$/hr * Gb'`` + is ``'$/(hr * Gb)'``), and each further ``/`` divides again, so a + chain reads the way it is written: USD per kW per month. A whole + power follows ``^``, as a unit prints it (``'m^2'``). A word may + carry a note in parentheses ('kW (AC)', 'USD (real)'); a string + whose parentheses do not balance keeps the first-slash reading. """ text = text.strip() if not text: return {} + if Unit._balanced(text): + return Unit._parse_chain(text) + components: Dict[str, int] = {} numerator_part = text @@ -117,6 +148,74 @@ def _parse(text: str) -> Dict[str, int]: return {k: v for k, v in components.items() if v != 0} + @staticmethod + def _prints_as_written(text: str) -> bool: + """A chain of two or more slashes, once parentheses that wrap the + whole string are set aside, or unbalanced parentheses: the canonical + form of either would not read back as the author wrote it.""" + if not Unit._balanced(text): + return True + while text.startswith('(') and text.endswith(')') and Unit._balanced(text[1:-1]): + text = text[1:-1].strip() + return len(Unit._split_top(text, '/')) > 2 + + @staticmethod + def _parse_chain(text: str) -> Dict[str, int]: + """The numerator, then each ``/``-separated piece divides it; a + parenthesised factor is a unit of its own, read the same way.""" + components: Dict[str, int] = {} + for i, piece in enumerate(Unit._split_top(text, '/')): + sign = 1 if i == 0 else -1 + for factor in Unit._split_top(piece, '*'): + factor = factor.strip() + if not factor or factor == '1': + continue + base, power = Unit._raised(factor) + if (base.startswith('(') and base.endswith(')') + and Unit._balanced(base[1:-1])): + sub = Unit._parse(base[1:-1]) + else: + sub = {base: 1} + for sym, exp in sub.items(): + components[sym] = components.get(sym, 0) + sign * exp * power + return {k: v for k, v in components.items() if v != 0} + + @staticmethod + def _raised(factor: str) -> Tuple[str, int]: + """``'m^2'`` → ``('m', 2)``: a whole power after ``^``; any other + factor is itself, to the first power.""" + match = _POWER.fullmatch(factor) + if match is None or not Unit._balanced(match.group(1)): + return factor, 1 + return match.group(1).strip(), int(match.group(2)) + + @staticmethod + def _split_top(text: str, sep: str) -> list: + """Split on ``sep`` outside parentheses.""" + parts, depth, start = [], 0, 0 + for i, ch in enumerate(text): + if ch == '(': + depth += 1 + elif ch == ')': + depth -= 1 + elif ch == sep and depth == 0: + parts.append(text[start:i]) + start = i + 1 + parts.append(text[start:]) + return parts + + @staticmethod + def _balanced(text: str) -> bool: + depth = 0 + for ch in text: + if ch == '(': + depth += 1 + elif ch == ')': + depth -= 1 + if depth < 0: + return False + return depth == 0 + @staticmethod def _split_product(text: str) -> list: """Split 'hr * Gb' into ['hr', 'Gb']. Single symbol returns [symbol]. @@ -214,6 +313,8 @@ def is_dimensionless(self) -> bool: return len(self._unit_components) == 0 def __str__(self): + if self._written is not None: + return self._written if not self._unit_components: return self._display_symbol or '' @@ -258,6 +359,19 @@ def __bool__(self): return bool(self._unit_components) or bool(self._display_symbol) +class _Unsettled: + __slots__ = () + + def __repr__(self) -> str: + return "UNSETTLED" + + +#: The unit of a loop row not derived yet, while its loop settles (see +#: ``core.loops._rederive_units``): ``+`` takes the other side's unit, a +#: product or a power of it stays unsettled. Never a line's unit. +UNSETTLED = _Unsettled() + + def unit_key(unit: Unit) -> tuple: """A unit as one value for "the same unit": its dimension and, for a pure number, its label. Two lines in ``'%'`` and in ``'x'`` add (both diff --git a/src/modeleon/core/variable.py b/src/modeleon/core/variable.py index 6479181..e36e548 100644 --- a/src/modeleon/core/variable.py +++ b/src/modeleon/core/variable.py @@ -28,16 +28,19 @@ """ from collections import namedtuple -from typing import Optional, Set, Any, Dict, List, Sequence, Tuple, TYPE_CHECKING, Self +from contextvars import ContextVar +from typing import (Optional, Set, Any, Dict, List, Sequence, Tuple, TYPE_CHECKING, Self, + overload) from .humanize import humanize_identifier from .component import Component -from .expr import Expr, MethodCall, Paren, Regrain, Restrict, Subscript, VarRef +from .expr import Expr, MethodCall, Paren, Regrain, Restrict, Subscript, TimeRef, VarRef from .qpath import QPath from .regrain import RegrainSpec from .shape import Shape +from .tracks import TrackValues from .variable_init import _VariableInit -from .variable_ops import _VariableArithmetic +from .variable_ops import _VariableArithmetic, _forget_placements if TYPE_CHECKING: from pathlib import Path @@ -138,18 +141,91 @@ def series_at_grain(label: str, values: Any, loc: 'TimeLoc', spec: Any, grain: s #: writes only what differs. Immutable values only (the two per-instance #: containers, ``_excel_props`` and ``_dependency_refs``, are built per #: instance); the key set is exactly the fields the constructor defines, -#: so ``vars(v)`` / ``hasattr`` see every field. +#: so ``vars(v)`` / ``hasattr`` see every field. ``_source_code`` is not +#: among them: it is derived on read (:class:`_SourceCode`) and a value in +#: ``__dict__`` would shadow the descriptor. _VARIABLE_DEFAULTS: Dict[str, Any] = { '_role_kwargs': None, '_roles_materialized': False, '_roles_provisional': False, '_blend_name': None, '_excel_layout': None, '_owner': None, '_component_name': None, '_value': None, '_expr': None, '_raw_formula_str': None, 'pyformula': None, - '_source_code': None, '_unit': None, '_unit_declared': False, '_keys': None, + '_unit': None, '_unit_declared': False, '_keys': None, '_keys_source': None, 'var_type': 'scalar', 'value_type': 'float', '_indexed_by': (), '_start': None, '_grain': None, '_regrain': None, '_schedule': None, '_extend': None, } + +#: The compounds whose shared text is being decided in this context. A row of +#: a loop container that lists itself (``row = Variable([row, 2.0])``) is +#: rewritten in place, so its items reach the row again; such a list shares +#: no text. +_SOURCE_RESOLVING: ContextVar[frozenset] = ContextVar( + 'modeleon_source_resolving', default=frozenset() +) + + +def _shared_source_code(owner: 'Variable', items: Tuple['Variable', ...]) -> Optional[str]: + """The one source text every Variable in ``items`` carries, else None.""" + resolving = _SOURCE_RESOLVING.get() + if id(owner) in resolving: + return None + token = _SOURCE_RESOLVING.set(resolving | {id(owner)}) + try: + shared: Optional[str] = None + for item in items: + code = item._source_code + if not code or (shared is not None and code != shared): + return None + shared = code + return shared + finally: + _SOURCE_RESOLVING.reset(token) + + +class _SourceCode: + """``Variable._source_code``, derived when read rather than at construction. + + A non-data descriptor (``__get__`` only): a text written on the + instance — a helper's resolved call (``PMT(0.005, 360, ...)``), the + public setter, the ``None`` an adopted typed-dates line writes — lands + in ``__dict__`` and wins. With none written, the text follows what the + instance says it is: its own formula (``_source_is_formula``, the IF + family) or the one text its list items share (``_source_items``, a + ``Variable([a, b, ...])`` compound), else None. The formula render + inlines every anonymous operand, so the helpers used to pay for it at + every call for a text almost nobody reads; nested conditions paid for + the inner text twice. + """ + + @overload + def __get__(self, obj: None, objtype: Optional[type] = None) -> '_SourceCode': ... + + @overload + def __get__(self, obj: 'Variable', objtype: Optional[type] = None) -> Optional[str]: ... + + def __get__(self, obj: Optional['Variable'], objtype: Optional[type] = None) -> Any: + if obj is None: + return self + if obj._source_is_formula: + return obj.formula + items = obj._source_items + if items is not None: + return _shared_source_code(obj, items) + return None + _object_setattr = object.__setattr__ +def _under_settings(node: Any) -> bool: + """True when ``node`` is, or sits inside, a container that holds + settings rather than lines (an ExcelView).""" + seen: set = set() + while node is not None and id(node) not in seen: + seen.add(id(node)) + if getattr(type(node), '_settings_only', False): + return True + node = getattr(node, '__dict__', {}).get('_parent') + return False + + class Variable(_VariableInit, _VariableArithmetic, Component): # ``__eq__`` is overridden on ``_VariableArithmetic`` to return a # comparison Variable (formula building), not a bool. Python drops @@ -175,6 +251,18 @@ class Variable(_VariableInit, _VariableArithmetic, Component): #: Does this value depend on ``mo.time`` (itself, or through the #: unnamed intermediates it is built from)? Set with the expression. _reads_time: bool = False + #: Were its operands read by DATE onto its window (``_start``) — two + #: windows met in an operator or a function that is one call per + #: period? A renderer then reads each operand's cell of the same date; + #: a value on a window of its own read otherwise (``start=``, a + #: re-grain, a copy) keeps its operands' positions. + _read_by_date: bool = False + #: Where ``_source_code`` comes from when no text was written on the + #: instance (see :class:`_SourceCode`): the Variable's own formula, or + #: the text the Variables of its list share. + _source_is_formula: bool = False + _source_items: Optional[Tuple['Variable', ...]] = None + _source_code = _SourceCode() """ Pure variable - computation logic only @@ -302,7 +390,7 @@ def __init__(self, value=None, value_type: str = "float", var_type: str = "scala built by the operator methods and ``__init__`` is called with ``formula=``. """ - # The 24 fields every Variable starts from, in one C-level write + # The fields every Variable starts from, in one C-level write # (``_VARIABLE_DEFAULTS``): the body below writes a field only when # its value is NOT the default. Per-field writes through # ``__setattr__`` dominated the cost of a Variable(), and every @@ -372,11 +460,27 @@ def __init__(self, value=None, value_type: str = "float", var_type: str = "scala # The ``.path`` property below synthesizes a floating-namespace # path from ``id(self)`` until adoption crystallizes a rooted # path. - super().__init__( - display_name=display_name, - excel_props=excel_props, - description=description, - ) + if (display_name is None and excel_props is None and description is None + and type(self) is Variable): + # What ``Component.__init__`` (with ``Base.__init__``) writes + # when given nothing, in its order: empty identity fields and a + # fresh props dict. Every operator result is built this way, and + # walking that chain — a ``__setattr__`` call per field — was + # over half the cost of a ``Variable()``. Keep the two in step + # (``test_component.py`` compares them). A subclass may extend + # the chain, so it always walks it. + fields = self.__dict__ + fields['_qualified_id'] = None + fields['_python_name'] = None + fields['_display_name'] = None + fields['_description'] = None + fields['_excel_props'] = {} + else: + super().__init__( + display_name=display_name, + excel_props=excel_props, + description=description, + ) # Opaque layout-config attribute. Writer reads ``.format`` off it # to apply per-cell number_format overrides. ``None`` is the # inherit-from-parent signal for a resolver that walks the tree. @@ -392,8 +496,8 @@ def __init__(self, value=None, value_type: str = "float", var_type: str = "scala # ``_raw_formula_str`` (the fallback string form for formulas # passed or assigned as raw strings), ``pyformula`` (the captured # rendering of a ``pyformula=`` input, which marks the Variable as - # an input) and ``_source_code`` (a friendlier source snippet the - # ``functions/*`` helpers set) all start as None. + # an input) all start as None; ``_source_code`` (a friendlier source + # snippet the ``functions/*`` helpers set) is derived on read. # _dependency_refs mirrors _expr.iter_refs() (with dedup); populated by # construction sites that build an AST. Kept as a cached list rather # than a live property so callers don't re-walk the tree on every read. @@ -537,6 +641,14 @@ def __init__(self, value=None, value_type: str = "float", var_type: str = "scala self._unit = value._unit if getattr(value, '_unit_declared', False): self._unit_declared = True + # Nor its keys or axes: ``mo.Variable(growth * 2, display_name=…)`` + # still runs along the keys the product ran along. + wrapped_keys = getattr(value, '_keys', None) + if self._keys is None and wrapped_keys: + self._keys = list(wrapped_keys) + self._keys_source = getattr(value, '_keys_source', None) + if not self._indexed_by and getattr(value, '_indexed_by', None): + self._indexed_by = tuple(value._indexed_by) if (start is None) != (grain is None): raise ValueError( "A located Variable needs both `start` and `grain` (or neither, " @@ -638,7 +750,6 @@ def time(self) -> Optional['TimeLoc']: # unchanged into any projection — regardless of a model default. # A FIBERED value with list tracks is a time series too — time # lives INSIDE each track (tracks outside, periods inside). - from .tracks import TrackValues if isinstance(self._value, TrackValues): if self._value.time_length is None: return None @@ -648,9 +759,15 @@ def time(self) -> Optional['TimeLoc']: node = getattr(self, '_owner', None) or getattr(self, '_parent', None) while node is not None and id(node) not in seen: seen.add(id(node)) - grain = getattr(node, 'default_grain', None) + # A container's window is set on the instance; a container + # without one is asked without its component fallback. + slots = node.__dict__ if hasattr(node, '__dict__') else {} + grain = slots.get('default_grain') + if grain is None and 'default_grain' not in slots: + grain = getattr(type(node), 'default_grain', None) + if not isinstance(grain, str): + grain = None if grain is not None: - slots = node.__dict__ if hasattr(node, '__dict__') else {} return TimeLoc(getattr(node, 'default_start', None), grain, slots.get('_default_first_day')) node = getattr(node, '_owner', None) or getattr(node, '_parent', None) @@ -672,6 +789,10 @@ def _on_adopted(self) -> None: Without it, a 3-element list in a 24-period model is accepted silently and detonates at an unrelated line. + Before either, the formulas placed through this line while it had + no window (``variable_ops._forget_placements``) are placed again: + it runs on a window now, or on its container's once that has one. + Idempotent; a no-op while the window is unresolvable (floating subtrees get their check at crystallization). """ @@ -679,6 +800,13 @@ def _on_adopted(self) -> None: owner = getattr(self, '_owner', None) or getattr(self, '_parent', None) if owner is None: return + if '_placed_through' in self.__dict__: + _forget_placements(self) + if _under_settings(owner): + # A view's settings (``timeline={'totals': ['quarter', 'year']}``) + # are words, not a series of the window: their length is not + # checked against the window. + return if not isinstance(self._value, _TrackValuesType()) and _typed_dates_line(self): # ``inputs.close = mo.DATE(2026, 1, 1)``: the line is a date # typed in, written to the book as its value - not a formula. @@ -815,12 +943,13 @@ def _on_adopted(self) -> None: # A provisionally-materialized variable still owes this pass: # its names came from the kwargs, and only the declaration can # say whether they are TRACKS or typos. + tracks_decl = None if self._role_kwargs is not None and ( not self._roles_materialized or self._roles_provisional): from .tracks_decl import resolve_tracks_decl label = (self._display_name or getattr(self, '_python_name', None) or 'variable') - decl = resolve_tracks_decl(owner) + decl = tracks_decl = resolve_tracks_decl(owner) if decl is None: # The window and the declaration may live on DIFFERENT # ancestors: a floating subtree with its own window @@ -873,8 +1002,10 @@ def _on_adopted(self) -> None: ) # --- live synthesis — AFTER the extent law, so a # ragged track dies with the teaching error, never a raw - # IndexError from the splice loop. - self._synthesize_blend(owner, window) + # IndexError from the splice loop. The declaration resolved + # above (when this line owed the role pass) still holds: + # nothing since has touched the containers. + self._synthesize_blend(owner, window, tracks_decl) return # --- extent law --- @@ -1024,7 +1155,7 @@ def _bind_track(self, name: str, operand: Any) -> None: # A binding changes the source data — the synthesized blend # track must re-splice from the new tracks. if window is not None and window.grain is not None: - self._synthesize_blend(owner, window) + self._synthesize_blend(owner, window, decl) def _validate_tracks_against(self, decl, label: str) -> None: """Every authored track name must be declared.""" @@ -1181,17 +1312,24 @@ def _materialize_roles(self, names, label: str) -> None: if self._value.time_length is not None: self.var_type = 'list' - def _entered_mask(self, role, periods: int, shown: bool = False): + def _entered_mask(self, role, periods: int, shown: bool = False, + readings=None): """Per period, was track ``role`` - a formula - entered? None when its own blanks answer (a track entered by hand) or it reads no entries. A formula is a number even over months nobody entered (a blank counts as zero), so it is asked of the entries it is built from (``core.entries``). ``shown``: read through a lag as well — the mask - a derived line keeps as ``_blend_entered``.""" + a derived line keeps as ``_blend_entered``. ``readings``: the lines' + readings shared across the model (``entries.readings_under``).""" from .tracks import TrackValues tv = self._value if (not isinstance(role, str) or not isinstance(tv, TrackValues) - or role not in tv.roles or not isinstance(tv[role], list)): + or role not in tv or not isinstance(tv[role], list)): + return None + if isinstance((self._role_kwargs or {}).get(role), list): + # Entered as a list - of numbers, or a row of references read + # cell by cell: its own blanks answer (``track_expr`` would + # only look every cell over to say so). return None authored = self.track_expr(role) if authored is None and role in (self._role_kwargs or {}): @@ -1201,11 +1339,12 @@ def _entered_mask(self, role, periods: int, shown: bool = False): return None from .entries import entered_mask, entry_lines, shown_entered_mask if shown: - return shown_entered_mask(expr, role, periods, own=authored is not None) - entries = entry_lines(expr, role, own=authored is not None) + return shown_entered_mask(expr, role, periods, own=authored is not None, + readings=readings) + entries = entry_lines(expr, role, own=authored is not None, readings=readings) return entered_mask(entries, periods) if entries else None - def _synthesize_blend(self, owner, window) -> None: + def _synthesize_blend(self, owner, window, decl=None) -> None: """Grow the blend's synthesized track on a DATA line. ``live[t] = given[t] if t ≤ boundary else follow[t]`` — with a @@ -1218,10 +1357,10 @@ def _synthesize_blend(self, owner, window) -> None: after-boundary tail continues the inherited live, not the follow track. Scalar and length-1 tracks splice as constants. Lines with their own start=/grain= are skipped (their positions are not the - window's). + window's). ``decl``: the tracks declaration the caller resolved + for ``owner`` already; resolved here when not given. """ from .time import _parse, _period_labels - from .tracks import TrackValues from .tracks_decl import resolve_tracks_decl self._blend_entered = None @@ -1235,7 +1374,8 @@ def _synthesize_blend(self, owner, window) -> None: return label = (self._display_name or getattr(self, '_python_name', None) or 'variable') - decl = resolve_tracks_decl(owner) + if decl is None: + decl = resolve_tracks_decl(owner) b = getattr(decl, 'blend', None) if decl is not None else None # A copy carried into a context that no longer synthesizes its @@ -1243,7 +1383,7 @@ def _synthesize_blend(self, owner, window) -> None: # stale series instead of detonating the mismatch law there. if self._blend_name is not None and ( b is None or b.name != self._blend_name): - if self._blend_name in tv.roles: + if self._blend_name in tv: shed = tv.as_dict() shed.pop(self._blend_name, None) if shed: @@ -1258,10 +1398,12 @@ def _synthesize_blend(self, owner, window) -> None: # A derived line computes its live value from its operands' # live values; its actual is still a formula that shows 0 over # months nobody entered; this mask records which were entered. - given0 = tv[b.given] if b.given in tv.roles else None + given0 = tv[b.given] if b.given in tv else None if isinstance(given0, list): - self._blend_entered = self._entered_mask(b.given, len(given0), - shown=True) + from .entries import readings_under + self._blend_entered = self._entered_mask( + b.given, len(given0), shown=True, + readings=readings_under(owner)) # A DERIVED line normally inherits live through the # broadcast — but one computed inside a CLASS BODY ran # before its operands had live (provisional @@ -1272,12 +1414,11 @@ def _synthesize_blend(self, owner, window) -> None: # (recurrence/lag) are NOT splice-equal; they stay without # live and the mismatch law surfaces any cross-use loudly. tv0 = self._value - from .tracks import TrackValues as _TV - if not isinstance(tv0, _TV): + if not isinstance(tv0, TrackValues): return - decl0 = resolve_tracks_decl(owner) if True else None - b0 = getattr(decl0, 'blend', None) if decl0 is not None else None - if b0 is None or b0.name in tv0.roles: + # The declaration resolved above still holds: nothing since + # has touched the containers. + if b.name in tv0: return from .expr import SelfRef, FuncCall, MethodCall def _stateful(node) -> bool: @@ -1316,7 +1457,7 @@ def _stateful(node) -> bool: # spelling lands as a role in the value with no synthesis flag. authored = ( (self._role_kwargs is not None and b.name in self._role_kwargs) - or (self._expr is None and b.name in tv.roles + or (self._expr is None and b.name in tv and self._blend_name != b.name) ) if authored: @@ -1327,17 +1468,22 @@ def _stateful(node) -> bool: ) if window.start is None or window.periods is None: return # declaration-time validation demands a full window - labels = _period_labels( - window.start, None, window.periods, window.grain - ) + periods = window.periods if b.until is None: # Boundary-less splice: ``given`` wins wherever it carries a # value, ``follow`` fills the rest. Same rule the bounded # form applies BEFORE its boundary — with no date to keep # current, which suits a given track whose values arrive - # irregularly and incompletely. - before_boundary = [True] * len(labels) + # irregularly and incompletely. No label is compared, so + # none is spelled: the window counts its periods, and its + # start is parsed as the labels would have parsed it, so a + # malformed one fails the same way. + _parse(window.start, window.grain) + before_boundary = [True] * periods else: + labels = _period_labels( + window.start, None, periods, window.grain + ) try: until_anchor = _parse(b.until, window.grain) except (ValueError, TypeError) as exc: @@ -1357,7 +1503,7 @@ def _stateful(node) -> bool: # override spelling) continues its INHERITED live after the # boundary — the follow track would replay the output splice # this law exists to kill. - prior_live = tv[b.name] if b.name in tv.roles else None + prior_live = tv[b.name] if b.name in tv else None if self._expr is not None and prior_live is not None: follow = prior_live else: @@ -1392,53 +1538,62 @@ def _in_record() -> bool: self._blend_name = b.name return - def _cell(track, t): + def _cells(track) -> list: + """One side, a cell per period: a missing side is holes + throughout, a short list ends in holes.""" if track is None: - return None + return [None] * periods if isinstance(track, list): if len(track) == 1: - return track[0] # length-1 broadcasts, like a scalar - return track[t] if t < len(track) else None - return track # scalar splices as a constant + return track * periods # length-1 broadcasts, like a scalar + if len(track) >= periods: + return track + return track + [None] * (periods - len(track)) + return [track] * periods # scalar splices as a constant + + from .entries import readings_under + readings = readings_under(owner) def _entered(role, series): """Per period: was this side entered? (``_entered_mask``)""" if series is None: return None - return self._entered_mask(role, len(labels)) + return self._entered_mask(role, periods, readings=readings) from .regrain import hole_value - blank = hole_value(tv) - masks = { - 'given': _entered(b.given, given), - 'follow': (None if follow is prior_live - else _entered(b.follow, follow)), - } + blank: list = [] # what neither side gives counts as, once a cell asks + given_mask = _entered(b.given, given) + follow_mask = (None if follow is prior_live + else _entered(b.follow, follow)) # Per period, was the given side - a formula - entered? Kept as # ``_blend_entered``: the formula's number alone cannot tell an # entered zero from a month nobody entered. - self._blend_entered = masks['given'] + self._blend_entered = given_mask + given_cells = _cells(given) + follow_cells = _cells(follow) live = [] - for t in range(len(labels)): + for t in range(periods): if before_boundary[t]: - first, second, mask = given, follow, masks['given'] + first, second, mask = given_cells, follow_cells, given_mask else: - first, second, mask = follow, given, masks['follow'] - cell = _cell(first, t) + first, second, mask = follow_cells, given_cells, follow_mask + cell = first[t] if cell is None or (mask is not None and not mask[t]): - cell = _cell(second, t) + cell = second[t] if cell is None: - cell = blank # neither side: the formula's 0, as in the book + if not blank: + blank.append(hole_value(tv)) + cell = blank[0] # neither side: the formula's 0, as in the book live.append(cell) merged = tv.as_dict() merged[b.name] = live # Length-1 lists broadcast (the extent law's "single value" # case); the synthesized full-window track would violate the # equal-length law beside them — materialize the broadcast. - if window.periods > 1: + if periods > 1: for role, track in list(merged.items()): if isinstance(track, list) and len(track) == 1: - merged[role] = track * window.periods + merged[role] = track * periods self._value = TrackValues(merged) self._blend_name = b.name if self.var_type != 'list': @@ -1633,12 +1788,16 @@ def at(self, grain: Optional[str] = None, *, # slice even before adoption. if getattr(self, '_owner', None) is not None: sliced.__dict__['_owner'] = self._owner + # The slice runs on the container's window without being + # one of its lines: a window set on the container later + # moves it all the same (``multi_variable``). + self._owner.__dict__['_sliced'] = True if grain is not None: return sliced.at(grain) return sliced if grain is None: raise TypeError( - "at() needs a grain ('quarter'/'year') and/or a " + "at() needs a grain ('quarter'/'half'/'year') and/or a " "track= coordinate." ) if isinstance(self._value, TrackValues): @@ -1840,9 +1999,12 @@ def source_code(self) -> Optional[str]: ``functions/`` helpers (financial, recurrence, cohort, …) set this for their results; plain inputs and operator-built - intermediates leave it ``None``. Inspection tools read it via - this property when they want the concrete call shape rather - than the AST view. + intermediates leave it ``None``. The condition family (``IF``, + ``AND``, ``OR``, ``NOT``, ``CHOOSE`` and the logical helpers) + keeps no snapshot: its text is :attr:`formula` rendered when it is + read, so it follows a later rename or a formula set by hand, as + ``.formula`` does. Inspection tools read it via this property + when they want the concrete call shape rather than the AST view. Returns ``None`` when no call-site snapshot was captured. """ return self._source_code @@ -1854,6 +2016,13 @@ def source_code(self, value: Optional[str]) -> None: """ self._source_code = None if value is None else str(value) + def _source_from_formula(self) -> None: + # ``_source_code`` reads as the formula, rendered when read (see + # ``_SourceCode``). A text written before (``elementwise`` stamps + # the resolved call) would win over the rule, so it goes. + self.__dict__.pop('_source_code', None) + self.__dict__['_source_is_formula'] = True + @property def is_formula(self) -> bool: """True when this Variable carries a formula (AST or raw string). @@ -1882,24 +2051,33 @@ def expr(self, value: Optional[Expr]) -> None: Construction sites and operator methods assign via this setter so ``_expr`` and ``_dependency_refs`` stay consistent. """ - self._expr = value + # Plain instance fields: no class declares a descriptor for any of + # them, so they go straight into ``__dict__`` — what ``__setattr__`` + # does with them, minus a Python-level call per field. Every + # operator result gets its expression through here. + fields = self.__dict__ + fields['_expr'] = value if value is None: - self._dependency_refs = [] - self._reads_time = False + fields['_dependency_refs'] = [] + fields['_reads_time'] = False return seen = set() refs: List['Variable'] = [] for var in value.iter_refs(): - if id(var) not in seen: - seen.add(id(var)) + key = id(var) + if key not in seen: + seen.add(key) refs.append(var) - self._dependency_refs = refs - from .expr import Regrain, TimeRef + fields['_dependency_refs'] = refs # A re-grained value is computed FOR its own window — whatever time # its native source read lives in the source's window. - self._reads_time = isinstance(value, TimeRef) or ( - not isinstance(value, Regrain) - and any(getattr(r, '_reads_time', False) for r in refs)) + reads_time = isinstance(value, TimeRef) + if not reads_time and not isinstance(value, Regrain): + for r in refs: + if getattr(r, '_reads_time', False): + reads_time = True + break + fields['_reads_time'] = reads_time # A formula over a row that is not assigned yet computes a # provisional value: remember what it waits on, and let the # placeholder find it when the loop closes. @@ -1911,7 +2089,7 @@ def expr(self, value: Optional[Expr]) -> None: elif var._awaits: awaits = awaits or set() awaits.update(var._awaits) - self._awaits = awaits + fields['_awaits'] = awaits if awaits: for placeholder in awaits: dependents = placeholder._forward_dependents @@ -1923,7 +2101,10 @@ def _set_expr(self, expr: Expr) -> None: existing call sites (``x._set_expr(node)``) that would otherwise need a mechanical rewrite. New code should use ``x.expr = node``. """ - self.expr = expr + # ``expr`` is a property, which ``__setattr__`` hands straight to + # ``object.__setattr__``; going there directly skips a Python-level + # call on every operator. + _object_setattr(self, 'expr', expr) def _as_operand(self, parenthesize: bool = False) -> Expr: """Return this Variable as an operand in a larger expression. @@ -2147,6 +2328,8 @@ def shift(self, periods: int = 1, fill_value=0.0) -> 'Variable': original_fill_value = fill_value if isinstance(fill_value, Variable): + from ..functions.lag import check_fill + check_fill(fill_value, "shift(..., fill_value=") fill_value = fill_value._value if periods > 0: @@ -2168,7 +2351,12 @@ def shift(self, periods: int = 1, fill_value=0.0) -> 'Variable': # ``DATE(...)`` for dates, a bare number for numerics. Stringifying # here would lose the type and mis-render (quoted text / bare date). kwargs: Dict[str, Any] = {} - if not (isinstance(original_fill_value, float) and original_fill_value == 0.0): + if isinstance(original_fill_value, Variable): + # A line's fill stays a reference to its cell, as mo.lag keeps + # it: the workbook reads it live, and a loop container checks + # its unit and settles the row by it when it closes. + kwargs["fill_value"] = VarRef(original_fill_value) + elif not (isinstance(original_fill_value, float) and original_fill_value == 0.0): kwargs["fill_value"] = fill_value result = Variable() @@ -2176,6 +2364,16 @@ def shift(self, periods: int = 1, fill_value=0.0) -> 'Variable': result._value = shifted result.var_type = 'list' result._keys = list(self._keys) if self._keys is not None else None + # mo.lag's unit law, the one a loop container derives the row by + # again: the source's unit, a fill in a unit is in the source's. + from ..functions.lag import lag_unit + try: + unit = lag_unit(self, original_fill_value, result) + except ValueError as exc: + raise ValueError(str(exc).replace("mo.lag takes", "shift() takes", 1) + .replace("in mo.lag,", "in .shift(),", 1)) from None + if unit is not None: + result._unit = unit return result @@ -2317,15 +2515,45 @@ def _get_by_index(self, key): For a scalar source, only ``[0]`` is valid (a convenience so arithmetic producing a scalar still looks list-ish). Non-zero - indices on a scalar raise ``IndexError``. + indices on a scalar raise ``IndexError``. A tracked row's period + is that period of every track — one live cell per track. """ + if isinstance(self._value, TrackValues): + # ``closing[-1]`` of a plan/actual row: each track's own last + # period (a scalar track is the same in every period), written + # as one reference per track. It used to raise (an index on a + # scalar) or, for ``[0]``, copy the whole row as numbers. + from .regrain import hole_value + blank = hole_value(self._value) + picked: Dict[str, Any] = {} + for role, track in self._value.items(): + cell = track[key] if isinstance(track, list) else track + if cell is None: + # A period not entered yet (an actual still to come) + # reads as a blank cell does in a formula: 0 — but a + # line of dates or words has no zero. + if blank is None: + raise IndexError( + f"Period {key} of {self.id!r} is blank in its " + f"{role!r} track, and a line of dates or words " + f"has no zero to stand in for it." + ) + cell = blank + picked[role] = cell + result = Variable() + result._set_expr(Subscript(VarRef(self), key)) + result._value = TrackValues(picked) + result.var_type = 'scalar' + result.value_type = self.value_type + result._unit = self._unit + return result if not isinstance(self._value, list): if key == 0: return Variable(value=self._value, var_type=self.var_type, value_type=self.value_type) raise IndexError( f"Index {key} on a scalar Variable — only ``[0]`` is valid " - f"(as a convenience for list-shaped APIs). " - f"Use ``var.value`` to read the scalar directly." + f"(as a convenience for list-shaped APIs). A scalar row is " + f"one cell: read it as it is, by its name." ) result = Variable() diff --git a/src/modeleon/core/variable_init.py b/src/modeleon/core/variable_init.py index b4630ea..fb91c1c 100644 --- a/src/modeleon/core/variable_init.py +++ b/src/modeleon/core/variable_init.py @@ -14,7 +14,7 @@ from __future__ import annotations -from typing import Any, Dict, FrozenSet, List, Optional, TYPE_CHECKING +from typing import Any, Dict, FrozenSet, List, Optional, Tuple, TYPE_CHECKING from .expr import Expr, ListExpr, Literal, VarRef @@ -65,6 +65,7 @@ class _VariableInit: _EXCEL_PROP_KEYS: FrozenSet[str] pyformula: Optional[str] _source_code: Optional[str] + _source_items: Optional[Tuple["Variable", ...]] _value: Any _expr: Optional[Expr] _raw_formula_str: Optional[str] @@ -108,8 +109,10 @@ def _init_from_formula(self, formula, var_type: str) -> str: self._dependency_refs = list(formula._dependency_refs) else: self._set_expr(VarRef(formula)) - if formula._source_code is not None: - self._source_code = formula._source_code + # One read: the source text may be rendered on read. + source_code = formula._source_code + if source_code is not None: + self._source_code = source_code return var_type if isinstance(formula, list) and len(formula) > 0: @@ -179,8 +182,18 @@ def _init_from_value(self, value, var_type: str, value_type: str) -> tuple[str, else: self._set_expr(VarRef(value)) self._value = _blanks_read_as_zero(value._value) - if value._source_code is not None: - self._source_code = value._source_code + # A copy of a line still waiting on its loop waits with it, and + # one whose loop failed fails with it: its numbers are + # provisional, never a value to hand out. + awaits = getattr(value, '_awaits', None) + if awaits and not getattr(self, '_awaits', None): + setattr(self, '_awaits', set(awaits)) # declared on Variable + if getattr(value, '_loop_error', None): + self._loop_error = value._loop_error + # One read: the source text may be rendered on read. + source_code = value._source_code + if source_code is not None: + self._source_code = source_code return var_type, value_type if (isinstance(value, list) and len(value) > 0 @@ -211,13 +224,13 @@ def _build_compound_formula(self, items: list) -> None: expr_items.append(Literal(item)) self._set_expr(ListExpr(expr_items)) - # Copy _source_code if every Variable item shares the same one - var_items = [v for v in items if isinstance(v, Variable)] - if var_items and all(v._source_code for v in var_items): - # ``all(...)`` above guarantees every entry is a non-empty str. - source_codes: List[str] = [v._source_code for v in var_items] # type: ignore[misc] - if len(set(source_codes)) == 1: - self._source_code = source_codes[0] + # The compound's source text is the one its Variable items share, + # decided when someone reads it (``Variable._source_code``): an + # item's own text may be a formula rendered on read, and a list of + # conditions is a line nobody asks the source of in the usual run. + var_items = tuple(v for v in items if isinstance(v, Variable)) + if var_items: + self._source_items = var_items @staticmethod def _coerce_literal(value, value_type: str): diff --git a/src/modeleon/core/variable_ops.py b/src/modeleon/core/variable_ops.py index 690a579..ba30715 100644 --- a/src/modeleon/core/variable_ops.py +++ b/src/modeleon/core/variable_ops.py @@ -18,24 +18,47 @@ this class so the operator surface is available everywhere Variable is, while the file on disk stays coherent. -Lazy imports for :class:`Variable` inside each method avoid the -circular import that would arise if we imported at module scope -(``variable.py`` imports this module; this module needs Variable to -construct results). +:class:`Variable` is imported lazily, once (``_variable_class``), to +avoid the circular import that would arise if we imported at module +scope (``variable.py`` imports this module; this module needs Variable +to construct results). """ from __future__ import annotations +import functools +import math +import os +import weakref from datetime import date, datetime, time, timedelta -from typing import Any, List, Optional, Tuple, TYPE_CHECKING +from typing import Any, List, Optional, Sequence, Tuple, TYPE_CHECKING from .expr import BinOp, Compare, Expr, Literal, UnaryOp, VarRef +from .shape import keys_along +from .tracks import TrackValues if TYPE_CHECKING: from .unit import Unit from .variable import Variable +#: :class:`~modeleon.core.variable.Variable`, bound on first use. +#: ``variable.py`` imports this module before it defines the class, so it +#: cannot be imported at load time; a function-level import, though, looks +#: the module up again on every call, which costs more than the arithmetic +#: of a scalar operator. Every operator below reads it. +_Variable: Optional[type[Variable]] = None + + +def _variable_class() -> type[Variable]: + """The :class:`Variable` class, imported once (see ``_Variable``).""" + global _Variable + if _Variable is None: + from .variable import Variable + _Variable = Variable + return _Variable + + # Excel's day 0: a date's serial number counts the days since then. _EXCEL_EPOCH_ORDINAL = date(1899, 12, 30).toordinal() _ONE_DAY = timedelta(days=1) @@ -140,13 +163,18 @@ def _date_serial_op(a: Any, b: Any, op: Any, op_symbol: Optional[str]) -> Any: return '#VALUE!' +def _on_open_loop(v: Any) -> bool: + """A row read before its line, or built on one: its loop has not + closed (see :mod:`modeleon.core.loops`), so its unit is provisional.""" + return bool(getattr(v, '_awaits', None)) or getattr(v, '_forward_of', None) is not None + + def _provisional_unit(unit_fn: Any, left: Any, right: Any, *operands: Any) -> Any: """``unit_fn(left, right)`` — except over a row whose loop has not - closed (see :mod:`modeleon.core.loops`): its unit is not known yet, - so a mismatch is not an error here; the loop derives every unit - again when it closes, and reports a real mismatch there.""" - if any(getattr(v, '_awaits', None) or getattr(v, '_forward_of', None) is not None - for v in operands): + closed: its unit is not known yet, so a mismatch is not an error + here; the loop derives every unit again when it closes, and reports + a real mismatch there.""" + if any(_on_open_loop(v) for v in operands): try: return unit_fn(left, right) except ValueError: @@ -154,7 +182,12 @@ def _provisional_unit(unit_fn: Any, left: Any, right: Any, *operands: Any) -> An return unit_fn(left, right) -_GRAIN_ORDER = {'day': 0, 'month': 1, 'quarter': 2, 'year': 3} +def _grain_rank(grain: Any) -> int: + """The calendar's rank of ``grain`` (finest 0, the hour); an unknown + word ranks with the finest. Read lazily: ``core.time`` imports this module's + neighbours.""" + from .calendar import GREGORIAN + return GREGORIAN.rank(grain) if GREGORIAN.knows(grain) else 0 def _mixed_first_periods(left: Any, lt: Any, right: Any, rt: Any) -> str: @@ -183,15 +216,28 @@ def first(v: Any, t: Any) -> str: def _mixed_grains(left: Any, left_grain: str, right: Any, right_grain: str) -> str: """Two lines of different grains in one formula: name both, their grains, and the re-grain that joins them.""" - adjective = {'day': 'daily', 'month': 'monthly', 'quarter': 'quarterly', - 'year': 'yearly'} + adjective = {'hour': 'hourly', 'day': 'daily', 'week': 'weekly', 'month': 'monthly', + 'quarter': 'quarterly', 'half': 'half-yearly', 'year': 'yearly', + 'iso_year': 'ISO-yearly'} def describe(v: Any, grain: str) -> str: label = (getattr(v, '_display_name', None) or getattr(v, '_python_name', None)) - return f"{label!r} ({grain}s)" if label else f"a {adjective.get(grain, grain)} line" - - finer = left if _GRAIN_ORDER.get(left_grain, 0) < _GRAIN_ORDER.get(right_grain, 0) else right + word = adjective.get(grain, grain) + return f"{label!r} ({grain}s)" if label else f"{'an' if word == 'hourly' else 'a'} {word} line" + + from .calendar import GREGORIAN + if (GREGORIAN.knows(left_grain) and GREGORIAN.knows(right_grain) + and not GREGORIAN.comparable(left_grain, right_grain)): + # Weeks and months: neither rolls up into the other. + return ( + f"{describe(left, left_grain)} and {describe(right, right_grain)} meet " + f"in one formula at grains that do not nest: {left_grain}s straddle " + f"{right_grain}s, so neither line can be re-grained into the other " + f"without splitting a period. Build both at a grain they share " + f"(days), or keep them apart." + ) + finer = left if _grain_rank(left_grain) < _grain_rank(right_grain) else right coarse = right_grain if finer is left else left_grain code = getattr(finer, '_python_name', None) hint = f"{code}.at('{coarse}')" if code else f".at('{coarse}')" @@ -203,11 +249,752 @@ def describe(v: Any, grain: str) -> str: ) +def _onto_window(var: Any, vals: list, off: int, total: int, union_start: str, + grain: str, union_first: Any, other: str) -> list: + """``vals`` of ``var`` materialized onto a combined window of ``total`` + periods from ``union_start``, its own first value ``off`` periods in: + the cells outside its own extent follow its DECLARED ``extend=`` + rule — zero (a flow is absent), hold (a rate stays in force; held + backward from the first value when extended into the past), or + none/undeclared → an error naming both extents and the rules to declare (``other`` + spells the other side's). Zero continues numbers: a line of dates or + text it would continue with 0 is refused, pointing to hold. A time + field is defined in every period, not only the window's.""" + from .expr import TimeRef + time_expr = getattr(var, '_expr', None) + if isinstance(time_expr, TimeRef): + field = time_expr.field + if field == 'index': + return [i - off + 1 for i in range(total)] + from .time import time_field_series + return time_field_series(field, union_start, grain, total, union_first) + rule = getattr(var, '_extend', None) + if off == 0 and len(vals) == total: + return list(vals) + kind = _not_numbers(vals) if rule == 'zero' else None + if kind is not None: + plural, one = kind + raise ValueError( + f"{_line_name(var)} holds {plural}, and extend=mo.zero() would continue it " + f"with 0 — no {one} — in the periods of the combined window it lacks " + f"(it spans {len(vals)} period(s) from {var.time.start}; the other " + f"side spans {other}). Hold its {one} there instead: extend=mo.hold()." + ) + out = [] + for i in range(total): + j = i - off + if 0 <= j < len(vals): + out.append(vals[j]) + elif rule == 'zero': + out.append(0.0) + elif rule == 'hold': + out.append(vals[0] if j < 0 else vals[-1]) + else: + raise ValueError(_uncovered(var, len(vals), var.time.start, other)) + return out + + +def _uncovered(var: Any, n: int, start: str, other: str, reads: bool = False) -> str: + """The refusal of a series with no rule that does not cover the + combined window: a line is told to declare one; an unnamed formula, + which cannot (``reads``: placed by the lines it reads), to become a + line that does.""" + extent = (f"it spans {n} period(s) from {start}" + f"{', the periods of the lines it reads' if reads else ''}; " + f"the other side spans {other}") + named = var._display_name or getattr(var, '_python_name', None) + if named or var._expr is None: + return (f"{_line_name(var)} does not cover the combined window ({extent}). " + f"Say what continues it: extend=mo.zero() (a flow) or extend=mo.hold() " + f"(a rate) — never a silent guess.{_regrained(var)}") + home = _home_of(var) if reads else None + where = (f"a line of {home[0]}, the container {_line_name(home[1])} runs on, " + if home else "a line ") + line = f"{home[0]}. = " if home else "" + return ( + f"{_line_name(var)} does not cover the combined window ({extent}). A formula " + f"carries no extend= rule: make it {where}that says what continues it — " + f"{line}mo.Variable({_formula_code(var)}, extend=mo.zero()) (a flow) or " + f"extend=mo.hold() (a rate) — never a silent guess." + ) + + +def _home_of(var: Any) -> Optional[Tuple[str, Any]]: + """The container a formula's periods come from — the path of the one + the first located line it reads stands in — with that line; None + where none stands in one.""" + pending = [var] + seen = set() + while pending: + node = pending.pop() + for ref in node._expr.iter_refs(): + if id(ref) in seen: + continue + seen.add(id(ref)) + place = _period_ref(ref) + if place == 'formula': + pending.append(ref) + elif place is not None and place != _NO_WINDOW_YET: + owner = getattr(ref, '_owner', None) or _forward_home(ref) + return (str(owner.path), ref) if owner is not None else None + return None + + +def _regrained(var: Any) -> str: + """For a re-grained copy of a line with an ``extend=`` rule — the copy + keeps none: the recipe of a partial coarse period is not the rule's — + the way to read it by date.""" + from .expr import Regrain + expr = getattr(var, '_expr', None) + source = expr.source if isinstance(expr, Regrain) else None + rule = getattr(source, '_extend', None) + if rule not in ('zero', 'hold'): + return "" + return ( + f" A re-grained line keeps no rule of the line it re-grains " + f"({_line_name(source)} declares extend=mo.{rule}()): combine the lines " + f"at their own grain and re-grain the result (.at('{var._grain}')), or " + f"give the re-grained line a rule of its own." + ) + + +def _not_numbers(vals: list) -> Optional[Tuple[str, str]]: + """``('dates', 'date')`` or ``('text', 'text')`` for a series of dates + or text — its first value neither blank nor an error says — else + None.""" + for v in vals: + if v is None or (isinstance(v, str) and v.startswith('#')): + continue + if isinstance(v, (date, datetime)): + return ('dates', 'date') + if isinstance(v, str): + return ('text', 'text') + return None + return None + + +# ─── Lines on other time windows ─────────────────────────────────── +# +# One reading for the operators and for every function that is one call +# per period (``functions._helpers.align_located``): series on other +# windows meet by DATE, each continued by its own ``extend=`` rule. + +#: :func:`_reads_on` of a formula whose periods have no one date. +_NO_ONE_WINDOW = 'no one window' +#: :func:`_place_of` of a series that has no window yet — an input not +#: adopted, a formula over such inputs: read by position until it has one. +_NO_WINDOW_YET = 'no window yet' +_UNREAD = object() +#: Counts the changes of windows (:func:`_windows_changed`): a placement +#: kept on a formula at an earlier count was derived from windows that +#: may have moved since, and is walked again. The count starts at random, +#: so a placement unpickled from another process is never trusted. +_WINDOWS_EPOCH = int.from_bytes(os.urandom(6), 'big') + + +class _Readers(list): + """The formulas placed through a line, held weakly + (:func:`_placed_through`). A copy or a pickle of the line starts with + none, and the placements kept before it are walked again + (:func:`_readers_anew`): a copied formula's placement was derived + through the line's original, whose adoption the copy never hears of.""" + + __slots__ = () + + def __reduce__(self) -> Tuple[Any, ...]: + return (_readers_anew, ()) + + +def _readers_anew() -> "_Readers": + _windows_changed() + return _Readers() + + +@functools.lru_cache(maxsize=4096) +def _starts_on(label: str, grain: str) -> Any: + """The first day of the period ``label`` names: windows are ordered by + date, never as text (``'2026-9'`` comes before ``'2026-10'``).""" + from .calendar import GREGORIAN + return GREGORIAN.ruler(grain).parse(label) + + +def _forward_home(line: Any) -> Any: + """The loop container a row read before its line stands in for (a + placeholder), or None.""" + forward = line.__dict__.get('_forward_of') + return forward[0] if forward else None + + +def _window_of(line: Any) -> Any: + """Where ``line`` runs: its ``time``; for a row of a loop container + read before its line, the container's window, which the row holds a + period of for every period.""" + loc = line.time + if loc is None: + home = _forward_home(line) + if home is not None and isinstance(line._value, list): + from .time import resolve_default_window + from .variable import TimeLoc + window = resolve_default_window(home) + if window is not None and window.grain is not None: + return TimeLoc(window.start, window.grain, window.first_day) + return loc + + +def _series_length(value: Any) -> Optional[int]: + """How many periods a value runs along: a list's length, a tracked + value's track length; None for a scalar.""" + if isinstance(value, list): + return len(value) + if isinstance(value, TrackValues): + return value.time_length + return None + + +def _a_constant(var: Any) -> bool: + """A one-value line on a window of more periods than one: the extent + check admits it as a constant, read in every period as a number is. One located on + its own (``start=``, a re-grain), continued by a rule, or on a + one-period window holds that one period.""" + if var._grain is not None or var._extend is not None or var._schedule is not None: + return False + from .time import resolve_default_window + owner = (getattr(var, '_owner', None) or getattr(var, '_parent', None) + or _forward_home(var)) + window = resolve_default_window(owner) if owner is not None else None + return window is None or window.periods != 1 + + +def _period_ref(ref: Any) -> Any: + """How a line a formula reads places it: ``(loc, n)`` for a series of + ``n`` periods on a window, ``'formula'`` for an unnamed formula read + through, ``_NO_WINDOW_YET`` for an input with no window yet, None for + nothing that runs on a window (a scalar, a line along keys or an + axis, a constant).""" + value = ref._value + n: Optional[int] + if isinstance(value, list): + n = len(value) + if not n or ref._keys: + return None + elif isinstance(value, TrackValues): + n = value.time_length + if not n: + return None + else: + return None + if ref._indexed_by: + return None + loc = _window_of(ref) + if loc is not None and loc.start is not None: + return None if n == 1 and _a_constant(ref) else (loc, n) + return 'formula' if ref._expr is not None else _NO_WINDOW_YET + + +def _same_window(a: Any, b: Any) -> bool: + """Two places, ``(loc, n)``, on one window: one grain, period count + and first day, and starts that name one period.""" + (la, na), (lb, nb) = a, b + if la.grain != lb.grain or na != nb or la.first_day != lb.first_day: + return False + return la.start == lb.start or _starts_on(la.start, la.grain) == _starts_on(lb.start, lb.grain) + + +def _windows_changed() -> None: + """A window moved — a container's ``default_start``, ``default_grain`` + or ``default_periods`` set, a line or a container detached: every + placement kept on a formula (:func:`_reads_on`) is walked again when + it is next read.""" + global _WINDOWS_EPOCH + _WINDOWS_EPOCH += 1 + + +def _kept_placement(var: Any) -> Any: + """The placement kept on the unnamed formula ``var`` (:func:`_reads_on`, + :func:`_keep_runs_on`) — or ``_UNREAD``: none kept, or one kept before + a window moved.""" + kept = var.__dict__.get('_reads_on') + if kept is not None and kept[1] == _WINDOWS_EPOCH: + return kept[0] + return _UNREAD + + +def _placed_through(line: Any, node: Any) -> None: + """Note on ``line`` — a series with no window yet, an unnamed formula + — that the placement kept on ``node`` was derived through it: once + ``line`` is adopted it runs on a window, and ``node`` is placed again + (:func:`_forget_placements`).""" + readers = line.__dict__.get('_placed_through') + if readers is None: + readers = line.__dict__['_placed_through'] = _Readers() + readers.append(weakref.ref(node)) + n = len(readers) + if n >= 64 and not (n & (n - 1)): + # Formulas discarded since, dropped each time the list doubles. + readers[:] = [ref for ref in readers if ref() is not None] + + +def _forget_placements(line: Any) -> None: + """``line`` was adopted: drop the placement kept on every formula + placed through it (:func:`_placed_through`), and through those — each + is walked again, on the window ``line`` runs on now.""" + stack = [line] + seen: set = set() + while stack: + readers = stack.pop().__dict__.pop('_placed_through', None) + if not readers: + continue + for ref in readers: + node = ref() + if node is not None and id(node) not in seen: + seen.add(id(node)) + node.__dict__.pop('_reads_on', None) + stack.append(node) + + +def _scan(node: Any, found: dict, open_: set) -> Tuple[Any, list]: + """One pass over the lines the unnamed formula ``node`` reads: + ``(place, [])`` — its :func:`_reads_on` — or ``(None, pending)``, the + unnamed formulas it reads that are not placed yet (``found``, by id, + or kept on them); one still open (``open_``) reads ``node`` itself + and places nothing. Placed, ``node`` is noted on each line it was + placed through (:func:`_placed_through`).""" + window: Any = None + pending: list = [] + through: list = [] + for ref in node._expr.iter_refs(): + place = _period_ref(ref) + if place == 'formula': + place = found.get(id(ref), _kept_placement(ref)) + if place is _UNREAD: + if id(ref) not in open_: + pending.append(ref) + continue + through.append(ref) + elif place == _NO_WINDOW_YET: + through.append(ref) + continue + if place is None: + continue + if place == _NO_ONE_WINDOW: + window = _NO_ONE_WINDOW + break + if window is None: + window = place + elif place != window and not _same_window(place, window): + window = _NO_ONE_WINDOW + break + if window != _NO_ONE_WINDOW: + if pending: + return None, pending + if window is not None and _series_length(node._value) != window[1]: + window = _NO_ONE_WINDOW + for ref in through: + _placed_through(ref, node) + return window, [] + + +def _reads_on(var: Any) -> Any: + """Where an unnamed formula's periods run: ``(loc, n)``, the window + every located series it reads runs on — through the unnamed formulas + it reads — when they share one and the formula holds one value per + period of it; None when it reads none (its periods are positions + only); ``_NO_ONE_WINDOW`` otherwise (lines of other windows paired by + position, or another count of values). Kept on each formula placed, + so a chain of formulas is walked once — for as long as the windows it + was derived from stand: the placement is dropped when a line it was + derived through is adopted (:func:`_forget_placements`) and walked + again after any window moved (:func:`_windows_changed`), so a formula + is placed the same whatever the order the model was built in.""" + known = _kept_placement(var) + if known is not _UNREAD: + return known + found: dict = {} + open_: set = set() + stack = [var] + while stack: + node = stack.pop() + if id(node) in found: + continue + place, pending = _scan(node, found, open_) + if pending: + open_.add(id(node)) + stack.append(node) + stack.extend(pending) + continue + open_.discard(id(node)) + found[id(node)] = place + if not _on_open_loop(node): + node.__dict__['_reads_on'] = (place, _WINDOWS_EPOCH) + return found[id(var)] + + +def _place_of(line: Any, value: Any, homes: Optional[dict] = None) -> Any: + """Where the series ``value`` of the operand ``line`` runs, to read it + by date: ``(loc, n, own)`` — the window and its period count, the + line's own (``own``) or, for an unnamed formula, the one of the lines + it reads (:func:`_reads_on`); None where it runs on no window (a + scalar, a value typed in, a line along keys or an axis, a constant: + read by position); ``_NO_WINDOW_YET`` for a series that has none yet + (an input not adopted, a formula that reads no located line: read by + position until it has one); ``_NO_ONE_WINDOW`` for a formula whose + periods have no one date. ``homes`` keeps, by container, the window + its lines share, so one call resolves it once.""" + n: Optional[int] + if isinstance(value, list): + n = len(value) + if not n: + return None + elif isinstance(value, TrackValues): + n = value.time_length + if not n: + return None + else: + return None + if not isinstance(line, _Variable or _variable_class()): + return None + if line._indexed_by or (line._keys and isinstance(value, list)): + return None + home = None if line._grain is not None else ( + line.__dict__.get('_owner') or line.__dict__.get('_parent')) + if homes is not None and home is not None: + loc = homes.get(id(home), _UNREAD) + if loc is _UNREAD: + loc = homes[id(home)] = line.time + elif home is None and line._grain is None and line._expr is not None: + # An unnamed formula stands in no container: no window of its own. + loc = _window_of(line) if '_forward_of' in line.__dict__ else None + else: + loc = _window_of(line) + if loc is not None and loc.start is not None: + if n == 1 and _a_constant(line): + return None + return (loc, n, True) + if line._expr is None: + return _NO_WINDOW_YET + read = _reads_on(line) + if read is None: + return _NO_WINDOW_YET + if read == _NO_ONE_WINDOW: + return read + return (read[0], n, False) if read[1] == n else _NO_ONE_WINDOW + + +def _keep_runs_on(result: Any, place: Any, through: Sequence[Any]) -> None: + """Keep on an operator's ``result`` read by position where its periods + run — the window its operands share (:func:`_meet`) — so a formula + built on it is placed without walking back through it; noted on the + operands it was derived ``through`` (:func:`_placed_through`).""" + if place is None: + return + fields = result.__dict__ + if place == _NO_ONE_WINDOW or _series_length(fields['_value']) == place[1]: + fields['_reads_on'] = (place, _WINDOWS_EPOCH) + for line in through: + _placed_through(line, result) + + +def _formula_code(line: Any) -> str: + formula = getattr(line, 'formula', None) + return formula if formula else "..." + + +def _off_its_window(line: Any, loc: Any, n: int, own: bool, other: str) -> str: + """The refusal of a series that holds no rule — an unnamed formula + placed by the lines it reads, a tracked line — and does not cover the + combined window.""" + if own: + return ( + f"{_line_name(line)} holds tracks on {n} period(s) from {loc.start}, " + f"and the other side spans {other}: a tracked line is read on its own " + f"window only — the lines beside it must lie within it." + ) + return _uncovered(line, n, loc.start, other, reads=True) + + +def _no_one_window(line: Any, total: int, start: str) -> str: + return ( + f"{_line_name(line)} has no one window to read by date: it pairs lines of " + f"other windows by position, or holds another count of values than the " + f"lines it reads, while the other lines meet on {total} period(s) from " + f"{start}. Make it a line on the window it runs on — " + f"mo.Variable({_formula_code(line)}, start=..., grain=...)." + ) + + +def _on_dates(lines: Sequence[Any], values: list) -> Tuple[list, Any]: + """``values`` — ``values[i]`` the value of the operand ``lines[i]`` — + read as ``a + b`` reads two windows: each series on a window of its + own materialized onto the combined window by its declared ``extend=`` + rule (:func:`_onto_window`). Returns the values and the combined + window, a ``TimeLoc`` — or the values as given and None when fewer + than two series run on windows, or all on one (the positional zip is + then by date). An unnamed formula, placed by the lines it reads + (:func:`_place_of`), and a tracked line hold no rule: each must cover + the combined window. Two grains, a short first period inside the + combined window and a formula whose periods have no one date are + refused.""" + out, window, _, _ = _meet(lines, values) + return out, window + + +def _meet(lines: Sequence[Any], values: list) -> Tuple[list, Any, Any, list]: + """:func:`_on_dates`, with where a result read by position runs: the + window its dated operands share, ``(loc, n)`` — ``_NO_ONE_WINDOW`` + beside a formula whose periods have no one date, None beside none — + or the combined window when one was formed; and the operands that + placement was derived through (unnamed formulas, series with no + window yet), for :func:`_keep_runs_on`.""" + dated: List[Tuple[int, Any, Any, int, bool]] = [] + adrift: Any = None + through: list = [] + homes: dict = {} + for i, line in enumerate(lines): + place = _place_of(line, values[i], homes) + if place is None: + continue + if place == _NO_WINDOW_YET: + through.append(line) + continue + if place == _NO_ONE_WINDOW: + adrift = line if adrift is None else adrift + through.append(line) + continue + loc, n, own = place + if not own: + through.append(line) + dated.append((i, line, loc, n, own)) + if not dated: + return values, None, (None if adrift is None else _NO_ONE_WINDOW), through + _, first, t0, n0, _ = dated[0] + if all((loc == t0 and n == n0) or _same_window((loc, n), (t0, n0)) + for _, _, loc, n, _ in dated[1:]): + # One window — positions are dates. + return values, None, ((t0, n0) if adrift is None else _NO_ONE_WINDOW), through + days: dict = {} + for _, _, loc, _, _ in dated: + days.setdefault(loc.grain, set()).add(loc.first_day) + if any(len(d) > 1 for d in days.values()): + for j, (_, left, lt, _, _) in enumerate(dated): + for _, right, rt, _, _ in dated[j + 1:]: + if lt.grain == rt.grain and lt.first_day != rt.first_day: + # One side's first period is short. It may open the + # combined window, never sit inside it: on the other + # side the same period starts on another day. + st, ot = (lt, rt) if lt.first_day is not None else (rt, lt) + if (ot.first_day is not None or _starts_on(st.start, st.grain) + >= _starts_on(ot.start, ot.grain)): + raise ValueError(_mixed_first_periods(left, lt, right, rt)) + for _, line, loc, _, _ in dated[1:]: + if loc.grain != t0.grain: + raise ValueError(_mixed_grains(first, t0.grain, line, loc.grain)) + from .time import _count, _parse + from .variable import TimeLoc + grain = t0.grain + starts = [_starts_on(loc.start, grain) for _, _, loc, _, _ in dated] + k0 = min(range(len(dated)), key=starts.__getitem__) + union = dated[k0][2] + anchor = _parse(union.start, grain) + offsets = [0 if s == starts[k0] else _count(anchor, _parse(loc.start, grain), grain) - 1 + for s, (_, _, loc, _, _) in zip(starts, dated)] + total = max(off + n for off, (_, _, _, n, _) in zip(offsets, dated)) + if adrift is not None: + raise ValueError(_no_one_window(adrift, total, union.start)) + out = list(values) + for k, (off, (i, line, loc, n, own)) in enumerate(zip(offsets, dated)): + if len(dated) == 2: + _, _, other_loc, other_n, _ = dated[1 - k] + other = f"{other_n} period(s) from {other_loc.start}" + else: + other = f"{total} period(s) from {union.start} between them" + if not own or isinstance(values[i], TrackValues): + if off or n != total: + raise ValueError(_off_its_window(line, loc, n, own, other)) + continue + out[i] = _onto_window(line, values[i], off, total, union.start, grain, + union.first_day, other) + window = TimeLoc(union.start, grain, union.first_day) + return out, window, window, through + + +def _on_loop_window(var: Any, x: list, row: Any, loop: Any, n: int) -> Optional[list]: + """The series ``x`` of the line ``var``, read inside a loop container + whose rows (``row`` one of them) run on ``loop`` (a ``TimeLoc``) over + ``n`` periods: by DATE, as the operators read it — each period the + line's value of that date, the periods it lacks continued by its own + ``extend=`` rule (:func:`_onto_window`). None where it is read as it + is: on the loop's window, or on none (a constant, a value typed in, a + line along keys). A line without a rule, an unnamed formula that + does not cover the window, another grain and a formula whose periods + have no one date are refused as the operators refuse them.""" + place = _place_of(var, x) + if place is None or place == _NO_WINDOW_YET: + return None + if place == _NO_ONE_WINDOW: + raise ValueError(_no_one_window(var, n, loop.start)) + loc, k, own = place + if _same_window((loc, k), (loop, n)): + return None + grain = loop.grain + if loc.grain != grain: + raise ValueError(_mixed_grains(var, loc.grain, row, grain)) + if loc.first_day != loop.first_day: + # A short first period may open the loop's window, never sit + # inside it (as in :func:`_meet`). + st, ot = (loc, loop) if loc.first_day is not None else (loop, loc) + if ot.first_day is not None or _starts_on(st.start, grain) >= _starts_on(ot.start, grain): + raise ValueError(_mixed_first_periods(row, loop, var, loc)) + from .time import _count, _parse + if _starts_on(loc.start, grain) >= _starts_on(loop.start, grain): + off = _count(_parse(loop.start, grain), _parse(loc.start, grain), grain) - 1 + else: + off = 1 - _count(_parse(loc.start, grain), _parse(loop.start, grain), grain) + other = f"{n} period(s) from {loop.start}, the loop's window" + if own: + return _onto_window(var, x, off, n, loop.start, grain, loop.first_day, other) + if off <= 0 and n - off <= k: + return list(x[-off:n - off]) + raise ValueError(_off_its_window(var, loc, k, own, other)) + + +def _line_name(v: Any) -> str: + """How a message names a line: its display or python name, else its + formula (``mo.time.days`` as typed), else "an unnamed line".""" + label = getattr(v, '_display_name', None) or getattr(v, '_python_name', None) + if label: + return repr(label) + formula = getattr(v, 'formula', None) + if formula: + return f"the line {'mo.' + formula if formula.startswith('time.') else formula}" + return "an unnamed line" + + +def _line_code(v: Any) -> Optional[str]: + """The line as code a message can suggest typing: its python name, + else its formula in brackets; ``None`` when neither exists.""" + name = getattr(v, '_python_name', None) + if name: + return name + formula = getattr(v, 'formula', None) + return f"({formula})" if formula else None + + +def _period_count(series: Any) -> Optional[int]: + """How many periods ``series`` runs along, or ``None`` when it is not a + period series: a list of several values, a list of any length that + is located in time (``start=``/``grain=``, ``extend=``, a schedule — + a one-value located flow still meets every period), or a tracked + line over periods.""" + values = getattr(series, '_value', None) + if isinstance(values, TrackValues): + n = values.time_length + return n if n is not None and n > 1 else None + if not isinstance(values, list): + return None + located = ( + getattr(series, '_grain', None) is not None + or getattr(series, '_extend', None) is not None + or getattr(series, '_schedule', None) is not None + ) + if len(values) > 1 or (located and len(values) >= 1): + return len(values) + return None + + +def _keys_against_periods(keyed: Any, labels: List[Any], series: Any, n: int) -> str: + """A line that runs along keys met a period series in one formula: + name both, show the pairing a cell-by-cell zip would make, give the + fixes that run (one key, the switch, or both lines along the same + keys / axis).""" + shown = ", ".join(str(k) for k in labels) + pairs = f"{labels[0]} with the first period" + if len(labels) > 1: + pairs += f", {labels[1]} with the second" + if len(labels) > 2: + pairs += ", …" + code = _line_code(keyed) + first = labels[0] + if code is None: + pick = f"pick one key of {_line_name(keyed)}" + elif isinstance(first, str) and getattr(keyed, '_keys', None): + pick = f'pick one key, {code}["{first}"]' + else: + pick = f"pick one cell, {code}[0]" + # The series can be given the keys only when it is a bare input a + # person typed; a computed line, a time property or an unnamed line + # gets the switch instead, which reads one key by a number. + typeable = ( + getattr(series, '_python_name', None) and getattr(series, '_expr', None) is None + ) + if getattr(keyed, '_keys', None): + along = f"its keys ({shown})" + literal = ", ".join(f"{k!r}: …" for k in labels) + second = (f"give {_line_name(series)} the same keys, mo.Variable({{{literal}}})" + if typeable else + f"switch with mo.CHOOSE(pick, {code or 'the line'}), one number picking a key") + else: + axis = keyed._indexed_by[0] + axis_code = getattr(axis, '_python_name', None) + axis_label = axis_code or getattr(axis, '_display_name', None) + along = f"the axis {axis_label!r} ({shown})" if axis_label else f"an axis ({shown})" + second = (f"lay {_line_name(series)} along the same axis, " + f"indexed_by=[{axis_code or '…'}]" + if typeable else + f"switch with mo.CHOOSE(pick, {code or 'the line'}), one number picking a cell") + values_word = "1 value" if n == 1 else f"{n} values" + return (f"{_line_name(keyed)} runs along {along} and {_line_name(series)} runs along " + f"periods ({values_word}): paired cell by cell ({pairs}) the numbers would " + f"look right and be wrong. {pick[0].upper() + pick[1:]}; or {second}.") + + +def _keys_against_keys(left: Any, lk: List[Any], right: Any, rk: List[Any]) -> str: + """A keyed list met a line on an axis whose labels differ: nobody + aligns the pair (keys align with keys, axes with axes), so a zip by + position would pair unrelated labels.""" + def along(v: Any, labels: List[Any]) -> str: + shown = ", ".join(str(k) for k in labels) + if getattr(v, '_keys', None): + return f"its keys ({shown})" + axis = v._indexed_by[0] + name = getattr(axis, '_python_name', None) or getattr(axis, '_display_name', None) + return f"the axis {name!r} ({shown})" if name else f"an axis ({shown})" + return (f"{_line_name(left)} runs along {along(left, lk)} and {_line_name(right)} runs " + f"along {along(right, rk)}: paired cell by cell the labels would not match. " + f"Lay both lines along the same axis (indexed_by=[…]), or give both the same " + f"keys in the same order.") + + #: Unit words read as a year or a month, for the per-period hint. _YEAR_WORDS = frozenset({"year", "years", "yr", "yrs", "y", "год", "года", "лет"}) _MONTH_WORDS = frozenset({"month", "months", "mo", "mon", "мес", "месяц", "месяца"}) +def _month_and_year(left: Any, right: Any, left_over: dict, right_over: dict) -> Optional[str]: + """The hint for two units that differ only in a month word against a + year word at the same power (months / yrs, USD/month / USD/yr): a + label never rescales, so the step between them is written out.""" + if len(left_over) != 1 or len(right_over) != 1: + return None + (lw, lexp), = left_over.items() + (rw, rexp), = right_over.items() + if lexp != rexp or lexp not in (1, -1): + return None + if lw.lower() in _MONTH_WORDS and rw.lower() in _YEAR_WORDS: + months, years = str(left), str(right) + elif lw.lower() in _YEAR_WORDS and rw.lower() in _MONTH_WORDS: + months, years = str(right), str(left) + else: + return None + to_years, to_months = ("/ 12", "* 12") if lexp == 1 else ("* 12", "/ 12") + return ( + f" A unit is a label and never rescales: a line in {months} does not " + f"become {years} when its numbers are divided or multiplied by 12. " + f"Write the step: (x {to_years}).set_unit('{years}') for {months} " + f"into {years}, or (x {to_months}).set_unit('{months}') for {years} " + f"into {months}." + ) + + def _incompatible_units(left: Any, right: Any) -> str: """Why two units do not add — and, when one side is the other times a word or two, that those words are dimensions: a string unit always @@ -227,6 +1014,9 @@ def what(u: Any) -> str: right_over = {k: v for k, v in rc.items() if lc.get(k) != v} smaller, larger = ((lc, right_over) if not left_over else (rc, left_over) if not right_over else (None, None)) + convert = ("convert one side to the other's unit before the operation, " + "or drop the unit on one side if the arithmetic is intentional.") + hinted = True if (smaller and larger and all(v == 1 for v in larger.values()) and not set(larger) & set(smaller)): words = sorted(larger) @@ -261,11 +1051,92 @@ def what(u: Any) -> str: ) else: message += "." - return message + ( - " Otherwise convert one side to the other's unit before the " - "operation, or drop the unit on one side if the arithmetic is " - "intentional." - ) + elif (time_hint := _month_and_year(left, right, left_over, right_over)): + message += time_hint + convert = "drop the unit on one side if the arithmetic is intentional." + else: + hinted = False + # 'Otherwise' only follows a hint; alone, the advice opens the sentence. + return message + (f" Otherwise {convert}" if hinted + else f" {convert[0].upper()}{convert[1:]}") + + +#: Words that label a count of a small step - basis points, percentage +#: points. A label never rescales: 50 bps is written 0.005, 2 pp is 0.02. +_STEP_WORDS = {'bps': ('50 bps', '0.005', '10_000'), 'pp': ('2 pp', '0.02', '100')} + + +def _numbers(value: Any) -> list: + """Every number in a value - a scalar, a list, each track's cells.""" + if isinstance(value, TrackValues): + return [n for v in value._tracks.values() for n in _numbers(v)] + if isinstance(value, (list, tuple)): + return [n for v in value for n in _numbers(v)] + if isinstance(value, bool) or not isinstance(value, (int, float)): + return [] + return [value] + + +def _own_numbers(var: Any) -> list: + return _numbers(var._value) + + +def _holds_a_count(var: Any, numbers: Any = None) -> bool: + """A bps or pp line holding a count (50 where 0.005 is meant). + ``numbers`` reads the numbers a line is judged by (all of its own by + default).""" + unit = var._get_unit() + if unit is None or unit._unit_components or unit._display_symbol not in _STEP_WORDS: + return False + return any(abs(n) >= 1 for n in (numbers or _own_numbers)(var)) + + +def _holds_a_fraction(var: Any, numbers: Any = None) -> bool: + """A % line holding fractions only (0.05, never 5).""" + unit = var._get_unit() + if unit is None or unit._unit_components or unit._display_symbol != '%': + return False + shares = (numbers or _own_numbers)(var) + return bool(shares) and not any(abs(n) >= 1 for n in shares) + + +def _refuse_a_count_added_to_a_share(left: Any, right: Any, numbers: Any = None, + where: Optional[str] = None) -> None: + """A bps or pp line holding a count (50) meets a % line holding + fractions (0.05) in a sum: a label never rescales, so the result is + wrong under any reading - 0.05 + 50 is 50.05, not 0.055. ``numbers`` + reads the numbers a side is judged by (all of its own by default); + ``where`` names the function that adds, compares or picks the two + (``'mo.MAX'``) in place of ``+``.""" + for counted, share in ((left, right), (right, left)): + if not (_holds_a_count(counted, numbers) and _holds_a_fraction(share, numbers)): + continue + word = counted._get_unit()._display_symbol + example, fraction, step = _STEP_WORDS[word] + code = getattr(counted, '_python_name', None) or _line_name(counted) + met = (f"added to {_line_name(share)} (%)" if where is None + else f"beside {_line_name(share)} (%) in {where}") + wrong = "the sum would be wrong" if where is None else "the two cannot be read as one" + raise ValueError( + f"{_line_name(counted)} is in {word} and holds a count, {met}, which " + f"holds a fraction: a label never rescales, so {wrong}. A {word} line " + f"holds the fraction ({example} is {fraction}), or the count is " + f"converted first: {code} / {step}." + ) + + +def _refuse_counts_among_shares(lines: Sequence[Any], numbers: Any = None, + where: Optional[str] = None) -> None: + """:func:`_refuse_a_count_added_to_a_share` over every two of the + ``lines`` a function adds, compares or picks, in one pass: a line is a + count, a fraction or neither on its own, so the pair to refuse is the + first count and the first fraction among them.""" + counted = next((v for v in lines if _holds_a_count(v, numbers)), None) + if counted is None: + return + share = next((v for v in lines if _holds_a_fraction(v, numbers)), None) + if share is not None: + _refuse_a_count_added_to_a_share(counted, share, numbers, where) class _VariableArithmetic: @@ -315,6 +1186,42 @@ def _reject_pending_roles(*operands) -> None: f"BEFORE the formulas that read it)." ) + @staticmethod + def _reject_keys_against_periods(left: Any, right: Any) -> None: + """Refuse a line that runs along keys against a period series. + + Zipped cell by cell, a keyed list (or a line on one finite + non-time axis) and a series — a list, a located list, a tracked + line — pair Bear with January: numbers that look right and are + wrong. Keyed against keyed is alignment's business + (``_align_keyed_values``), an axis against an axis the axis + expansion's; a keyed list against an axis of other labels is + nobody's and is refused too. A scalar, a one-value bare list and + a raw Python list (``keyed * other.value``, positional on + purpose) broadcast as before. + """ + lk, rk = keys_along(left), keys_along(right) + if lk is not None and rk is not None: + left_dict = bool(getattr(left, '_keys', None)) + right_dict = bool(getattr(right, '_keys', None)) + if left_dict != right_dict and [str(k) for k in lk] != [str(k) for k in rk]: + raise ValueError(_keys_against_keys(left, lk, right, rk)) + return + if lk is None and rk is None: + return + if lk is not None: + keyed, labels, series = left, lk, right + else: + keyed, labels, series = right, rk or [], left + n = _period_count(series) + if n is None: + return + if (getattr(series, '_indexed_by', None) or ()) and ( + getattr(keyed, '_indexed_by', None) or () + ): + return # both on axes — the axis expansion crosses them + raise ValueError(_keys_against_periods(keyed, labels, series, n)) + @staticmethod def _broadcast_operation(left, right, op, safe_divide: bool = False, op_symbol: Optional[str] = None): @@ -332,13 +1239,26 @@ def _broadcast_operation(left, right, op, safe_divide: bool = False, it: ``date + 5`` is a date, ``date * 5`` is a number and ``date < 5`` is a bool, all computed on the date's serial number. """ - def _is_excel_error(v) -> bool: - return isinstance(v, str) and v.startswith('#') + # Whether a blank reads as "" (a comparison against text) is one + # question per call, not one per cell. + blank_reads_text = op_symbol in _COMPARISONS def safe_op(a, b): - if _is_excel_error(a): + ta, tb = type(a), type(b) + if (ta is float or ta is int) and (tb is float or tb is int): + # Two plain numbers — most cells of a model. None of the + # markers below can be one of them (not text, not blank, + # not a date), so only the zero divisor is left to ask. + if safe_divide and b == 0: + return '#DIV/0!' + try: + return op(a, b) + except Exception: + return '#VALUE!' + # An Excel-style error marker propagates, the left one first. + if isinstance(a, str) and a.startswith('#'): return a - if _is_excel_error(b): + if isinstance(b, str) and b.startswith('#'): return b # A blank counts as zero, as in Excel: a formula over a # month nobody entered computes over 0 (blank + 5 = 5, @@ -346,9 +1266,9 @@ def safe_op(a, b): # whether an actual was entered is the blend's question, # asked of its entries (``core.entries``). if a is None: - a = '' if isinstance(b, str) and op_symbol in _COMPARISONS else 0 + a = '' if isinstance(b, str) and blank_reads_text else 0 if b is None: - b = '' if isinstance(a, str) and op_symbol in _COMPARISONS else 0 + b = '' if isinstance(a, str) and blank_reads_text else 0 if safe_divide and b == 0: return '#DIV/0!' if isinstance(a, (date, timedelta)) or isinstance(b, (date, timedelta)): @@ -391,12 +1311,12 @@ def _align_keyed_values(left, right): Raises if key sets disagree — silently reordering a mismatch would produce a silently wrong result. """ - from .variable import Variable - left_val = left._value if isinstance(left, Variable) else left - right_val = right._value if isinstance(right, Variable) else right + variable_cls = _Variable or _variable_class() + left_val = left._value if isinstance(left, variable_cls) else left + right_val = right._value if isinstance(right, variable_cls) else right - left_keys = getattr(left, '_keys', None) if isinstance(left, Variable) else None - right_keys = getattr(right, '_keys', None) if isinstance(right, Variable) else None + left_keys = getattr(left, '_keys', None) if isinstance(left, variable_cls) else None + right_keys = getattr(right, '_keys', None) if isinstance(right, variable_cls) else None if (left_keys is not None and right_keys is not None and isinstance(right_val, list)): @@ -418,9 +1338,9 @@ def _align_keyed_values(left, right): @staticmethod def _propagate_keys(left, right): """Result keys come from whichever operand has them; left wins if both.""" - from .variable import Variable - left_keys = getattr(left, '_keys', None) if isinstance(left, Variable) else None - right_keys = getattr(right, '_keys', None) if isinstance(right, Variable) else None + variable_cls = _Variable or _variable_class() + left_keys = getattr(left, '_keys', None) if isinstance(left, variable_cls) else None + right_keys = getattr(right, '_keys', None) if isinstance(right, variable_cls) else None if left_keys is not None: return list(left_keys) if right_keys is not None: @@ -465,6 +1385,17 @@ def _unit_div(left_unit, right_unit): result = lu / right_unit return result if result._unit_components else None + @staticmethod + def _unit_mod(left_unit, right_unit): + """Resulting unit for ``%`` - ``MOD``'s law: the remainder keeps + the unit both sides share, a number goes with any, two units are + refused.""" + try: + return _VariableArithmetic._unit_add(left_unit, right_unit) + except ValueError as exc: + raise ValueError(str(exc).replace( + "Cannot add/subtract", "Cannot take the remainder of", 1)) from None + @staticmethod def _unit_add(left_unit, right_unit): """Resulting unit for addition/subtraction. @@ -483,16 +1414,24 @@ def _unit_add(left_unit, right_unit): @staticmethod def _unit_pow(unit, exponent): - """Resulting unit for ``x ** n``. - - Only defined for integer exponents (``hr ** 2 → hr^2``). Non-int - exponents drop the unit — Modeleon doesn't track fractional-power - units because they rarely appear in financial modeling and the - rendering gets ugly. + """Resulting unit for ``x ** n``: each dimension's power times + ``n``, when every one comes out whole — ``hr ** 2`` is ``hr^2``, + ``hr ** 2.0`` too, ``(USD^2) ** 0.5`` is ``USD``. Otherwise None: + a fractional power of a dimension (``USD ** 0.5``) has no unit to + write, and an exponent that is not a number (a line, a list of + them) may differ by period. """ - if unit is None or not isinstance(exponent, int): + if (unit is None or isinstance(exponent, bool) + or not isinstance(exponent, (int, float)) or not math.isfinite(exponent)): return None - result = unit ** exponent + powers = {} + for symbol, power in unit._unit_components.items(): + raised = power * exponent + if abs(raised - round(raised)) > 1e-9: + return None + powers[symbol] = round(raised) + from .unit import Unit + result = Unit(_components=powers) # ``hr ** 0`` → dimensionless; collapse the empty-components # Unit to ``None`` to match the ``mul`` / ``div`` convention. return result if result._unit_components else None @@ -513,8 +1452,8 @@ def _binary_op(self, other, op_symbol: str, op_func, unit_fn, op_symbol: Text inserted between operands in the formula string. op_func: Element-wise function applied for value computation. unit_fn: ``(left_unit, right_unit) -> result_unit`` or ``None``. - Pass ``None`` for operators that don't track units - (``**``, ``%``, and ``//`` when scalar is on the right). + ``None`` for ``**``, whose exponent sets the unit + afterwards. safe_divide: Route through the safe-division path so 1/0 becomes ``"#DIV/0!"`` instead of raising. parenthesize: Wrap compound-unnamed-formula operands in parens @@ -523,49 +1462,59 @@ def _binary_op(self, other, op_symbol: str, op_func, unit_fn, In the formula string, operand order flips; dependency refs still record ``[self, other]`` in that order. """ - from .variable import Variable + variable_cls = _Variable or _variable_class() - self_u = self._get_unit() self._reject_pending_roles(self, other) - if isinstance(other, Variable): + # ``result`` is a plain Variable built right here, and its class + # declares no descriptor for any field written below: its own + # ``__dict__`` takes each write exactly as ``__setattr__`` would, + # without a Python-level call per field. + if isinstance(other, variable_cls): left, right = (other, self) if reversed else (self, other) expr_node = BinOp(op_symbol, left._as_operand(parenthesize), right._as_operand(parenthesize)) - result = Variable() + result = variable_cls() result._set_expr(expr_node) + fields = result.__dict__ + if op_symbol in ('+', '-') and not any( + getattr(v, '_awaits', None) or getattr(v, '_forward_of', None) is not None + for v in (left, right)): + _refuse_a_count_added_to_a_share(left, right) + self._reject_keys_against_periods(left, right) lv, rv = self._align_keyed_values(left, right) expanded = self._expand_axised_operands(left, right, lv, rv) if expanded is not None: lv, rv = expanded - aligned = self._align_by_date(left, right, lv, rv) - if aligned is not None: - lv, rv, _union_start, _union_grain, _union_first = aligned + (lv, rv), window, runs_on, through = _meet((left, right), [lv, rv]) + if window is not None: # The result is located at the union window explicitly — - # downstream alignment then knows where it starts. - result._start = _union_start - result._grain = _union_grain - result._first_day = _union_first - from .tracks import TrackValues + # downstream alignment then knows where it starts — and + # marked as read by date there (a renderer reads its + # operands' cells of the same date). + fields['_start'], fields['_grain'], fields['_first_day'] = window + fields['_read_by_date'] = True if isinstance(lv, TrackValues) or isinstance(rv, TrackValues): # The tracks broadcast law: zip per role; # a plain operand broadcasts into every track; # differing role sets die loudly inside ``combine``. - result._value = TrackValues.combine( + fields['_value'] = TrackValues.combine( lv, rv, lambda a, b: self._broadcast_operation( a, b, op_func, safe_divide=safe_divide, op_symbol=op_symbol ), ) else: - result._value = self._broadcast_operation( + fields['_value'] = self._broadcast_operation( lv, rv, op_func, safe_divide=safe_divide, op_symbol=op_symbol ) if unit_fn is not None: - result._unit = _provisional_unit( + fields['_unit'] = _provisional_unit( unit_fn, left._get_unit(), right._get_unit(), left, right) + if window is None: + _keep_runs_on(result, runs_on, through) else: - from .tracks import TrackValues + self_u = self._get_unit() def _combine(a, b): if isinstance(a, TrackValues) or isinstance(b, TrackValues): @@ -585,123 +1534,56 @@ def _combine(a, b): else: expr_node = BinOp(op_symbol, self._as_operand(parenthesize), Literal(other)) val = _combine(self._value, other) - result = Variable() + result = variable_cls() result._set_expr(expr_node) - result._value = val + fields = result.__dict__ + fields['_value'] = val if unit_fn is not None: # Scalars are treated as dimensionless when computing the output unit - result._unit = _provisional_unit( + fields['_unit'] = _provisional_unit( unit_fn, *((None, self_u) if reversed else (self_u, None)), self) - if isinstance(result._value, list) and len(result._value) > 1: - result.var_type = "list" - elif type(result._value).__name__ == 'TrackValues' and result._value.time_length: - result.var_type = "list" + value = fields['_value'] + if isinstance(value, list) and len(value) > 1: + fields['var_type'] = "list" + elif type(value).__name__ == 'TrackValues' and value.time_length: + fields['var_type'] = "list" # A date-valued result (e.g. ``start + 90``, ``end - 1``) must carry # the datetime value type so the writer formats the cell as a date. - sample = (result._value[0] if isinstance(result._value, list) and result._value - else result._value) + sample = value[0] if isinstance(value, list) and value else value if isinstance(sample, (date, datetime)): - result.value_type = "datetime" + fields['value_type'] = "datetime" k_left, k_right = (other, self) if reversed else (self, other) - result._keys = self._propagate_keys(k_left, k_right) - result._indexed_by = self._propagate_indexed_by(k_left, k_right) + fields['_keys'] = self._propagate_keys(k_left, k_right) + fields['_indexed_by'] = self._propagate_indexed_by(k_left, k_right) return result @staticmethod def _align_by_date(left: Any, right: Any, lv: Any, rv: Any): - """Date-aligned zip — calendar alignment of two located operands. - - Engages ONLY when both operands are list-valued Variables with - resolvable time locations that disagree (different starts or - different lengths at one grain). Same-window operands keep the - byte-identical positional path. Different GRAINS refuse — that - is re-grain territory, not alignment. - - Each side materializes onto the union window; cells outside an - operand's own extent follow its DECLARED ``extend=`` rule — - zero (a flow is absent), hold (a rate stays in force; held - backward from the first value when extended into the past), - or none/undeclared → a teaching error naming both extents. - Returns ``(lv', rv', union_start, grain)`` or ``None`` when - not applicable. + """Date-aligned zip — calendar alignment of two operands on other + windows (:func:`_on_dates`, which every function that is one call + per period shares). + + Engages ONLY when both operands are series on windows that + disagree (different starts or different lengths at one grain): a + line's own window, or for an unnamed formula the one of the lines + it reads. Same-window operands keep the byte-identical positional + path. Different GRAINS refuse — that is re-grain territory, not + alignment. + + Each side materializes onto the union window: cells outside an + operand's own extent follow its DECLARED ``extend=`` rule — zero + (a flow is absent), hold (a rate stays in force; held backward + from the first value when extended into the past), or + none/undeclared → an error naming both extents and the rules to declare. A plain + line beside a tracked one is read on the tracked line's window. + Returns ``(lv', rv', union_start, grain, union_first)`` or + ``None`` when not applicable. """ - from .tracks import TrackValues - from .variable import Variable - if not isinstance(left, Variable) or not isinstance(right, Variable): - return None - lt, rt = left.time, right.time - if (lt is not None and rt is not None and lt.start is not None - and rt.start is not None and lt.grain == rt.grain - and lt.first_day != rt.first_day): - # One side's first period is short. It may open the combined - # window, never sit inside it: on the other side the same - # period starts on another day. Tracked lines too. - stub, other = (left, right) if lt.first_day is not None else (right, left) - st, ot = stub.time, other.time - if ot.first_day is not None or st.start >= ot.start: - raise ValueError(_mixed_first_periods(left, lt, right, rt)) - if isinstance(lv, TrackValues) or isinstance(rv, TrackValues): - return None # tracks share the ambient window by the extent law - if not isinstance(lv, list) or not isinstance(rv, list): + out, window = _on_dates((left, right), [lv, rv]) + if window is None: return None - if lt is None or rt is None or lt.start is None or rt.start is None: - return None - if lt.start == rt.start and len(lv) == len(rv): - return None - if lt.grain != rt.grain: - raise ValueError(_mixed_grains(left, lt.grain, right, rt.grain)) - from .time import _count, _parse - grain = lt.grain - la, ra = _parse(lt.start, grain), _parse(rt.start, grain) - # Union window in period indices relative to the earlier start. - # ``_count`` is INCLUSIVE (Jan→Apr = 4 labels); the offset is - # one less. - off_l = 0 if lt.start <= rt.start else _count(ra, la, grain) - 1 - off_r = 0 if rt.start <= lt.start else _count(la, ra, grain) - 1 - total = max(off_l + len(lv), off_r + len(rv)) - union_start = min(lt.start, rt.start) - union_first = lt.first_day if lt.start <= rt.start else rt.first_day - - def project(var: "Variable", vals: list, off: int, other: str) -> list: - from .expr import TimeRef - time_expr = getattr(var, '_expr', None) - if isinstance(time_expr, TimeRef): - # Time is defined in every period, not only the window's. - field = time_expr.field - if field == 'index': - return [i - off + 1 for i in range(total)] - from .time import time_field_series - return time_field_series(field, union_start, grain, total, union_first) - out = [] - rule = getattr(var, '_extend', None) - for i in range(total): - j = i - off - if 0 <= j < len(vals): - out.append(vals[j]) - elif rule == 'zero': - out.append(0.0) - elif rule == 'hold': - out.append(vals[0] if j < 0 else vals[-1]) - else: - label = (var._display_name - or getattr(var, '_python_name', None) - or 'variable') - raise ValueError( - f"{label!r} does not cover the combined window " - f"(it spans {len(vals)} period(s) from " - f"{var.time.start}; the other side spans " - f"{other}). Say what continues it: " - f"extend=mo.zero() (a flow) or extend=mo.hold() " - f"(a rate) — never a silent guess." - ) - return out - - l_span = f"{len(rv)} period(s) from {rt.start}" - r_span = f"{len(lv)} period(s) from {lt.start}" - return (project(left, lv, off_l, l_span), - project(right, rv, off_r, r_span), - union_start, grain, union_first) + return out[0], out[1], window.start, window.grain, window.first_day @staticmethod def _expand_axised_operands(left: Any, right: Any, lv: Any, rv: Any): @@ -817,10 +1699,10 @@ def __rtruediv__(self, other) -> "Variable": def __pow__(self, other) -> "Variable": result = self._binary_op(other, "**", lambda a, b: a ** b, None) - # Scalar integer exponent → raise the unit to that power. - # Variable-on-the-right exponents leave the unit as None. - from .variable import Variable - if not isinstance(other, Variable): + # A constant exponent raises the unit (``_unit_pow``); one that + # is a line leaves none. + variable_cls = _Variable or _variable_class() + if not isinstance(other, variable_cls): result._unit = self._unit_pow(self._get_unit(), other) return result @@ -828,24 +1710,25 @@ def __rpow__(self, other) -> "Variable": return self._binary_op(other, "**", lambda a, b: a ** b, None, reversed=True) def __mod__(self, other) -> "Variable": - return self._binary_op(other, "%", lambda a, b: a % b, None) + return self._binary_op(other, "%", lambda a, b: a % b, self._unit_mod) def __rmod__(self, other) -> "Variable": - return self._binary_op(other, "%", lambda a, b: a % b, None, reversed=True) + return self._binary_op(other, "%", lambda a, b: a % b, self._unit_mod, + reversed=True) def __floordiv__(self, other) -> "Variable": return self._binary_op(other, "//", lambda a, b: a // b, self._unit_div) def __rfloordiv__(self, other) -> "Variable": - return self._binary_op(other, "//", lambda a, b: a // b, None, reversed=True) + return self._binary_op(other, "//", lambda a, b: a // b, self._unit_div, + reversed=True) # ─── Unary dunders ───────────────────────────────────────── def __neg__(self) -> "Variable": """-a (unary negation)""" - from .variable import Variable - from .tracks import TrackValues - result = Variable() + variable_cls = _Variable or _variable_class() + result = variable_cls() result._set_expr(UnaryOp("-", self._as_operand(parenthesize=True))) def neg(x): @@ -886,13 +1769,12 @@ def __pos__(self) -> "Variable": def _comparison_op(self, other, op_symbol: str, op_func) -> "Variable": """Shared implementation for all comparison operators.""" - from .variable import Variable + variable_cls = _Variable or _variable_class() # ``self`` is always a :class:`Variable` at runtime — the mixin is # only mounted on Variable. The casts below tell mypy that. self_var: "Variable" = self # type: ignore[assignment] self._reject_pending_roles(self, other) - from .tracks import TrackValues def _combine(a, b): if isinstance(a, TrackValues) or isinstance(b, TrackValues): @@ -903,25 +1785,45 @@ def _combine(a, b): ) return self._broadcast_operation(a, b, op_func, op_symbol=op_symbol) - if isinstance(other, Variable): - result = Variable() + # A plain Variable built right here: its fields go straight into + # its ``__dict__``, as in ``_binary_op``. + if isinstance(other, variable_cls): + # Over a row whose loop is open the units are provisional: the + # loop checks the comparison when it closes. + lu, ru = self_var._get_unit(), other._get_unit() + if (lu is not None and ru is not None and lu != ru + and not _on_open_loop(self_var) and not _on_open_loop(other)): + raise ValueError(_incompatible_units(lu, ru).replace( + "Cannot add/subtract", "Cannot compare", 1)) + result = variable_cls() result._set_expr(Compare(op_symbol, VarRef(self_var), VarRef(other))) + fields = result.__dict__ + self._reject_keys_against_periods(self, other) lv, rv = self._align_keyed_values(self, other) expanded = self._expand_axised_operands(self, other, lv, rv) if expanded is not None: lv, rv = expanded - result._value = _combine(lv, rv) + # Two windows meet by date, as in ``_binary_op``. + (lv, rv), window, runs_on, through = _meet((self, other), [lv, rv]) + if window is not None: + fields['_start'], fields['_grain'], fields['_first_day'] = window + fields['_read_by_date'] = True + fields['_value'] = _combine(lv, rv) + if window is None: + _keep_runs_on(result, runs_on, through) else: - result = Variable() + result = variable_cls() result._set_expr(Compare(op_symbol, VarRef(self_var), Literal(other))) - result._value = _combine(self._value, other) - - if isinstance(result._value, list) and len(result._value) > 1: - result.var_type = 'list' - elif type(result._value).__name__ == 'TrackValues' and result._value.time_length: - result.var_type = 'list' - result._keys = self._propagate_keys(self, other) - result._indexed_by = self._propagate_indexed_by(self, other) + fields = result.__dict__ + fields['_value'] = _combine(self._value, other) + + value = fields['_value'] + if isinstance(value, list) and len(value) > 1: + fields['var_type'] = 'list' + elif type(value).__name__ == 'TrackValues' and value.time_length: + fields['var_type'] = 'list' + fields['_keys'] = self._propagate_keys(self, other) + fields['_indexed_by'] = self._propagate_indexed_by(self, other) return result # ─── Comparison dunders ──────────────────────────────────── diff --git a/src/modeleon/display/code.py b/src/modeleon/display/code.py index b3254af..a3cba19 100644 --- a/src/modeleon/display/code.py +++ b/src/modeleon/display/code.py @@ -355,7 +355,7 @@ def _emit_descendants( from ..core.multi_variable import MultiVariableBase for name in mv._component_order: - child = mv._components[name] + child = mv._component(name) child_lhs = f"{prefix}.{name}" if isinstance(child, Variable): out.append(_render_variable( @@ -400,7 +400,7 @@ def _collect_scope_vars(mv: "MultiVariableBase") -> set["Variable"]: def visit(node: "MultiVariableBase") -> None: for cname in node._component_order: - child = node._components[cname] + child = node._component(cname) if isinstance(child, MultiVariableBase): visit(child) elif isinstance(child, Variable): diff --git a/src/modeleon/display/html.py b/src/modeleon/display/html.py index 7e9a962..b3381e7 100644 --- a/src/modeleon/display/html.py +++ b/src/modeleon/display/html.py @@ -294,7 +294,7 @@ def _make_overview_sheet(root, direct_var_names: list[str]): # Bulk-assign — the property setter installs the dict into the # underlying storage in one shot. Per-key writes through the # property getter would write into a snapshot, not the storage. - virt._components = {name: root._components[name] for name in direct_var_names} + virt._components = {name: root._component(name) for name in direct_var_names} return virt @@ -971,7 +971,7 @@ def _collect_rows(mv, direct_only: bool = False) -> tuple[list, int]: max_cols = 1 hidden = _units_hidden(mv) for name in mv._component_order: - comp = mv._components[name] + comp = mv._component(name) if isinstance(comp, Variable): label = _label_with_unit(comp, hidden) values = comp._value if isinstance(comp._value, list) else [comp._value] @@ -1257,7 +1257,7 @@ def model_html(root) -> str: direct_var_names: list = [] sub_mvs: list = [] for name in root._component_order: - comp = root._components[name] + comp = root._component(name) if isinstance(comp, Variable): direct_var_names.append(name) elif isinstance(comp, MultiVariableBase): diff --git a/src/modeleon/functions/__init__.py b/src/modeleon/functions/__init__.py index 0d15e5a..7880c40 100644 --- a/src/modeleon/functions/__init__.py +++ b/src/modeleon/functions/__init__.py @@ -4,17 +4,45 @@ Organized by topic: - :mod:`aggregate` — ``SUM``, ``MAX``, ``MIN``, ``AVERAGE`` -- :mod:`mathfn` — ``ABS``, ``ROUND``, ``INT``, ``MOD`` +- :mod:`criteria` — ``SUMIF``, ``SUMIFS``, ``COUNTIF``, ``COUNTIFS``, + ``AVERAGEIF``, ``AVERAGEIFS``, ``MAXIFS``, ``MINIFS``, ``COUNT``, + ``COUNTA`` +- :mod:`mathfn` — ``ABS``, ``ROUND``, ``INT``, ``MOD``, ``ROUNDUP``, + ``ROUNDDOWN``, ``TRUNC``, ``CEILING``, ``FLOOR``, ``POWER``, ``SQRT``, + ``EXP``, ``LN``, ``LOG``, ``LOG10``, ``SIGN``, ``PRODUCT`` - :mod:`conditional` — ``IF``, ``AND``, ``OR``, ``NOT``, ``CHOOSE``, ``ISBLANK`` +- :mod:`logical` — ``IFERROR``, ``ISERROR``, ``ISNUMBER``, ``XOR``, + ``IFS``, ``SWITCH`` +- :mod:`lookup` — ``MATCH``, ``INDEX``, ``XLOOKUP``, ``VLOOKUP``, + ``HLOOKUP`` - :mod:`dates` — ``YEAR``, ``MONTH``, ``DAY``, ``DATE``, ``EDATE``, ``EOMONTH``, ``DAYS360``, - ``YEARFRAC``, ``DAYS``, ``TODAY`` -- :mod:`text` — ``LEN``, ``UPPER``, ``LOWER``, ``CONCAT`` + ``YEARFRAC``, ``DAYS``, ``TODAY``, ``DATEDIF``, ``WEEKDAY``, ``WEEKNUM``, + ``NETWORKDAYS``, ``WORKDAY`` +- :mod:`text` — ``LEN``, ``UPPER``, ``LOWER``, ``CONCAT``, ``LEFT``, + ``RIGHT``, ``MID``, ``TRIM``, ``FIND``, ``SEARCH``, ``SUBSTITUTE``, + ``VALUE``, ``TEXT``, ``REPT``, ``TEXTJOIN`` - :mod:`recurrence` — ``recurrence``, ``recurrence_sum``, ``cumsum`` - :mod:`lag` — ``lag`` - :mod:`schedule` — ``schedule`` - :mod:`cohort` — ``cohort_retention`` -- :mod:`financial` — ``IRR``, ``NPV``, ``XIRR``, ``XNPV``, ``PMT``, ``FV``, ``PV`` +- :mod:`financial` — ``IRR``, ``NPV``, ``XIRR``, ``XNPV``, ``PMT``, ``FV``, ``PV``, + ``NPER``, ``IPMT``, ``PPMT``, ``RATE``, ``CUMIPMT``, ``CUMPRINC``, + ``SLN``, ``DB``, ``DDB``, ``SYD`` +- :mod:`statistics` — ``MEDIAN``, ``STDEV_S``, ``STDEV_P``, ``VAR_S``, ``VAR_P``, + ``PERCENTILE_INC``, ``PERCENTILE_EXC``, ``QUARTILE_INC``, ``QUARTILE_EXC``, + ``LARGE``, ``SMALL``, ``RANK_EQ``, ``RANK_AVG``, and the older names + ``STDEV``, ``STDEVP``, ``VAR``, ``VARP``, ``PERCENTILE``, ``QUARTILE``, ``RANK`` + +How each name is shaped — one call per period, a reduction, a call +over ranges — and how the workbook writes it is the function +registry's to say (:mod:`registry`); every function registers itself +from its own module. Every aggregate — ``SUM``, ``AVERAGE``, ``MIN``, +``MAX``, ``PRODUCT``, ``COUNT``, ``COUNTA``, the statistics — follows one +rule: several series are combined period by period, as ``a + b`` +combines them (``SUM(a, b)`` is ``=SUM(B2, B3)`` in every column); one +series alone is reduced over its periods (``SUM(a)`` is ``=SUM(B2:F2)``); +scalars only stay one number. The :class:`Unit` type lives in :mod:`modeleon.core.unit` — it's a primitive used by :class:`Variable` itself, not a formula helper. @@ -25,8 +53,9 @@ a ``funccall`` node named ``SUM`` in JSON). Functions that are native to a specific target — the Excel financial helpers (``IRR``, ``NPV``, ``XIRR``, ``PMT``, ``FV``, ``PV``) -and the Excel date shift helpers (``EDATE``, ``EOMONTH``) — pass -``render_backends=frozenset({'excel'})`` through +and the Excel date helpers (``EDATE``, ``EOMONTH``, ``YEARFRAC``, +``DAYS``, ``DATEDIF``, ``WEEKDAY``, ``WEEKNUM``, ``NETWORKDAYS``, +``WORKDAY``) — pass ``render_backends=frozenset({'excel'})`` through :func:`._helpers.make_func_var` / ``_wrap_with_formula``. Other renderers fall back to inlining the Python-computed ``_value``. Inspect compat for a given target with :func:`modeleon.check_compat`. @@ -36,29 +65,66 @@ from .aggregate import AVERAGE, MAX, MIN, SUM from .cohort import cohort_retention from .conditional import AND, CHOOSE, IF, ISBLANK, NOT, OR -from .dates import DATE, DAY, DAYS, DAYS360, EDATE, EOMONTH, MONTH, TODAY, YEAR, YEARFRAC -from .financial import FV, IRR, NPV, PMT, PV, XIRR, XNPV -from .mathfn import ABS, INT, MOD, ROUND +from .criteria import ( + AVERAGEIF, AVERAGEIFS, COUNT, COUNTA, COUNTIF, COUNTIFS, MAXIFS, MINIFS, SUMIF, SUMIFS, +) +from .dates import ( + DATE, DATEDIF, DAY, DAYS, DAYS360, EDATE, EOMONTH, MONTH, NETWORKDAYS, TODAY, WEEKDAY, + WEEKNUM, WORKDAY, YEAR, YEARFRAC, +) +from .financial import ( + CUMIPMT, CUMPRINC, DB, DDB, FV, IPMT, IRR, NPER, NPV, PMT, PPMT, PV, RATE, SLN, SYD, XIRR, + XNPV, +) +from .logical import IFERROR, IFS, ISERROR, ISNUMBER, SWITCH, XOR +from .lookup import HLOOKUP, INDEX, MATCH, VLOOKUP, XLOOKUP +from .mathfn import ( + ABS, CEILING, EXP, FLOOR, INT, LN, LOG, LOG10, MOD, POWER, PRODUCT, ROUND, ROUNDDOWN, + ROUNDUP, SIGN, SQRT, TRUNC, +) from .lag import lag from .schedule import schedule from .recurrence import cumsum, recurrence, recurrence_sum -from .text import CONCAT, LEN, LOWER, UPPER +from .statistics import ( + LARGE, MEDIAN, PERCENTILE, PERCENTILE_EXC, PERCENTILE_INC, QUARTILE, QUARTILE_EXC, + QUARTILE_INC, RANK, RANK_AVG, RANK_EQ, SMALL, STDEV, STDEV_P, STDEV_S, STDEVP, VAR, + VAR_P, VAR_S, VARP, +) +from .text import ( + CONCAT, FIND, LEFT, LEN, LOWER, MID, REPT, RIGHT, SEARCH, SUBSTITUTE, TEXT, TEXTJOIN, TRIM, + UPPER, VALUE, +) __all__ = [ # Aggregates "SUM", "MAX", "MIN", "AVERAGE", + # Conditional aggregates and counts + "SUMIF", "SUMIFS", "COUNTIF", "COUNTIFS", "AVERAGEIF", "AVERAGEIFS", "MAXIFS", "MINIFS", + "COUNT", "COUNTA", # Math - "ABS", "ROUND", "INT", "MOD", + "ABS", "ROUND", "INT", "MOD", "ROUNDUP", "ROUNDDOWN", "TRUNC", "CEILING", "FLOOR", + "POWER", "SQRT", "EXP", "LN", "LOG", "LOG10", "SIGN", "PRODUCT", # Conditional / logical "IF", "AND", "OR", "NOT", "CHOOSE", "ISBLANK", + "IFERROR", "ISERROR", "ISNUMBER", "XOR", "IFS", "SWITCH", + # Lookup / reference + "MATCH", "INDEX", "XLOOKUP", "VLOOKUP", "HLOOKUP", # Dates "YEAR", "MONTH", "DAY", "DATE", "EDATE", "EOMONTH", "DAYS360", "TODAY", - "YEARFRAC", "DAYS", + "YEARFRAC", "DAYS", "DATEDIF", "WEEKDAY", "WEEKNUM", "NETWORKDAYS", "WORKDAY", # Text "LEN", "UPPER", "LOWER", "CONCAT", + "LEFT", "RIGHT", "MID", "TRIM", "FIND", "SEARCH", "SUBSTITUTE", "VALUE", "TEXT", "REPT", + "TEXTJOIN", # Financial "IRR", "NPV", "XIRR", "XNPV", "PMT", "FV", "PV", + "NPER", "IPMT", "PPMT", "RATE", "CUMIPMT", "CUMPRINC", "SLN", "DB", "DDB", "SYD", + # Statistics + "MEDIAN", "STDEV_S", "STDEV_P", "VAR_S", "VAR_P", + "PERCENTILE_INC", "PERCENTILE_EXC", "QUARTILE_INC", "QUARTILE_EXC", + "LARGE", "SMALL", "RANK_EQ", "RANK_AVG", + "STDEV", "STDEVP", "VAR", "VARP", "PERCENTILE", "QUARTILE", "RANK", # Domain helpers (no Excel equivalent — stay lowercase) "cumsum", "lag", "schedule", "recurrence", "recurrence_sum", "cohort_retention", diff --git a/src/modeleon/functions/_helpers.py b/src/modeleon/functions/_helpers.py index 33d21f2..48cc035 100644 --- a/src/modeleon/functions/_helpers.py +++ b/src/modeleon/functions/_helpers.py @@ -6,15 +6,28 @@ into its raw value plus an AST node, render an argument for the ``_source_code`` string, and assemble a result Variable with a ``=FUNC(args)`` formula. These helpers consolidate that boilerplate. + +:func:`elementwise` is the one law for a function that is one call per +period — the cap and the floor ``MIN`` / ``MAX`` write in every column, +and the element-wise helpers built on it: scalars broadcast as ``a + b`` does, +a one-value series broadcasts, other length mismatches are refused, +tracked operands lift per coordinate, an error marker is the period's +result, the unit is the function's law (:func:`function_unit`). """ from __future__ import annotations import functools -from typing import Any, Callable +import inspect +import typing +from typing import Any, Callable, Sequence from ..core.expr import FuncCall, Literal, VarRef +from ..core.shape import keys_along +from ..core.tracks import TrackValues +from ..core.unit import UNSETTLED from ..core.variable import Variable +from ..core.variable_ops import _line_name, _on_dates, _VariableArithmetic def val(x: Any) -> Any: @@ -67,25 +80,181 @@ def src(value) -> str: return (value.python_name or value.path.leaf) if isinstance(value, Variable) else repr(value) -#: Functions whose result is NOT in the operands' unit — counts, -#: rates, calendar spans. +#: Functions whose result is NOT in the operands' unit — counts, rates, +#: calendar spans, positions, an exponential, a logarithm, a sign, a +#: boolean, text, a date. _UNITLESS_RESULTS = frozenset({ - 'COUNT', 'COUNTA', 'COUNTIF', 'COUNTIFS', - 'IRR', 'XIRR', 'RATE', 'YEARFRAC', 'DAYS', 'DAY', 'MONTH', 'YEAR', + 'COUNT', 'COUNTA', 'IRR', 'XIRR', 'YEARFRAC', 'MATCH', + 'EXP', 'LN', 'LOG', 'LOG10', 'SIGN', + 'AND', 'OR', 'NOT', 'XOR', 'ISBLANK', 'ISERROR', 'ISNUMBER', + 'DATE', 'EDATE', 'EOMONTH', 'TODAY', 'WORKDAY', 'DAYS', 'DAYS360', 'DATEDIF', + 'DAY', 'MONTH', 'YEAR', 'WEEKDAY', 'WEEKNUM', 'NETWORKDAYS', + 'LEN', 'UPPER', 'LOWER', 'CONCAT', 'LEFT', 'RIGHT', 'MID', 'TRIM', 'FIND', + 'SEARCH', 'SUBSTITUTE', 'VALUE', 'TEXT', 'REPT', 'TEXTJOIN', }) -def _shared_arg_unit(ast_args: list): - """The single unit shared by every unit-bearing Variable operand - (top-level VarRef args), else None — never guess.""" - from ..core.expr import VarRef +#: Functions that add, compare or pick their operands, so the operands +#: must share a unit the way ``+`` needs it: the position of the first +#: operand that is a value (an IF's condition, a CHOOSE's index are not). +#: A remainder and a multiple share one unit with what they divide by +#: (``MOD(months, 12)`` is in months, ``MOD(USD, hr)`` is refused). +#: IFS and SWITCH pick every other operand: their laws are declared +#: beside them. +_SAME_UNIT_FROM = {'SUM': 0, 'AVERAGE': 0, 'MIN': 0, 'MAX': 0, 'IF': 1, 'CHOOSE': 1, + 'IFERROR': 0, 'MOD': 0, 'CEILING': 0, 'FLOOR': 0} + +#: A unit law: the operand nodes of a call and the reader of a node's +#: unit → the result's unit. +UnitLaw = Callable[[Sequence[Any], Callable[[Any], Any]], Any] + +#: The functions with a law of their own — a power, a product, a +#: rounding, a conditional total, a pick from every other operand, the +#: financial functions — each declared next to the function +#: (``unit_law('POWER', ...)``). +_UNIT_LAWS: dict[str, UnitLaw] = {} + + +def unit_law(name: str, law: UnitLaw) -> None: + """Declare the unit law of ``name`` (see :data:`_UNIT_LAWS`).""" + _UNIT_LAWS[name] = law + + +def function_unit(func_name: str, args: Sequence[Any], unit_of: Callable[[Any], Any], + *, own_law: bool = True) -> Any: + """The unit ``func_name`` gives over its operand nodes, each one's unit + read by ``unit_of``; an operand the function cannot take is refused. + The one law a function's result is built by and a loop container + derives its rows' units again by: the function's declared law; else + none for a count, a boolean, text or a date; else the unit the values + share where the function adds, compares or picks them; else the unit + every unit-bearing operand shares (``ABS`` of $ is $). A call the + workbook does not write as a spreadsheet function (``own_law`` + False: a ``pyformula``, whatever its name) is opaque: no unit, unless + its author declares one (``pyformula(unit=...)``). While a loop settles, ``unit_of`` may read a row as + :data:`UNSETTLED`, and so may the answer.""" + if not own_law: + return None + law = _UNIT_LAWS.get(func_name) + if law is not None: + return law(args, unit_of) + if func_name in _UNITLESS_RESULTS: + return None + start = _SAME_UNIT_FROM.get(func_name) + if start is not None: + return same_unit(func_name, args[start:], unit_of) + return _shared_arg_unit(args, unit_of) + + +def same_unit(func_name: str, nodes: Sequence[Any], unit_of: Callable[[Any], Any]) -> Any: + """The unit ``nodes`` share where ``func_name`` adds, compares or picks + them: two different units are refused, as ``+`` refuses them; a + number or an unlabelled line goes with any. Pure numbers under two + labels add, and the result names neither. A row its loop has not + settled yet goes with any too; with no other unit, the answer is + :data:`UNSETTLED`.""" + from ..core.unit import unit_key + from ..core.variable_ops import _line_name + first: Any = None + keys = set() + waiting = False + for node in nodes: + unit = unit_of(node) + if unit is UNSETTLED: + waiting = True + continue + if unit is None: + continue + if first is None: + first = (node, unit) + elif unit != first[1]: + names = None + if isinstance(first[0], VarRef) and isinstance(node, VarRef): + names = (_line_name(first[0].var), _line_name(node.var)) + paths = (str(first[0].var.path), str(node.var.path)) + if names[0] == names[1] and not any(p.startswith('__floating__') for p in paths): + names = (repr(paths[0]), repr(paths[1])) # two records' 'Amount' + raise mixed_units_error(func_name, first[1], unit, names) + keys.add(unit_key(unit)) + refuse_counted_shares(func_name, nodes) + if first is None: + return UNSETTLED if waiting else None + return first[1] if len(keys) == 1 else None + + +def refuse_counted_shares(func_name: str, nodes: Sequence[Any]) -> None: + """The check ``+`` makes of a bps or pp line holding a count (50) + beside a % line holding a fraction (0.05), made wherever ``func_name`` + adds, compares or picks the lines (:func:`same_unit`'s callers: a + total, a pick, a lag's fill, a conditional total): the two cannot be + read as one. A line its loop has not closed holds placeholders, so the + loop makes the check when it closes (``core.loops._check_counts``).""" + from ..core.variable_ops import _on_open_loop, _refuse_counts_among_shares + lines = [n.var for n in nodes if isinstance(n, VarRef)] + if len(lines) < 2 or any(_on_open_loop(v) for v in lines): + return + _refuse_counts_among_shares(lines, where=f"mo.{func_name}") + + +def waits_on_loop(var: Any) -> bool: + """A Variable built on a loop row read before its line: its unit is + provisional, and the loop derives and checks it when it closes.""" + from ..core.variable_ops import _on_open_loop + return _on_open_loop(var) + + +def operand_unit(node: Any) -> Any: + """The unit of a built call's operand: a line's own; a constant, text + and a list of lines (a range, whose law reads its lines) have none; a + call inside it (an operator joined to a cell) gives its own law's.""" + if isinstance(node, VarRef): + return getattr(node.var, '_unit', None) + if isinstance(node, FuncCall): + return function_unit(node.func, node.args, operand_unit) + return None + + +def mixed_units_error(func_name: str, first: Any, second: Any, + names: Sequence[str] | None = None) -> ValueError: + """The refusal of two units in one function, worded as ``+`` words + it (which side is a dimension, which a labelled number, the hint).""" + from ..core.variable_ops import _incompatible_units + detail = _incompatible_units(first, second).split(": ", 1)[1] + who = f"{names[0]} and {names[1]}" if names else "they" + return ValueError(f"mo.{func_name} takes its values in one unit; {who} are in {detail}") + + +def refuse_mixed_units(func_name: str, operands: Sequence[Any]) -> None: + """Two operands in different units where ``func_name`` adds, compares + or picks them: refused, as ``+`` refuses them. An operand without a + unit (a number, an unlabelled line) goes with any.""" + from ..core.variable_ops import _line_name + first = None + for v in operands: + u = getattr(v, '_unit', None) if isinstance(v, Variable) else None + if u is None: + continue + if first is None: + first = (v, u) + elif u != first[1]: + raise mixed_units_error(func_name, first[1], u, + (_line_name(first[0]), _line_name(v))) + + +def _shared_arg_unit(ast_args: Sequence[Any], unit_of: Callable[[Any], Any]) -> Any: + """The generic law: the single unit shared by every unit-bearing + Variable operand (top-level VarRef args), else None — never guess; + unsettled while one of them is.""" from ..core.unit import unit_key units = [] seen = set() for node in ast_args: - var = getattr(node, 'var', None) if isinstance(node, VarRef) else None - u = getattr(var, '_unit', None) if var is not None else None + if not isinstance(node, VarRef): + continue + u = unit_of(node) + if u is UNSETTLED: + return UNSETTLED if u is not None and unit_key(u) not in seen: seen.add(unit_key(u)) units.append(u) @@ -101,6 +270,7 @@ def make_func_var( source_code: str | None = None, render_backends: frozenset | None = None, compute_backends: frozenset | None = None, + any_value_type: bool = False, ) -> Variable: """Build a Variable wrapping a ``=FUNC(args)`` formula + computed value. @@ -122,6 +292,13 @@ def make_func_var( function natively at runtime (``None`` = universal). Rarely populated today; distinguishes render target from compute target when a backend can emit a call but not evaluate it. + any_value_type: Carry the unit whatever ``value_type`` says — a + row typed by a first cell that holds an error marker is still + a row of numbers. Otherwise only a number carries one. + + The unit is the function's law (:func:`function_unit`) — by the name + the workbook writes; a call native to no renderer (``render_backends`` + empty, as :func:`pyformula` builds) has none unless declared. """ result = Variable(formula=FuncCall( func_name, ast_args, @@ -131,25 +308,353 @@ def make_func_var( result._value = value result.var_type = var_type result.value_type = value_type - # Unit preservation, the generic law: a numeric function over - # operands that share ONE unit keeps it — an IF-capped amount stays $, - # ROUND($) is $. Functions whose RESULT is a different kind of - # number (counts, rates, day-spans) are excluded; mixed or absent - # units stamp nothing. - if value_type in ('int', 'float') and func_name not in _UNITLESS_RESULTS: - unit = _shared_arg_unit(ast_args) - if unit is not None: - result._unit = unit + # The function's unit law (:func:`function_unit`) — what a loop + # container derives the row's unit by again — refuses what the + # function cannot take; over a loop row not settled yet the units are + # provisional, and the loop checks them when it closes. + try: + unit = function_unit(func_name, ast_args, operand_unit, + own_law=render_backends != frozenset()) + except ValueError: + if not waits_on_loop(result): + raise + unit = None + if unit is not None and (any_value_type or value_type in ('int', 'float')): + result._unit = unit if source_code is not None: result._source_code = source_code return result +# ─── One call per period ──────────────────────────────────────────── + + +def is_error(value: Any) -> bool: + """An Excel error token (``'#DIV/0!'``, ``'#N/A'``) as arithmetic + carries it.""" + return isinstance(value, str) and value.startswith('#') + + +def is_series(value: Any) -> bool: + """A period-shaped operand: a list, or a Variable holding one (a row, + or a row per track).""" + if isinstance(value, Variable): + return isinstance(value._value, (list, TrackValues)) + return isinstance(value, list) + + +def align_keyed(func_name: str, lines: Sequence[Any], + values: list[Any]) -> tuple[list[Any], list[Any] | None]: + """The values of a call that is one per period, its lines along keys + combined key by key as ``a + b`` combines them: a line along the same + keys in another order is read in the first one's order; other keys, + and a line along keys against a period series (Bear would meet + January), are refused as ``a + b`` refuses them. ``lines[i]`` is the + operand ``values[i]`` came from (a value typed in has none). Returns + the values and the keys the result runs along — the first keyed + line's, or None when no line runs along keys.""" + if not any(isinstance(v, (list, TrackValues)) for v in values): + return values, None # scalars run along no keys + operands = [a for a in lines if isinstance(a, Variable)] + # Only a pair with a side along keys can be refused: a call over many + # lines none of which runs along keys is read once, not pair by pair. + along = [keys_along(a) is not None for a in operands] + if any(along): + for i, line in enumerate(operands): + for j in range(i + 1, len(operands)): + if along[i] or along[j]: + _VariableArithmetic._reject_keys_against_periods(line, operands[j]) + first: Any = None + out = list(values) + for i, (line, value) in enumerate(zip(lines, values)): + keys = getattr(line, '_keys', None) if isinstance(line, Variable) else None + if not keys or not isinstance(value, list): + continue + if first is None: + first = line + continue + if list(keys) == list(first._keys): + continue + if set(keys) != set(first._keys) or len(keys) != len(value): + raise ValueError( + f"mo.{func_name} combines {_line_name(first)} and {_line_name(line)} key " + f"by key, and they run along other keys " + f"({', '.join(map(str, first._keys))} and {', '.join(map(str, keys))}): " + f"give both the same keys." + ) + at = {key: j for j, key in enumerate(keys)} + out[i] = [value[at[key]] for key in first._keys] + return out, (None if first is None else list(first._keys)) + + +def series_node(node: Any) -> bool: + """An operand node of a built call holding a series: a line holding a + list (or a row per track), or a list typed in.""" + if isinstance(node, VarRef): + return is_series(node.var) + return isinstance(node, Literal) and isinstance(node.value, list) + + +def shared_unit(args: Sequence[Any]) -> Any: + """The single unit shared by every unit-bearing Variable operand — an + aggregation of $ is $. Mixed or absent units → None (never guess).""" + from ..core.unit import unit_key + first = None + keys = set() + for a in args: + unit = getattr(a, '_unit', None) if isinstance(a, Variable) else None + if unit is None: + continue + keys.add(unit_key(unit)) + if first is None: + first = unit + return first if len(keys) == 1 else None + + +def first_variable_value_type(args: Sequence[Any], fallback: str = 'float') -> str: + """The value type of the first Variable operand, else ``fallback``.""" + for a in args: + if isinstance(a, Variable): + return a.value_type + return fallback + + +def align_located(lines: Sequence[Any], values: list[Any]) -> tuple[list[Any], Any]: + """The values of a call that is one per period, its series on other + windows aligned by date as ``a + b`` aligns them (one reading for + both): each materialized onto the combined window by its own + ``extend=`` rule, one that does not cover it refused, a first period + that is short only at the window's start, two grains refused; an + unnamed formula runs on the window of the lines it reads, a plain + line beside a tracked one on the tracked line's window, a line along + keys and a one-value constant on none. ``lines[i]`` is the operand + ``values[i]`` came from (a value typed in has none). Returns the + values and the combined window, a ``TimeLoc`` — or the values as given + and None when fewer than two series run on windows, or all on one + (the positional zip is then by date).""" + if not any(isinstance(v, (list, TrackValues)) for v in values): + return values, None # scalars run on no window + return _on_dates(lines, values) + + +def locate(var: Variable, window: Any) -> None: + """Put ``var`` on ``window`` (a ``TimeLoc``) explicitly, as an operator + puts its result on the combined window: downstream alignment then + knows where it starts, and a renderer that its operands were read by + date there.""" + var._start, var._grain, var._first_day = window + var._read_by_date = True + + +def _typed_blank(raw: Any, fill: Any) -> Any: + """A ``None`` typed into a call read as ``fill``, in a list typed in + too — the formula holds the 0 the renderer writes for it.""" + if raw is None: + return fill + if isinstance(raw, list): + return [fill if v is None else v for v in raw] + return raw + + +def refuse_other_lengths(values: Sequence[Any]) -> None: + """Series of other lengths among one call's values, refused as the + operators refuse them; a one-value series broadcasts. A function with + a loop of its own asks it once it has read lines on other windows by + date (:func:`align_located`), where a series with no window of its + own cannot be paired with the combined window by position.""" + lengths = [len(v) for v in values if isinstance(v, list)] + n = max(lengths, default=0) + for k in lengths: + if k not in (1, n): + raise ValueError(_other_lengths(k, n)) + + +def _other_lengths(k: int, n: int) -> str: + return (f"Cannot broadcast lists of different lengths: {k} vs {n}. " + f"Element-wise ops need equal lengths, or one side of length 1 " + f"which broadcasts over the other.") + + +def refuse_unpaired(lines: Sequence[Any], values: Sequence[Any], window: Any) -> None: + """For a function with a loop of its own, once :func:`align_located` + has read its ``lines``: the series it cannot pair period by period, + refused as the operators refuse them. Over a combined ``window`` every + series must fill it. Otherwise a line whose series is of another + length than the call's longest is refused — its cells hold nothing + for the periods it lacks, so the workbook could not read what a + positional pairing would — while a list typed in keeps its positional + reading, its last value standing in, as the cell writes it. A line + refused is named; a one-value series broadcasts.""" + longest = max((len(v) for v in values if isinstance(v, list)), default=None) + if longest is None: + return + for line, v in zip(lines, values): + if not isinstance(v, list) or len(v) in (1, longest): + continue + if isinstance(line, Variable): + beside = ("" if window is None else + f"; the other lines meet on {longest} period(s) from {window.start}") + raise ValueError( + f"{_line_name(line)} cannot be paired with the call's other series " + f"period by period (different lengths: {len(v)} vs {longest}): it runs " + f"on no window of its own to read by date{beside}. Give it the " + f"periods of the lines beside it, or put it on the window it runs on " + f"(a line of a container, or mo.Variable(..., start=..., grain=...))." + ) + if window is not None: + raise ValueError(_other_lengths(len(v), longest)) + + +def _per_period(values: list[Any], call: Callable[[list[Any]], Any]) -> Any: + """``call`` over the operands of one period at a time; a scalar or a + one-value series stands in every period, as the operators broadcast + it. Scalars only: one call.""" + lists = [v for v in values if isinstance(v, list)] + if not lists: + return call(values) + n = max(len(v) for v in lists) + refuse_other_lengths(lists) + + def at(v: Any, i: int) -> Any: + if isinstance(v, list): + return v[i] if len(v) == n else v[0] + return v + + return [call([at(v, i) for v in values]) for i in range(n)] + + +def _check_arity(func_name: str, pyfunc: Callable[..., Any], n: int) -> None: + """Refuse a ``pyfunc`` that cannot take the call's ``n`` operands at + entry — it would otherwise fail in every period and read as + ``#VALUE!`` throughout. A builtin without a signature (``max``) is + trusted.""" + try: + signature = inspect.signature(pyfunc) + except (TypeError, ValueError): + return + try: + signature.bind(*([None] * n)) + except TypeError: + raise TypeError( + f"{func_name}: the function given takes {signature}, but the call " + f"has {n} operand{'s' if n != 1 else ''}." + ) from None + + +def elementwise( + func_name: str, + pyfunc: Callable[..., Any], + args: Sequence[Any], + *, + value_type: str | None = None, + blank: Any = None, + propagate_errors: bool = True, + exceptions: typing.Literal['marker', 'raise'] = 'marker', + source_code: str | None = None, + render_backends: frozenset | None = None, + typed_blank: Any = None, +) -> Variable: + """Build the Variable for ``func_name`` as one call per period. + + ``pyfunc`` takes one period's operand values positionally and gives + that period's value. Over scalars only it is called once and the + result is a scalar; a list operand gives a series, scalars and + one-value series broadcasting into every period and other length + mismatches refused, as ``a + b`` does; located series on other + windows are aligned by date as ``a + b`` aligns them + (:func:`align_located`), the result on the combined window; lines + along keys combine key by key (:func:`align_keyed`), the result + along the same keys; a tracked operand lifts per coordinate. The + AST is a :class:`FuncCall` over the operands. + + An error marker among a period's operands is that period's result — + the first in reading order — unless ``propagate_errors`` is False, + when the markers reach ``pyfunc`` (``IFERROR`` reads them). A blank + operand (``None``) reaches ``pyfunc`` as is, or as ``blank`` when one + is given (``0`` for arithmetic, ``''`` for text); a ``None`` typed + into the call itself (or into a list typed in) reads as + ``typed_blank`` when one is given — the 0 the renderer writes for it + — while a line's blank cell stays a blank. A call that fails + is the spreadsheet's error: ``ZeroDivisionError`` → ``#DIV/0!``, + ``TypeError`` → ``#VALUE!``, ``OverflowError`` → ``#NUM!`` — unless + ``exceptions='raise'``, when it propagates (``MIN`` / ``MAX`` refuse + a text operand at build time). A ``pyfunc`` that cannot take the + call's operands is refused at entry, never read as ``#VALUE!`` in + every period. + + ``value_type`` defaults to the first Variable operand's, else + ``'float'``. The unit is the function's law (:func:`function_unit`): + the one every unit-bearing operand shares, unless the function + declares another. + """ + from ..core.shape import reject_axised + _check_arity(func_name, pyfunc, len(args)) + raws: list[Any] = [] + ast_args: list = [] + for a in args: + if isinstance(a, Variable): + reject_axised(a, func_name) + raw, node = operand(a) + if typed_blank is not None and not isinstance(a, Variable): + raw = _typed_blank(raw, typed_blank) + raws.append(raw) + ast_args.append(node) + raws, keys = align_keyed(func_name, args, raws) + raws, window = align_located(args, raws) + + def call(values: list[Any]) -> Any: + if propagate_errors: + for v in values: + if is_error(v): + return v + if blank is not None: + values = [blank if v is None else v for v in values] + if exceptions == 'raise': + return pyfunc(*values) + try: + return pyfunc(*values) + except ZeroDivisionError: + return '#DIV/0!' + except TypeError: + return '#VALUE!' + except OverflowError: + return '#NUM!' + + if any(isinstance(r, TrackValues) for r in raws): + value: Any = TrackValues.lift(lambda *fibers: _per_period(list(fibers), call), *raws) + var_type = 'list' if value.time_length else 'scalar' + else: + value = _per_period(raws, call) + var_type = 'list' if isinstance(value, list) else 'scalar' + if value_type is None: + value_type = first_variable_value_type(args, 'float') + if source_code is None: + source_code = f"{func_name}({', '.join(src(a) for a in args)})" + out = make_func_var( + func_name, ast_args, value, value_type, var_type, + source_code=source_code, render_backends=render_backends, any_value_type=True, + ) + return placed(out, window, keys) + + +def placed(out: Variable, window: Any, keys: list[Any] | None) -> Variable: + """``out`` on the combined ``window`` its operands were aligned onto + (:func:`align_located`), along the ``keys`` they ran along + (:func:`align_keyed`), as an operator places its result; either may + be None.""" + if window is not None: + locate(out, window) + if keys is not None and isinstance(out._value, list) and len(out._value) == len(keys): + out._keys = keys + return out + + def pyformula( fn: Callable[..., Any] | None = None, *, name: str | None = None, value_type: str = 'float', + unit: Any = None, ) -> Callable[..., Any]: """Turn an arbitrary Python function into a Variable-returning helper with tracked dependencies. @@ -187,6 +692,11 @@ def minimize(x): datetime results. ``var_type`` is inferred (``'list'`` if the function returns a list, otherwise ``'scalar'``). ``name`` defaults to the wrapped function's ``__name__`` uppercased. + + The engine cannot see inside the function, so the result has no + unit: a ratio of two USD lines is a pure number, a smoothed USD + series is still USD, and only the author knows which. Declare it + with ``unit=`` (``@pyformula(unit='USD')``), as ``set_unit`` would. """ def deco(inner: Callable[..., Any]) -> Callable[..., Any]: func_name = name or inner.__name__.upper() @@ -199,10 +709,11 @@ def wrapper(*args: Any) -> Variable: ] result_value = inner(*raw_args) var_type = 'list' if isinstance(result_value, list) else 'scalar' - return make_func_var( + out = make_func_var( func_name, ast_args, result_value, value_type, var_type, render_backends=frozenset(), ) + return out.set_unit(unit) if unit is not None else out return wrapper diff --git a/src/modeleon/functions/aggregate.py b/src/modeleon/functions/aggregate.py index f3616cd..28e3e34 100644 --- a/src/modeleon/functions/aggregate.py +++ b/src/modeleon/functions/aggregate.py @@ -1,13 +1,20 @@ # SPDX-License-Identifier: Apache-2.0 -"""Aggregation helpers — reduce Variables to a scalar. - -``SUM``, ``MAX``, ``MIN``, ``AVERAGE`` — all Excel-cased to match the cells -they compile to (``=SUM(B2:F2)``, ``=AVERAGE(B2:F2)``) and all variadic like -Excel: ``SUM(a, b, c)``, ``SUM(range)``, ``SUM(1, 2, 3)`` all work. - -``MIN`` / ``MAX`` over two or more operands with a series among them are -not a reduction but one call per period — the cap and the floor a -workbook writes in every column (``=MIN(C2, C3)``); see :func:`MIN`. +"""Aggregation helpers — ``SUM``, ``MAX``, ``MIN``, ``AVERAGE``. + +All Excel-cased to match the cells they compile to (``=SUM(B2:F2)``, +``=AVERAGE(B2:F2)``) and all variadic like Excel: ``SUM(a, b, c)``, +``SUM(range)``, ``SUM(1, 2, 3)`` all work. + +One rule for every aggregate: several series are combined period by +period, as ``a + b`` combines them; one series alone is reduced over its +periods; scalars only stay one number. ``SUM(revenue)`` is the total of +the row, ``=SUM(B2:F2)``; ``SUM(product_a, product_b)`` is the two lines' +total in every period, ``=SUM(B2, B3)`` in each column — a subtotal, as +``MAX(cash, floor)`` is a floor and ``MIN(sweep, balance)`` a cap; the +total of several rows over all their periods is ``SUM(a) + SUM(b)``. A +list of scalar lines given as one argument is its lines, one by one +(``SUM([p, q])`` is ``SUM(p, q)``); a list holding a series is refused, +since it reads as either of those two — the call names both. **Invariant:** every call returns a :class:`Variable` with a tracked formula — even when all operands are plain numbers. ``mo.SUM(1, 2, 3)`` @@ -18,11 +25,21 @@ from __future__ import annotations import builtins +import functools +import operator from typing import Any, List, Union from ..core.expr import Literal, VarRef from ..core.variable import Variable -from ._helpers import make_func_var, operand, src +from ._helpers import ( + _typed_blank, + elementwise, + first_variable_value_type, + is_series, + make_func_var, + src, +) +from .registry import per_period_arity, register Number = Union[int, float] AggOperand = Union[Variable, Number, List[Number]] @@ -54,49 +71,23 @@ def _flatten_args(args: tuple, func_name: str) -> tuple[List[Any], list]: flat.extend(a._value if isinstance(a._value, list) else [a._value]) ast_args.append(VarRef(a)) elif isinstance(a, list): - flat.extend(a) + # A ``None`` typed in is the 0 the formula holds; a line's + # blank cell (above) stays a blank the reduction skips. + flat.extend(_typed_blank(a, 0)) ast_args.append(Literal(a)) else: - flat.append(a) + flat.append(_typed_blank(a, 0)) ast_args.append(Literal(a)) return flat, ast_args -def _first_variable_value_type(args: tuple, fallback: str = 'float') -> str: - for a in args: - if isinstance(a, Variable): - return a.value_type - return fallback - - -def _common_unit(args: tuple): - """The single unit shared by every unit-bearing Variable operand — - an aggregation of $ is $. Mixed or absent units → None (never - guess).""" - from ..core.unit import unit_key - units = { - unit_key(a._unit) for a in args - if isinstance(a, Variable) and getattr(a, '_unit', None) is not None - } - if len(units) != 1: - return None - for a in args: - if isinstance(a, Variable) and getattr(a, '_unit', None) is not None: - return a._unit - return None - - def _result(func_name: str, args: tuple, ast_args: list, value: Any, value_type: str, var_type: str) -> Variable: source_code = f"{func_name}({', '.join(src(a) for a in args)})" - out = make_func_var( + return make_func_var( func_name, ast_args, value, value_type, var_type, - source_code=source_code, + source_code=source_code, any_value_type=True, ) - unit = _common_unit(args) - if unit is not None: - out._unit = unit - return out def _aggregate(func_name: str, args: tuple, value: Any, value_type: str) -> Variable: @@ -104,62 +95,102 @@ def _aggregate(func_name: str, args: tuple, value: Any, value_type: str) -> Vari return _result(func_name, args, ast_args, value, value_type, 'scalar') -def _is_series(a: Any) -> bool: - """A period-shaped operand: a list, or a Variable holding one (a row, - or a row per track).""" - from ..core.tracks import TrackValues - if isinstance(a, Variable): - return isinstance(a._value, (list, TrackValues)) - return isinstance(a, list) - - -def _per_period(values: List[Any], excel: Any) -> Any: - """``excel`` over the operands of one period at a time; a scalar or - a one-value series stands in every period, as the operators - broadcast it. Scalars only: one call.""" - lists = [v for v in values if isinstance(v, list)] - if not lists: - return excel(values) - n = builtins.max(len(v) for v in lists) - for v in lists: - if len(v) not in (1, n): - raise ValueError( - f"Cannot broadcast lists of different lengths: {len(v)} vs {n}. " - f"Element-wise ops need equal lengths, or one side of length 1 " - f"which broadcasts over the other." - ) - - def at(v: Any, i: int) -> Any: - if isinstance(v, list): - return v[i] if len(v) == n else v[0] - return v - - return [excel([at(v, i) for v in values]) for i in range(n)] +#: Each function's one number over every cell of two rows, for the +#: refusal of a list holding a series: what it is called, how it is +#: spelled (``{more}`` / ``{also}`` / ``{times}`` mark where further lines +#: go). +_OVER_ALL = { + "SUM": ("the total", "mo.SUM({a}) + mo.SUM({b}){more}"), + "AVERAGE": ("the mean", "(mo.SUM({a}) + mo.SUM({b}){more}) / " + "(mo.COUNT({a}) + mo.COUNT({b}){more})"), + "MIN": ("the smallest", "mo.MIN(mo.MIN({a}), mo.MIN({b}){also})"), + "MAX": ("the largest", "mo.MAX(mo.MAX({a}), mo.MAX({b}){also})"), + "PRODUCT": ("the product", "mo.PRODUCT({a}) * mo.PRODUCT({b}){times}"), + "COUNT": ("the count", "mo.COUNT({a}) + mo.COUNT({b}){more}"), + "COUNTA": ("the count", "mo.COUNTA({a}) + mo.COUNTA({b}){more}"), +} + + +def _lists_lines(arg: Any) -> bool: + return isinstance(arg, (list, tuple)) and any(isinstance(i, Variable) for i in arg) + + +def _refuse_listed_series(func_name: str, args: tuple) -> None: + """A list of lines given as one argument with a series in it reads as + two calls — one per period over the lines' cells, or one number over + every cell — and Python cannot know which was meant: refused, both + spelled out with the lines' own names.""" + for arg in args: + if not _lists_lines(arg): + continue + lines = [i for i in arg if isinstance(i, Variable)] + series = next((i for i in lines if is_series(i)), None) + if series is None: + continue + names = [src(i) for i in lines] + listed = f"[{', '.join(src(i) if isinstance(i, Variable) else repr(i) for i in arg)}]" + noun, spelled = _OVER_ALL[func_name] + if len(names) >= 2: + more = len(names) > 2 + per = (f"the lines one by one, mo.{func_name}({names[0]}, {names[1]}" + f"{', …' if more else ''})") + over = spelled.format(a=names[0], b=names[1], more=" + …" if more else "", + also=", …" if more else "", times=" * …" if more else "") + else: + per = f"the line itself, {names[0]}" + over = f"mo.{func_name}({names[0]})" + raise ValueError( + f"mo.{func_name}({listed}) gives its lines as one list, and {src(series)} in " + f"it is a series, so the call reads two ways: per period, {per}; {noun} over " + f"all their periods, {over}. Write the one you mean." + ) -def _elementwise(func_name: str, args: tuple, reduce: Any) -> Variable: - """``MIN`` / ``MAX`` over two or more operands with a series among - them: one call per period, so the cell per period is ``=MIN(C2, - C3)``. Tracks lift per coordinate, as under the operators.""" - from ..core.shape import reject_axised - from ..core.tracks import TrackValues - raws: List[Any] = [] - ast_args: list = [] - for a in args: - if isinstance(a, Variable): - reject_axised(a, func_name) - raw, node = operand(a) - raws.append(raw) - ast_args.append(node) - excel = _excel_reduce(reduce) - if any(isinstance(r, TrackValues) for r in raws): - value: Any = TrackValues.lift(lambda *fibers: _per_period(list(fibers), excel), *raws) - var_type = 'list' if value.time_length else 'scalar' - else: - value = _per_period(raws, excel) - var_type = 'list' - return _result(func_name, args, ast_args, value, - _first_variable_value_type(args, 'float'), var_type) +def _listed(func_name: str, args: tuple) -> tuple: + """The operands of a call given a list of scalar lines (and values) as + one argument: its items, one by one — ``mo.SUM([p, q])`` is + ``mo.SUM(p, q)``, written ``=SUM(B1, B2)``. A list holding a series is + refused (``_refuse_listed_series``).""" + _refuse_listed_series(func_name, args) + if not any(_lists_lines(a) for a in args): + return args + out: List[Any] = [] + for arg in args: + if not _lists_lines(arg): + out.append(arg) + continue + for item in arg: + if isinstance(item, (list, tuple, dict, set)): + raise TypeError( + f"{func_name}(): an item of a list of lines is a line or one value; " + f"got {type(item).__name__}." + ) + out.append(item) + return tuple(out) + + +def _by_period(func_name: str, args: tuple) -> bool: + """Is the call one per period: as many operands as the registry's + ``per_period_arity`` or more, a series among them?""" + arity = per_period_arity(func_name) + return arity is not None and len(args) >= arity and any(is_series(a) for a in args) + + +def _elementwise(func_name: str, args: tuple, reduce: Any, empty: Any = 0, + value_type: str | None = None) -> Variable: + """Two or more operands with a series among them: one call per + period, so the cell per period is ``=SUM(C2, C3)``. The period's + operands reduce as one range does (an error is the result, a blank is + skipped, nothing left is ``empty``), a ``None`` typed into the call + being the 0 the formula holds; located series on other windows are + aligned by date, tracks lift per coordinate and lines along the same + keys key by key, as under the operators. A text operand is refused + at build time (``TypeError``), not read as ``#VALUE!``: the + spreadsheet skips text in a referenced cell, so no error value would + match the workbook.""" + excel = _excel_reduce(reduce, empty, func_name) + return elementwise(func_name, lambda *values: excel(list(values)), args, + value_type=value_type, exceptions='raise', typed_blank=0) def _lifted_reduce(func_name: str, var: Variable, reducer) -> Variable: @@ -177,25 +208,38 @@ def _lifted_reduce(func_name: str, var: Variable, reducer) -> Variable: var._value, ) label = getattr(var, "python_name", None) or var.path.leaf - out = make_func_var( + return make_func_var( func_name, [VarRef(var)], lifted, 'float', 'scalar', - source_code=f"{func_name}({label})", + source_code=f"{func_name}({label})", any_value_type=True, ) - if getattr(var, '_unit', None) is not None: - out._unit = var._unit - return out -def _excel_reduce(reduce: Any, empty: Any = 0) -> Any: +# How a refusal names what the function does with its numbers. +_VERBS = {"SUM": "adds", "AVERAGE": "averages", "MIN": "compares", "MAX": "compares", + "PRODUCT": "multiplies"} + + +def _excel_reduce(reduce: Any, empty: Any = 0, name: str | None = None) -> Any: """``reduce`` over one range as the spreadsheet reads it: an error in the range is the result — the first one, in reading order — and a hole (a cell not entered yet) is skipped, as a blank is. With nothing left the result is ``empty``: 0 for SUM, MAX and MIN, ``#DIV/0!`` for - AVERAGE.""" + AVERAGE. Given the function's ``name``, text among the values is + refused at build time (``TypeError``) with a message naming both: the + spreadsheet skips text in a range, so no error value would match the + workbook.""" def excel(values: List[Any]) -> Any: for value in values: if isinstance(value, str) and value.startswith("#"): return value + if name is not None: + for value in values: + if isinstance(value, str): + raise TypeError( + f"{name} {_VERBS.get(name, 'reads')} numbers; {value!r} is not " + f"one. The spreadsheet skips text in a range, so no error value " + f"would match the workbook — give it the number." + ) known = [value for value in values if value is not None] return reduce(known) if known else empty return excel @@ -205,26 +249,48 @@ def _mean(values: List[Any]) -> Any: return builtins.sum(values) / len(values) +# The operands of one call added left to right, as ``a + b + c`` adds +# them: ``sum()`` compensates the rounding, and a period's subtotal would +# differ in its last digit from the same lines added with ``+``. +def _added(values: List[Any]) -> Any: + return functools.reduce(operator.add, values) + + +def _mean_added(values: List[Any]) -> Any: + return _added(values) / len(values) + + def SUM(*args: AggOperand) -> Variable: """Sum of Variables, scalars, or lists (renders ``=SUM(...)``). - Accepts ``SUM(a, b, c)``, ``SUM(list_var)``, ``SUM([1, 2, 3])``, or - ``SUM(1, 2, 3)``. Always returns a Variable with a tracked formula — - the Excel cell is the product, not the value. + Over ONE operand, the total of the row over its periods: + ``mo.SUM(revenue)`` → ``=SUM(B2:F2)``. Over two or more operands with + a series among them, the total PER PERIOD — a subtotal of lines, as + ``a + b`` adds them — with scalars broadcast and a line with tracks + taken track by track: ``mo.SUM(product_a, product_b)`` → ``=SUM(B2, + B3)`` in every column, inside a loop container too. Over scalars + only, one number. The total of several rows over all their periods + is ``mo.SUM(a) + mo.SUM(b)``. Always returns a Variable with a + tracked formula — the Excel cell is the product, not the value. Example:: total = mo.SUM(revenues) # =SUM(B2:F2) - total = mo.SUM(q1, q2, q3, q4) # =SUM(q1, q2, q3, q4) + subtotal = mo.SUM(a, b, c) # each period: =SUM(B2, B3, B4) + total = mo.SUM(q1, q2, q3, q4) # scalar lines: =SUM(B1, B2, B3, B4) total = mo.SUM(1, 2, 3) # =SUM(1, 2, 3), value = 6 """ + args = _listed("SUM", args) + if _by_period("SUM", args): + return _elementwise("SUM", args, _added) from ..core.tracks import TrackValues if (len(args) == 1 and isinstance(args[0], Variable) and isinstance(args[0]._value, TrackValues)): - return _lifted_reduce("SUM", args[0], _excel_reduce(builtins.sum)) + return _lifted_reduce("SUM", args[0], _excel_reduce(builtins.sum, name="SUM")) flat, _ = _flatten_args(args, "SUM") - return _aggregate("SUM", args, _excel_reduce(builtins.sum)(flat), - _first_variable_value_type(args, 'float')) + add = builtins.sum if len(args) == 1 else _added + return _aggregate("SUM", args, _excel_reduce(add, name="SUM")(flat), + first_variable_value_type(args, 'float')) def MAX(*args: AggOperand) -> Variable: @@ -237,15 +303,16 @@ def MAX(*args: AggOperand) -> Variable: ``=MAX(C4, Inputs!B2)`` in every column, inside a loop container too. Over scalars only, one number: ``mo.MAX(a, b)`` → ``=MAX(B2, B3)``. """ - if len(args) >= 2 and any(_is_series(a) for a in args): + args = _listed("MAX", args) + if _by_period("MAX", args): return _elementwise("MAX", args, builtins.max) from ..core.tracks import TrackValues if (len(args) == 1 and isinstance(args[0], Variable) and isinstance(args[0]._value, TrackValues)): - return _lifted_reduce("MAX", args[0], _excel_reduce(builtins.max)) + return _lifted_reduce("MAX", args[0], _excel_reduce(builtins.max, name="MAX")) flat, _ = _flatten_args(args, "MAX") - return _aggregate("MAX", args, _excel_reduce(builtins.max)(flat), - _first_variable_value_type(args, 'float')) + return _aggregate("MAX", args, _excel_reduce(builtins.max, name="MAX")(flat), + first_variable_value_type(args, 'float')) def MIN(*args: AggOperand) -> Variable: @@ -258,23 +325,49 @@ def MIN(*args: AggOperand) -> Variable: every column, inside a loop container too. Over scalars only, one number: ``mo.MIN(a, b)`` → ``=MIN(B2, B3)``. """ - if len(args) >= 2 and any(_is_series(a) for a in args): + args = _listed("MIN", args) + if _by_period("MIN", args): return _elementwise("MIN", args, builtins.min) from ..core.tracks import TrackValues if (len(args) == 1 and isinstance(args[0], Variable) and isinstance(args[0]._value, TrackValues)): - return _lifted_reduce("MIN", args[0], _excel_reduce(builtins.min)) + return _lifted_reduce("MIN", args[0], _excel_reduce(builtins.min, name="MIN")) flat, _ = _flatten_args(args, "MIN") - return _aggregate("MIN", args, _excel_reduce(builtins.min)(flat), - _first_variable_value_type(args, 'float')) + return _aggregate("MIN", args, _excel_reduce(builtins.min, name="MIN")(flat), + first_variable_value_type(args, 'float')) def AVERAGE(*args: AggOperand) -> Variable: - """Mean across Variables, scalars, or lists (renders ``=AVERAGE(...)``).""" + """Mean across Variables, scalars, or lists (renders ``=AVERAGE(...)``). + + Over ONE operand, the mean of the row over its periods: + ``mo.AVERAGE(price)`` → ``=AVERAGE(B2:F2)``. Over two or more operands + with a series among them, the mean PER PERIOD, with scalars + broadcast and a line with tracks taken track by track: + ``mo.AVERAGE(plan, actual)`` → ``=AVERAGE(B2, B3)`` in every column; a + period whose cells are all blank is ``#DIV/0!``. Over scalars only, + one number. + """ + args = _listed("AVERAGE", args) + if _by_period("AVERAGE", args): + return _elementwise("AVERAGE", args, _mean_added, "#DIV/0!", 'float') from ..core.tracks import TrackValues if (len(args) == 1 and isinstance(args[0], Variable) and isinstance(args[0]._value, TrackValues)): - return _lifted_reduce("AVERAGE", args[0], _excel_reduce(_mean, "#DIV/0!")) + return _lifted_reduce("AVERAGE", args[0], _excel_reduce(_mean, "#DIV/0!", "AVERAGE")) flat, _ = _flatten_args(args, "AVERAGE") - value = _excel_reduce(_mean, "#DIV/0!")(flat) + mean = _mean if len(args) == 1 else _mean_added + value = _excel_reduce(mean, "#DIV/0!", "AVERAGE")(flat) return _aggregate("AVERAGE", args, value, 'float') + + +register("SUM", shape="reduction", per_period_arity=2) +register("AVERAGE", shape="reduction", per_period_arity=2) +register("MAX", shape="reduction", per_period_arity=2) +register("MIN", shape="reduction", per_period_arity=2) +# ``COUNT``'s helper lives beside the conditional counts (functions/criteria.py). +register("COUNT", shape="reduction", per_period_arity=2) +# Written by the renderer without a helper of its own, for ``SUM(a * b)`` +# over two rows — its arrays are ranges only (a comma list would be read +# as more arrays). +register("SUMPRODUCT", shape="range", bare=True) diff --git a/src/modeleon/functions/conditional.py b/src/modeleon/functions/conditional.py index 89e6b8c..1fd49e4 100644 --- a/src/modeleon/functions/conditional.py +++ b/src/modeleon/functions/conditional.py @@ -3,15 +3,24 @@ All are Variable-aware and element-wise over list (period-indexed) operands, and render as their Excel equivalents (``=IF(...)``, -``=AND(...)``, ``=CHOOSE(...)``, …). +``=AND(...)``, ``=CHOOSE(...)``, …). Lines on other time windows are +read by date, as ``a + b`` reads them — each continued by its own +``extend=`` rule, the result on the combined window — and lines along +keys key by key. """ from __future__ import annotations from typing import Any +from ..core.shape import line_keys, reject_axised +from ..core.tracks import TrackValues from ..core.variable import Variable -from ._helpers import make_func_var, operand, text_operand +from ._helpers import ( + align_keyed, align_located, make_func_var, operand, placed, refuse_unpaired, + text_operand, +) +from .registry import register def _is_error(value: Any) -> bool: @@ -27,6 +36,15 @@ def _blank0(value: Any) -> Any: return 0 if value is None else value +# A branch's value in period ``i``: a list's own, its last value standing in +# past its end — a one-value series in every period, a shorter list typed in +# as its cell is written (a line of another length is refused before). +def _at(branch: Any, i: int) -> Any: + if isinstance(branch, list): + return branch[i] if i < len(branch) else branch[-1] + return branch + + def _value_type_of(sample: Any) -> str: """Modeleon value-type tag for a computed scalar sample.""" if isinstance(sample, bool): @@ -52,7 +70,6 @@ def IF(condition: Any, value_if_true: Any, value_if_false: Any) -> Variable: rating = mo.IF(ebitda > 200_000, "Excellent", mo.IF(ebitda > 100_000, "Good", "Poor")) """ - from ..core.shape import reject_axised for _arg, _where in ((condition, "IF(condition)"), (value_if_true, "IF(value_if_true)"), (value_if_false, "IF(value_if_false)")): @@ -60,17 +77,21 @@ def IF(condition: Any, value_if_true: Any, value_if_false: Any) -> Variable: cond_value, cond_expr = text_operand(condition) true_value, true_expr = text_operand(value_if_true) false_value, false_expr = text_operand(value_if_false) + args = (condition, value_if_true, value_if_false) + raws, keys = align_keyed("IF", args, [cond_value, true_value, false_value]) + (cond_value, true_value, false_value), window = align_located(args, raws) + if isinstance(cond_value, (list, TrackValues)): + # A condition per period pairs each period's branches; one + # condition chooses a branch whole, pairing nothing. + refuse_unpaired(args, [cond_value, true_value, false_value], window) - from ..core.tracks import TrackValues if any(isinstance(v, TrackValues) for v in (cond_value, true_value, false_value)): # Rank lifting: the ternary evaluates per coordinate; plain # operands broadcast into every role (mismatch law inside lift). def _tern(c, t, f): if isinstance(c, list): return [ - cv if _is_error(cv) - else _blank0((t[i] if isinstance(t, list) else t) if cv - else (f[i] if isinstance(f, list) else f)) + cv if _is_error(cv) else _blank0(_at(t, i) if cv else _at(f, i)) for i, cv in enumerate(c) ] return c if _is_error(c) else _blank0(t if c else f) @@ -80,13 +101,12 @@ def _tern(c, t, f): "IF", [cond_expr, true_expr, false_expr], lifted, 'float', 'list' if lifted.time_length else 'scalar', ) - return result + return placed(result, window, None) if isinstance(cond_value, list): value = [ c if _is_error(c) - else _blank0((true_value[i] if isinstance(true_value, list) else true_value) if c - else (false_value[i] if isinstance(false_value, list) else false_value)) + else _blank0(_at(true_value, i) if c else _at(false_value, i)) for i, c in enumerate(cond_value) ] var_type = 'list' @@ -124,8 +144,9 @@ def _tern(c, t, f): result = make_func_var( "IF", [cond_expr, true_expr, false_expr], value, value_type, var_type, ) - result._source_code = result.formula - return result + # The readable source is the formula itself, rendered when read. + result._source_from_formula() + return placed(result, window, keys) def _reduce_logical(reduce_fn, values: list) -> Any: @@ -156,6 +177,9 @@ def _logical(func_name: str, reduce_fn, *args: Any) -> Variable: v, e = text_operand(a) raws.append(v) exprs.append(e) + raws, keys = align_keyed(func_name, args, raws) + raws, window = align_located(args, raws) + refuse_unpaired(args, raws, window) if any(isinstance(r, list) for r in raws): n = max(len(r) for r in raws if isinstance(r, list)) @@ -172,8 +196,8 @@ def at(r: Any, i: int) -> Any: var_type = 'scalar' result = make_func_var(func_name, exprs, value, 'bool', var_type) - result._source_code = result.formula - return result + result._source_from_formula() + return placed(result, window, keys) def AND(*args: Any) -> Variable: @@ -200,8 +224,8 @@ def NOT(value: Any) -> Variable: calc = raw if _is_error(raw) else not bool(raw) var_type = 'scalar' result = make_func_var("NOT", [expr], calc, 'bool', var_type) - result._source_code = result.formula - return result + result._source_from_formula() + return placed(result, None, align_keyed("NOT", [value], [raw])[1]) def ISBLANK(value: Any) -> Variable: @@ -221,8 +245,8 @@ def ISBLANK(value: Any) -> Variable: calc = raw is None or raw == '' var_type = 'scalar' result = make_func_var("ISBLANK", [expr], calc, 'bool', var_type) - result._source_code = result.formula - return result + result._source_from_formula() + return placed(result, None, align_keyed("ISBLANK", [value], [raw])[1]) def CHOOSE(index: Any, *choices: Any) -> Variable: @@ -232,12 +256,118 @@ def CHOOSE(index: Any, *choices: Any) -> Variable: or, element-wise, ``mo.CHOOSE(phase_flag + 1, "A", "F")`` where ``phase_flag`` is a per-period Variable. A single index over series choices gives a series: each period picks its own value of the chosen - series. Out-of-range indices yield ``#VALUE!`` (matching Excel). + series (a tracked choice, track by track). A choice that runs along + keys — a keyed list ``mo.Variable({"Bear": 0.9, "Base": 1.0, "Bull": + 1.1})`` or a line on one finite axis that is not time — is, on its + own, one choice per key in key order: ``mo.CHOOSE(pick, growth)`` is + ``mo.CHOOSE(pick, growth["Bear"], growth["Base"], growth["Bull"])``, + one cell per choice in the workbook. Out-of-range indices yield + ``#VALUE!`` (matching Excel). """ + from ..core.tracks import TrackValues + + def _label(v: Variable) -> str: + if v._display_name or v.python_name: + return repr(v._display_name or v.python_name) + return f"the line {v.formula}" if v.formula else "an unnamed line" + + def _code(v: Variable) -> str: + if v.python_name: + return v.python_name + return f"({v.formula})" if v.formula else (v._display_name or "the_line") + + variables = [(role, a) for role, a in (('index', index), *(('choice', c) for c in choices)) + if isinstance(a, Variable)] + for role, arg in variables: + n_axes = len(getattr(arg, '_indexed_by', ()) or ()) + if n_axes >= 2: + raise ValueError( + f"CHOOSE: {_label(arg)} is laid out along {n_axes} axes — " + + ("an index is one number, or one per period, that picks a choice " + "by position. " if role == 'index' else + "a choice is one value, one per period, or one line along ONE set " + "of keys (a keyed list, or one finite axis), spread into one choice " + "per key. ") + + f"Pick one cell, {_code(arg)}[0], or lay the line along one axis." + ) + if isinstance(getattr(index, '_value', None), TrackValues): + roles = list(index._value.roles) + raise ValueError( + f"CHOOSE: the index {_label(index)} is a tracked line (tracks: " + f"{', '.join(roles)}) — an index is one number, or one per period, that " + f"picks a choice by position. Read one track first " + f"({_code(index)}.{roles[0]})." + ) + if isinstance(index, Variable) and line_keys(index) is not None: + keys = line_keys(index) or [] + first = (f"{_code(index)}[{index._keys[0]!r}]" if index._keys is not None + else f"{_code(index)}[0]") + raise ValueError( + f"CHOOSE: the index {_label(index)} runs along keys {', '.join(keys)} — an " + f"index is one number, or one per period, that picks a choice by " + f"position. Pick one key first ({first}), or make the choices run along " + f"these keys and give CHOOSE one number." + ) + choice_vars = [c for c in choices if isinstance(c, Variable)] + tracked_choices = [ + c for c in choice_vars if isinstance(getattr(c, '_value', None), TrackValues) + ] + if tracked_choices and isinstance(getattr(index, '_value', None), list): + c0 = tracked_choices[0] + roles = list(c0._value.roles) + raise ValueError( + f"CHOOSE: the index {_label(index)} is a series and {_label(c0)} is a tracked " + f"line (tracks: {', '.join(roles)}) — each period would pick a whole tracked " + f"line. Give CHOOSE one number, or read one track of each choice first " + f"({_code(c0)}.{roles[0]})." + ) + + # A choice that runs along keys is a SET of choices: one per key, in + # key order, each its own cell in the workbook — the long form + # ``CHOOSE(pick, growth["Bear"], growth["Base"], …)`` written short. + # Only on its own: beside other choices the numbering would depend on + # how many keys the line has, and two keyed lines would flatten into + # one row of cells. + along = [c for c in choice_vars if line_keys(c) is not None] + if along and len(choices) > 1: + c0 = along[0] + keys = line_keys(c0) or [] + cells = ", ".join( + f'{_code(c0)}[{k!r}]' if isinstance(k, str) else f"{_code(c0)}[{i}]" + for i, k in enumerate(c0._keys if c0._keys is not None else range(len(keys))) + ) + raise ValueError( + f"CHOOSE: {_label(c0)} runs along keys {', '.join(keys)} beside " + f"{len(choices) - 1} other choice(s) — CHOOSE spreads ONE line along its " + f"keys into one choice per key, and only on its own: " + f"mo.CHOOSE(pick, {_code(c0)}). With several choices name the cells: " + f"mo.CHOOSE(pick, {cells}, …), one cell per choice." + ) + spread: list[Any] = list(choices) + if along: + c0 = along[0] + if c0._expr is not None and getattr(c0, '_owner', None) is None: + raise ValueError( + f"CHOOSE: {_label(c0)} runs along keys but is a computed line with no " + f"cells of its own — spread by its cells, the workbook would hold its " + f"numbers, not formulas. Name it first (lines.growth_adj = {c0.formula}), " + f"then mo.CHOOSE(pick, lines.growth_adj)." + ) + if c0._keys is not None: + # A key that is not text is read as a POSITION by ``[]`` and by + # the workbook alike — spread by position so both agree. + spread = [c0[k] if isinstance(k, str) else c0[i] for i, k in enumerate(c0._keys)] + else: + spread = [c0[i] for i in range(len(line_keys(c0) or []))] + idx_raw, idx_expr = operand(index) - pairs = [text_operand(c) for c in choices] - choice_raws = [p[0] for p in pairs] + pairs = [text_operand(c) for c in spread] choice_exprs = [p[1] for p in pairs] + lines = [index, *spread] + raws, along_keys = align_keyed("CHOOSE", lines, [idx_raw, *(p[0] for p in pairs)]) + raws, window = align_located(lines, raws) + refuse_unpaired(lines, raws, window) + idx_raw, choice_raws = raws[0], raws[1:] def pick(i_val: Any, period: int) -> Any: if _is_error(i_val): @@ -282,5 +412,13 @@ def pick(i_val: Any, period: int) -> Any: result = make_func_var( "CHOOSE", [idx_expr, *choice_exprs], value, _value_type_of(sample), var_type, ) - result._source_code = result.formula - return result + result._source_from_formula() + return placed(result, window, along_keys) + + +register("IF", shape="elementwise") +register("AND", shape="elementwise") +register("OR", shape="elementwise") +register("NOT", shape="elementwise") +register("ISBLANK", shape="elementwise") +register("CHOOSE", shape="elementwise") diff --git a/src/modeleon/functions/criteria.py b/src/modeleon/functions/criteria.py new file mode 100644 index 0000000..ab21fb1 --- /dev/null +++ b/src/modeleon/functions/criteria.py @@ -0,0 +1,1581 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Conditional aggregates — ``SUMIF``, ``SUMIFS``, ``COUNTIF``, ``COUNTIFS``, +``AVERAGEIF``, ``AVERAGEIFS``, ``MAXIFS``, ``MINIFS`` — and the counts +``COUNT`` and ``COUNTA``. + +Each reads whole ranges and gives one number, written as the spreadsheet +writes it: ``=SUMIF(B2:F2, ">=10", B3:F3)``. The counts follow the rule +every aggregate follows: over two or more operands with a series among +them they are one call per period, as ``a + b`` combines its operands +(``=COUNT(B2, B3)`` in every column), over one series the count of its +cells, over scalars only one number. A range argument is a row +— a list-valued Variable, written as the range of its cells — or a +Python list of scalar lines whose cells stand one under another in a +column (a field down the records of a table, a block of scalar lines), +written as a vertical range, ``Deals!C3:C5``; one element of a row, +``flow[0]``, is such a line too. Cells that form no run cannot be one +range: the cell carries the computed value and ``check_compat`` lists it +— never a comma list, which these functions read as further arguments. + +The criteria is read as the spreadsheet reads it, once, in +:func:`parse_criteria` and :func:`matches`: a number, a date or a boolean +means equality; a string carries an optional operator — ``'>=10'``, +``'<>0'``, ``'<5'`` — and without one means equality; what follows the +operator is read as a cell entry: a number (``'10'``, ``'-2.5'``, +``'1e3'``, ``'5%'``), ``TRUE`` / ``FALSE``, a date written ``2026-01-31`` +(a time of day may follow: ``2026-01-31 12:00``), else text. An entry +that a spreadsheet reads as a number or a date in some locales and as +text in others — ``'1,000'``, ``'$100'``, ``'(5)'``, ``'12:00'``, +``'1/31/2026'``, ``'Jan 31, 2026'`` — is refused when the model is built, +with the form to write instead. Text compares case-insensitively, whole +cell, with the wildcards ``*`` (any run of characters) and ``?`` (one +character), ``~`` taking the character after it literally. A scalar +Variable is its value (a cell holding ``'>=10'`` is that criteria) and +is written as that cell; ``('>=', threshold)`` or +``mo.CONCAT('>=', threshold)`` joins an operator to a cell and is +written as ``">=" & B2``. A per-period criteria is not one: the +conditional sum for one period is a call over that period's column of +the records, the vertical case above. + +The edges, as the spreadsheet reads them. Two numbers are equal when +they differ by less than 2**-48 of the smaller (``0.1 + 0.2`` meets +``0.3``), as the spreadsheet compares them. A date is a number — its +serial — so a date cell meets a numeric criteria and a number cell a +date one. Text that reads as a number meets an equality (the text +``'10'`` equals 10) but never an ordering (``'>5'``); text equal to the +criteria's own text meets it too (the text ``TRUE`` meets ``'TRUE'``); +a boolean is never a number. ``''`` matches a blank cell and empty +text, ``'='`` only a blank cell, ``'<>'`` whatever is not blank; an +ordering with nothing after it compares text with the empty text +(``'>='`` meets every text, ``'<'`` none). ``'<>x'`` matches blanks, +numbers and errors too. An error cell matches only its own text +(``'#DIV/0!'``) or a ``'<>'``; a criteria cell holding an error is the +result. ``None`` as the criteria, typed or read from a cell left blank, +is the number 0, as a reference to an empty cell reads; joined to an +operator, ``('=', None)``, it is the operator alone, as a blank cell +joined to one is. Empty text typed into a line (``''``) is a blank cell +in the workbook and is read as one; the empty text a formula gives is a +cell: ``'<>'`` and ``COUNTA`` count it, ``''`` matches it, ``'='`` does +not. In the summed, averaged or ranked range a blank, a text and a +boolean are skipped, a date is its serial (``MAXIFS`` over dates gives +the date, ``AVERAGEIF`` the mean date, ``SUMIF`` a number of days), and +an error among the MATCHED cells is the result; nothing matched sums to +0, averages to ``#DIV/0!`` and ranks (``MAXIFS``, ``MINIFS``) to 0. Text +orders case-insensitively by code point (``'=', cap)`` with ``cap`` in hours +over amounts in USD is refused, as ``amount >= cap`` is. A typed +criteria and an unlabelled line go with any; a count has none. + +LibreOffice departs from Excel in three places, and the engine keeps +Excel's rule: there a boolean cell is a number (``TRUE`` meets ``'>0'`` +and is counted and summed), numeric text in a criteria range meets a +numeric criteria only as text (the text ``'10'`` meets ``"10"``, not +the number 10), and a wildcard matches a number's digits. + +Not covered: a criteria over a two-dimensional block (a table's period +block as one range), union ranges (``(B2:B5, D2:D5)``), and a criteria +that is a series. +""" + +from __future__ import annotations + +import bisect +import builtins +import math +import operator +import re +import unicodedata +import weakref +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import date, datetime, time, timedelta +from functools import partial +from itertools import chain +from typing import Any, Callable, Iterator, Sequence + +from ..core.expr import Expr, FuncCall, ListExpr, Literal, Subscript, VarRef +from ..core.shape import reject_axised +from ..core.unit import UNSETTLED +from ..core.variable import Variable +from ..core.variable_ops import _to_serial +from ._helpers import ( + UnitLaw, _per_period, align_keyed, align_located, is_error, make_func_var, placed, + operand_unit, same_unit, + shared_unit, src, unit_law, +) +from .aggregate import _excel_reduce, _mean, _refuse_listed_series +from .registry import per_period_arity, register + +Number = int | float +RangeArg = Variable | Sequence[Any] + +# ─── The criteria, as the spreadsheet reads it ────────────────────── + +#: Longest first, so ``'<>'`` and ``'>='`` are read before ``'<'`` and ``'>'``. +_OPERATORS = ('<>', '>=', '<=', '=', '>', '<') + +_NUMBER = re.compile(r'[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+|%)?', re.ASCII) +#: A date as every spreadsheet reads it, a time of day after it or not. +_ISO_DATE = re.compile( + r'(\d{4})-(\d{1,2})-(\d{1,2})(?: +(\d{1,2}):(\d{2})(?::(\d{2})(?:\.(\d{1,6}))?)?)?', + re.ASCII) +#: The ISO form with a ``T`` before the time, which not every spreadsheet reads. +_ISO_T = re.compile(r'\d{4}-\d{1,2}-\d{1,2}T\d{1,2}:\d{2}(?::\d{2}(?:\.\d+)?)?', + re.ASCII | re.IGNORECASE) +_DIGITS = frozenset('0123456789') +_LETTERS = re.compile(r'[^\W\d_]+') +#: What a number, a date or a time is written with besides digits: the +#: words of a date or a time of day, and the marks around the digits (a +#: currency sign besides). +_DATE_WORDS = frozenset({ + 'jan', 'january', 'feb', 'february', 'mar', 'march', 'apr', 'april', 'may', 'jun', + 'june', 'jul', 'july', 'aug', 'august', 'sep', 'sept', 'september', 'oct', 'october', + 'nov', 'november', 'dec', 'december', 'am', 'pm', +}) +_MARKS = frozenset(",./:-+()%'") +#: The error values a cell holds — text that only starts with ``#`` is text. +_ERRORS = frozenset({'#NULL!', '#DIV/0!', '#VALUE!', '#REF!', '#NAME?', '#NUM!', '#N/A'}) + + +def _as_number(text: str) -> Number | None: + """The number a cell entry reads as, else None: ``'10'``, ``'-2.5'``, + ``'1e3'``, ``'5%'``; never ``'nan'``, ``'1_000'``, ``'1e3%'`` or ``''``.""" + entry = text.strip() + if not _NUMBER.fullmatch(entry): + return None + if entry.endswith('%'): + return float(entry[:-1]) / 100 + try: + return int(entry) + except ValueError: + return float(entry) + + +def _as_date(found: re.Match[str]) -> date | datetime | None: + """The date (with its time of day, if one follows) a ``2026-01-31`` + entry stands for; None for a day before 1900 or a time past the day.""" + year, month, day, hour, minute, second, fraction = found.groups() + try: + when = datetime(int(year), int(month), int(day), int(hour or 0), int(minute or 0), + int(second or 0), int((fraction or '0').ljust(6, '0'))) + except ValueError: + return None + if when.year < 1900: + return None + return when if hour is not None else when.date() + + +def _no_such_day(found: re.Match[str]) -> bool: + """``2026-02-30``: no day of the calendar, so text to every spreadsheet.""" + year, month, day = (int(part) for part in found.groups()[:3]) + try: + date(year, month, day) + except ValueError: + return True + return False + + +def _read_by_locale(entry: str) -> bool: + """Does a spreadsheet read this entry as a number or a date in some + locales — digits among separators, a currency sign, parentheses, a + month's name, AM / PM: ``'1,000'``, ``'$100'``, ``'(5)'``, ``'12:00'``, + ``'1/31/2026'``, ``'Jan 31, 2026'``, ``'2026-01-31T12:00'``?""" + if not any(ch in _DIGITS for ch in entry): + return False + if _ISO_T.fullmatch(entry): + return True + if any(word.casefold() not in _DATE_WORDS for word in _LETTERS.findall(entry)): + return False + return all(ch in _DIGITS or ch in _MARKS or ch.isspace() or unicodedata.category(ch) == 'Sc' + for ch in _LETTERS.sub('', entry)) + + +def _read_entry(text: str) -> Any: + """What a criteria's comparand stands for: a number when the text reads + as one, ``TRUE`` / ``FALSE`` as booleans, a ``2026-01-31`` date as + that date, else the text itself; ValueError for an entry read as a + number or a date in some spreadsheets only (a day before 1900 among + them).""" + number = _as_number(text) + if number is not None: + return number + entry = text.strip() + if entry.upper() in ('TRUE', 'FALSE'): + return entry.upper() == 'TRUE' + found = _ISO_DATE.fullmatch(entry) + if found is not None: + when = _as_date(found) + if when is not None: + return when + if _no_such_day(found): + return text + if _read_by_locale(entry): + raise ValueError( + f"the criteria entry {entry!r} reads as a number or a date in some " + f"spreadsheets and as text in others. Write a number plainly ('1000', " + f"'-5'), a date as 2026-01-31 (or join the operator to a date value: " + f"('>=', mo.DATE(2026, 1, 31))), a time of day as a fraction of a day " + f"(12:00 is 0.5); to match the text itself, put '?' for one of its " + f"marks ('10?20')." + ) + return text + + +def _parse(criteria: Any) -> tuple[str, Any, str | None]: + """``(operator, comparand, text)``: the comparand as :func:`parse_criteria` + reads it, with the criteria's own text after the operator when it was + given as text (None otherwise).""" + if criteria is None: + return '=', 0, None + if isinstance(criteria, (bool, int, float, date, timedelta)): + return '=', criteria, None + text = str(criteria) + if text == '': + return '=', '', None + op, rest = '=', text + for candidate in _OPERATORS: + if text.startswith(candidate): + op, rest = candidate, text[len(candidate):] + break + if rest == '': + return op, None, None + return op, _read_entry(rest), rest + + +def parse_criteria(criteria: Any) -> tuple[str, Any]: + """``(operator, comparand)`` as Excel reads a criteria argument. + + A number, a date or a boolean is ``('=', itself)``; a string carries + an optional leading operator (``'<>'``, ``'>='``, ``'<='``, ``'='``, + ``'>'``, ``'<'``) and the rest is read as a cell entry — a number + when it reads as one, ``TRUE`` / ``FALSE`` as booleans, a + ``2026-01-31`` date (a time of day may follow) as that date, else + text. ``''`` is the blank criterion ``('=', '')`` — a blank cell or + empty text; an operator with nothing after it has None for its + comparand (``'='`` a blank cell only, ``'<>'`` whatever is not + blank); ``None`` — a cell left blank — is ``('=', 0)``, as a reference + to an empty cell reads. An entry read as a number in some locales + only (``'1,000'``, ``'12:00'``, ``'1/31/2026'``) is a ValueError. + """ + op, comparand, _text = _parse(criteria) + return op, comparand + + +def _wildcard(pattern: str) -> re.Pattern[str]: + """``*`` any run of characters, ``?`` one, ``~`` the character after + it taken literally (a last ``~`` is itself); anything else literal; + the whole cell, case-insensitively.""" + parts: list[str] = [] + i = 0 + while i < len(pattern): + ch = pattern[i] + if ch == '~' and i + 1 < len(pattern): + parts.append(re.escape(pattern[i + 1])) + i += 2 + continue + parts.append('.*' if ch == '*' else '.' if ch == '?' else re.escape(ch)) + i += 1 + return re.compile(''.join(parts), re.IGNORECASE | re.DOTALL) + + +def _compare(a: Any, b: Any, op: str) -> bool: + if op == '=': + return bool(a == b) + if op == '<>': + return bool(a != b) + if op == '>': + return bool(a > b) + if op == '<': + return bool(a < b) + if op == '>=': + return bool(a >= b) + return bool(a <= b) + + +_E48 = 2.0 ** -48 +_E53 = 2.0 ** 53 + + +def _whole(x: float) -> bool: + return x < _E53 and x == math.floor(x) + + +def _approx_equal(a: Number, b: Number) -> bool: + """Equal as the spreadsheet compares two numbers: the same, or apart by + less than 2**-48 of the smaller — never two whole numbers that are not + the same.""" + if a == b: + return True + if isinstance(a, int) and isinstance(b, int): + return False + x, y = float(a), float(b) + if x == 0.0 or y == 0.0: + return False + d = abs(x - y) + if not math.isfinite(d): + return False + x, y = abs(x), abs(y) + if _whole(d) and _whole(x) and _whole(y): + return False + return d < x * _E48 and d < y * _E48 + + +def _numbers_compare(a: Number, b: Number, op: str) -> bool: + """``a op b`` with equality as :func:`_approx_equal` sees it.""" + same = _approx_equal(a, b) + if op == '=': + return same + if op == '<>': + return not same + if op == '>': + return a > b and not same + if op == '<': + return a < b and not same + if op == '>=': + return a > b or same + return a < b or same + + +def _cell_number(value: Any) -> Number | None: + """A cell as a number: a number is itself, a date its serial (a + duration its days); text, a boolean, a blank and an error are not.""" + if value is None or isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return value + if isinstance(value, (date, timedelta)): + serial: Number = _to_serial(value) + return serial + return None + + +_EPOCH = datetime(1899, 12, 30) + + +def _from_serial(serial: float) -> date | datetime: + """The date a serial stands for; a datetime when it carries a time + of day (the mean of dates seldom lands on a whole day).""" + when = _EPOCH + timedelta(days=serial) + return when.date() if when.time() == time() else when + + +def _is_text(value: Any) -> bool: + return isinstance(value, str) and not is_error(value) + + +def _equal(comparand: Any, text: str | None) -> Callable[[Any], bool]: + """Whether a cell equals a number, a date or a boolean: a number or a + date by its serial; text that reads as a number as that number (a + boolean never does); text equal to the criteria's own ``text``, when + it was given as text, in any case.""" + folded = text.casefold() if text is not None else None + + def same_text(v: Any) -> bool: + return folded is not None and _is_text(v) and v.casefold() == folded + + if isinstance(comparand, bool): + return lambda v: v == comparand if isinstance(v, bool) else same_text(v) + target = _to_serial(comparand) + + def same(v: Any) -> bool: + number = _cell_number(v) + if number is None and _is_text(v): + if same_text(v): + return True + number = _as_number(v) + return number is not None and _approx_equal(number, target) + + return same + + +def _matcher(op: str, comparand: Any, text: str | None = None) -> Callable[[Any], bool]: + """Whether one cell's value meets ``op comparand``; ``text`` is the + criteria's own text after the operator, when it was given as text.""" + if comparand is None or (isinstance(comparand, str) and comparand == ''): + # Nothing after the operator. '' alone matches a blank cell and + # empty text; '=' a blank cell only; '<>' whatever is not blank, + # empty text included, as COUNTA counts it; an ordering compares + # text with the empty text. + if op == '=': + if comparand == '': + return lambda v: v is None or v == '' + return lambda v: v is None + if op == '<>': + return lambda v: v is not None + return lambda v: _is_text(v) and _compare(v.casefold(), '', op) + if isinstance(comparand, str): + if is_error(comparand): + token = comparand.upper() + if op == '=': + return lambda v: isinstance(v, str) and v.upper() == token + if op == '<>': + return lambda v: not (isinstance(v, str) and v.upper() == token) + return lambda v: False + if op in ('=', '<>'): + pattern = _wildcard(comparand) + + def equal(v: Any) -> bool: + return _is_text(v) and pattern.fullmatch(v) is not None + + return equal if op == '=' else (lambda v: not equal(v)) + folded = comparand.casefold() + return lambda v: _is_text(v) and _compare(v.casefold(), folded, op) + if op in ('=', '<>'): + same = _equal(comparand, text) + return same if op == '=' else (lambda v: not same(v)) + if isinstance(comparand, bool): + return lambda v: isinstance(v, bool) and _compare(v, comparand, op) + # An ordering reads numbers and dates only (a date criteria is its + # serial, as ``DATE(...)`` gives one); text that reads as a number + # takes no part in it. + target = _to_serial(comparand) + + def ordered(v: Any) -> bool: + number = _cell_number(v) + return number is not None and _numbers_compare(number, target, op) + + return ordered + + +def matches(value: Any, criteria: Any) -> bool: + """Does one cell's ``value`` meet ``criteria``, read by + :func:`parse_criteria`? ``matches('North', 'n*')`` is True, + ``matches(None, '<>')`` False, ``matches('10', 10)`` True — text that + reads as a number equals it — and ``matches('10', '>5')`` False.""" + return _matcher(*_parse(criteria))(value) + + +# ─── The arguments ────────────────────────────────────────────────── + + +@dataclass +class _Range: + values: list[Any] + node: Expr + text: str + unit: Any + value_type: str | None + #: The range read once, when the range is one (see ``_shared``). + shared: _Lines | None = None + + +@dataclass +class _Criterion: + test: Callable[[Any], bool] + node: Expr + text: str + #: The error a criteria cell holds, which is then the result. + error: str | None = None + #: ``(operator, comparand, text)`` the test is built from (see + #: ``_matcher``), which an index of the range answers; None for none. + parsed: tuple[str, Any, str | None] | None = None + + +def _label(var: Variable) -> str: + """How a line in a list is named in ``_source_code``: the record it + belongs to and its own name (``a.amount``), one element of a row as + ``a.flow[0]``, else its name.""" + return _label_of(var)[0] + + +# The name, and whether it is read off the line's own path alone — a path +# the line keeps (``_qualified_id``), replaced whenever the line moves or +# is named (one element of a row is named by the row's). +def _label_of(var: Variable) -> tuple[str, bool]: + parts = str(var.path).split('.') + if len(parts) >= 2 and parts[0] != '__floating__' and var.python_name: + return '.'.join(parts[-2:]), var._qualified_id is not None + inner = getattr(var, '_expr', None) + if isinstance(inner, Subscript) and isinstance(inner.base, VarRef): + return f"{_label(inner.base.var)}[{inner.key!r}]", False + return src(var), var._qualified_id is not None + + +def _series_value(raw: Any) -> bool: + return isinstance(raw, list) or hasattr(raw, 'roles') + + +def _typed_in(var: Variable) -> bool: + """Does the line hold values typed in, not a formula? One element of + a row (``flow[0]``) is the row's cell, so the row answers.""" + inner = var._expr + while isinstance(inner, Subscript) and isinstance(inner.base, VarRef): + var = inner.base.var + inner = var._expr + return not var.is_formula + + +def _cells(var: Variable, raw: Any) -> list[Any]: + """The cells of a line as the workbook holds them: empty text typed + in is written as nothing, so it is a blank; a formula's empty text + stays a cell.""" + cells = list(raw) if isinstance(raw, list) else [raw] + if _typed_in(var): + return [None if isinstance(c, str) and c == '' else c for c in cells] + return cells + + +def _range(arg: Any, where: str) -> _Range: + """A range argument: a Variable (a row, or one cell) or a list of + scalar lines; its cells' values, its AST node, its name in the + source, the unit and value type its own lines share.""" + shared = _shared(arg) + if shared is not None: + return shared.cells(where) + if isinstance(arg, (list, tuple)): + if not arg: + raise ValueError(f"{where}(): a range needs at least one cell; got [].") + values: list[Any] = [] + items: list[Expr] = [] + labels: list[str] = [] + for item in arg: + if isinstance(item, Variable): + reject_axised(item, where) + raw = item._value + if _series_value(raw): + raise _series_in_range(where, item) + values.extend(_cells(item, raw)) + items.append(VarRef(item)) + labels.append(_label(item)) + else: + values.append(item) + items.append(Literal(item)) + labels.append(repr(item)) + lines = [item for item in arg if isinstance(item, Variable)] + return _Range(values, ListExpr(items), f"[{', '.join(labels)}]", + shared_unit(lines), lines[0].value_type if lines else None) + raise TypeError( + f"{where}(): a range is a list-valued line (a row) or a list of scalar " + f"lines; got {type(arg).__name__}." + ) + + +def _tracked_range(where: str) -> ValueError: + return ValueError( + f"{where}() over a line with tracks reduces per coordinate — " + f"slice a coordinate first (.at(track='...'))." + ) + + +def _series_in_range(where: str, item: Variable) -> ValueError: + return ValueError( + f"{where}(): a range is a row or a list of scalar lines; " + f"{_label(item)} in the list is a series. Pass the row " + f"itself, or one element of it ({_label(item)}[0])." + ) + + +# ─── A range read once ────────────────────────────────────────────── + +# A row, or a list of lines, is read once and shared by every call over +# the same lines — the same objects, in the same order: one AST node that +# every call's formula holds, so a renderer writes the range once, and +# what the calls read of the lines (their cells, their names in the +# source, the unit they share) and derive from the cells (the numbers +# sorted, the cells indexed by value), each read or derived once and kept +# with the node. Each call first checks that every line still holds the +# value it held when read — the same object, and of a list each cell the +# same object, the list as long — and the same unit and path, and reads +# the lines again when one does not: the engine replaces a line's value +# when it computes it again (a loop closing, a schedule laid out on +# adoption), and a list can be changed in place (the caller's own list a +# line was given, the one ``value`` hands out). A list holding a value +# typed in among its lines is read by each call, as it is short. + +_VALUE = operator.attrgetter('_value') +_UNIT = operator.attrgetter('_unit') +_PATH = operator.attrgetter('_qualified_id') + +#: The ranges read, by the identities of their lines, each a weak +#: reference to its node; what was read lives on the node and goes with it. +_ROWS: dict[int, Any] = {} +_LISTS: dict[tuple[int, ...], Any] = {} + + +def _holds_empty_text(raw: Any) -> bool: + if isinstance(raw, list): + return any(isinstance(c, str) and c == '' for c in raw) + return isinstance(raw, str) and raw == '' + + +class _Lines: + __slots__ = ('lines', 'node', 'row', 'raws', 'lists', 'sizes', 'flat', 'units', 'rooted', + 'paths', 'named', 'blanks', 'first_type', 'text', 'series', 'kept') + + def __init__(self, lines: tuple[Variable, ...], node: Expr, row: bool) -> None: + self.lines = lines + self.node = node + self.row = row + self._read() + + # A copy or a pickle of the node takes none of it: the lines are read + # again where it is used. + def __reduce__(self) -> tuple[Any, ...]: + return (type(None), ()) + + def _read(self) -> None: + lines = self.lines + self.raws = tuple(map(_VALUE, lines)) + # The lists among the values — their lengths and their cells as read. + self.lists = tuple(raw for raw in self.raws if isinstance(raw, list)) + self.sizes = tuple(map(len, self.lists)) + self.flat = tuple(chain.from_iterable(self.lists)) + self.units = tuple(map(_UNIT, lines)) + self.first_type = lines[0].value_type + rooted: list[Variable] = [] + named: list[tuple[Variable, str]] = [] + labels: list[str] = [] + for var in lines: + label, by_path = ((src(var), var._qualified_id is not None) if self.row + else _label_of(var)) + labels.append(label) + if by_path: + rooted.append(var) + else: + named.append((var, label)) + self.rooted = tuple(rooted) + self.paths = tuple(map(_PATH, rooted)) + self.named = tuple(named) + self.text = labels[0] if self.row else f"[{', '.join(labels)}]" + self.blanks = tuple((var, _typed_in(var)) for var, raw in zip(lines, self.raws) + if _holds_empty_text(raw)) + self.series = any(map(_series_value, self.raws)) + self.kept: dict[Any, Any] = {} + + def _unchanged(self) -> bool: + lines = self.lines + if not (all(map(operator.is_, map(_VALUE, lines), self.raws)) + and tuple(map(len, self.lists)) == self.sizes + and all(map(operator.is_, chain.from_iterable(self.lists), self.flat)) + and all(map(operator.is_, map(_UNIT, lines), self.units)) + and all(map(operator.is_, map(_PATH, self.rooted), self.paths)) + and lines[0].value_type == self.first_type): + return False + name = src if self.row else _label + return (all(name(var) == label for var, label in self.named) + and all(_typed_in(var) is typed for var, typed in self.blanks)) + + def check(self) -> None: + if not self._unchanged(): + self._read() + + # What ``make`` derives from the lines as they are now, derived once. + def keep(self, key: Any, make: Callable[[], Any]) -> Any: + try: + return self.kept[key] + except KeyError: + value = self.kept[key] = make() + return value + + # The range as ``_range`` reads it, refused as it refuses. + def cells(self, where: str) -> _Range: + rng = self.kept.get('cells') + if rng is None: + rng = self.kept['cells'] = self._cells(where) + return rng + + def _cells(self, where: str) -> _Range: + if self.row: + var = self.lines[0] + reject_axised(var, where) + raw = self.raws[0] + if hasattr(raw, 'roles'): + raise _tracked_range(where) + return _Range(_cells(var, raw), self.node, self.text, self.units[0], + self.first_type, self) + values: list[Any] = [] + for var, raw in zip(self.lines, self.raws): + reject_axised(var, where) + if _series_value(raw): + raise _series_in_range(where, var) + values.extend(_cells(var, raw)) + return _Range(values, self.node, self.text, shared_unit(self.lines), self.first_type, + self) + + # The first line of each unit among the lines, with its node: what a + # check of their units reads — the first unit, the first other unit and + # each label of a pure number are first among them. + def unit_lines(self) -> tuple[list[Variable], list[Expr]]: + from ..core.unit import unit_key + + def make() -> tuple[list[Variable], list[Expr]]: + nodes: list[Expr] = (list(self.node.items) if isinstance(self.node, ListExpr) + else [self.node]) + seen: set[Any] = set() + lines: list[Variable] = [] + picked: list[Expr] = [] + for var, unit, node in zip(self.lines, self.units, nodes): + if unit is None: + continue + key = unit_key(unit) + if key not in seen: + seen.add(key) + lines.append(var) + picked.append(node) + return lines, picked + result: tuple[list[Variable], list[Expr]] = self.keep('units', make) + return result + + +# The range a row or a list of lines is, read once (see above); None for +# anything else, read by each call. +def _shared(arg: Any) -> _Lines | None: + if isinstance(arg, Variable): + return _find(_ROWS, id(arg), (arg,), True) + if isinstance(arg, (list, tuple)) and arg: + return _find(_LISTS, tuple(map(id, arg)), arg, False) + return None + + +def _find(table: dict[Any, Any], key: Any, arg: Sequence[Any], row: bool) -> _Lines | None: + ref = table.get(key) + node = ref() if ref is not None else None + if node is not None: + shared: _Lines = node.__dict__['_lines'] + shared.check() + return shared + if not row and not all(isinstance(item, Variable) for item in arg): + return None + lines = tuple(arg) + node = VarRef(lines[0]) if row else ListExpr([VarRef(var) for var in lines]) + shared = _Lines(lines, node, row) + node.__dict__['_lines'] = shared + table[key] = weakref.ref(node, partial(_forget, table, key)) + return shared + + +def _forget(table: dict[Any, Any], key: Any, ref: Any) -> None: + if table.get(key) is ref: + del table[key] + + +#: The ranges a call has just checked, by node, while it builds its result: +#: the unit law run then reads their lines by their own units (the only +#: reading a result's law makes), from what they were read as. +_CHECKED: dict[int, _Lines] = {} + + +# The ranges among a call's argument nodes — each checked by the call that +# holds it — taken as checked while the call builds its result. +@contextmanager +def _checked(nodes: Sequence[Any]) -> Iterator[None]: + added: list[int] = [] + for node in nodes: + shared = getattr(node, '__dict__', {}).get('_lines') + if shared is not None and id(node) not in _CHECKED: + _CHECKED[id(node)] = shared + added.append(id(node)) + try: + yield + finally: + for key in added: + _CHECKED.pop(key, None) + + +# A range's lines as a unit law reads them: a row is one, a list of lines +# each of its items — or, for a range the call has just checked, read by +# the lines' own units, the first line of each unit among them, over which +# every unit check answers the same. +def _unit_items(node: Any, unit_of: Callable[[Any], Any]) -> list[Any]: + shared = _CHECKED.get(id(node)) + if shared is not None and shared.node is node and unit_of is operand_unit: + return list(shared.unit_lines()[1]) + return list(node.items) if isinstance(node, ListExpr) else [node] + + +# ─── The cells indexed by value ───────────────────────────────────── + +# A range's cells by value, for an equality — a criteria's and an exact +# lookup's: the cells that can meet the value asked for, found without +# reading every cell, each then read by the test itself — so the index +# narrows the reading and never decides. A number or a date is filed by +# its serial (text that reads as a number by that number too), in order, +# so the cells within a hair of a value — the spreadsheet's equality of +# two numbers — are one slice; text by its case folded, and by its letters +# folded as the spreadsheet's text pattern compares them, one by one (text +# with a letter that folds otherwise is read for every value); a boolean +# by itself. + +#: Twice the spreadsheet's 2**-48: a slice this wide holds every number +#: equal to the value. +_FUZZ = 2.0 ** -47 + +#: How a character folds, as the spreadsheet's text pattern compares it +#: (see ``_fold``). +_FOLDED: dict[str, str | None] = {} + + +# A text as its letters fold for a pattern without wildcards — two texts a +# pattern of one matches fold alike — or None when a letter does not fold +# to one: its lower case, upper case, lower case again (the letters that +# share an upper case, ``i`` and ``ı``, fold to one). An ASCII letter's is +# its lower case. +def _fold(text: str) -> str | None: + if text.isascii(): + return text.lower() + out: list[str] = [] + for ch in text: + if ch in _FOLDED: + folded = _FOLDED[ch] + else: + low = ch.lower() + up = low.upper() if len(low) == 1 else '' + key = up.lower() if len(up) == 1 else '' + folded = _FOLDED[ch] = key if len(key) == 1 else None + if folded is None: + return None + out.append(folded) + return ''.join(out) + + +class _Index: + __slots__ = ('serials', 'at', 'odd', 'bools', 'folded', 'keyed', 'loose', 'exact') + + def __init__(self, values: list[Any]) -> None: + numbers: list[tuple[Any, int]] = [] + self.odd: list[int] = [] + self.bools: dict[bool, list[int]] = {True: [], False: []} + self.folded: dict[str, list[int]] = {} + self.keyed: dict[str, list[int]] = {} + self.loose: list[int] = [] + #: The numbers and dates by serial, as a lookup compares them. + self.exact: dict[Any, list[int]] = {} + for i, v in enumerate(values): + if isinstance(v, bool): + self.bools[v].append(i) + continue + number = _cell_number(v) + if number is not None: + self.exact.setdefault(number, []).append(i) + elif _is_text(v): + self.folded.setdefault(v.casefold(), []).append(i) + key = _fold(v) + if key is None: + self.loose.append(i) + else: + self.keyed.setdefault(key, []).append(i) + number = _as_number(v) + if number is None: + continue + if isinstance(number, float) and not math.isfinite(number): + self.odd.append(i) + else: + numbers.append((number, i)) + numbers.sort(key=operator.itemgetter(0)) + self.serials = [n for n, _i in numbers] + self.at = [i for _n, i in numbers] + + # The cells a number or a date can equal: those within a hair of it. + def near(self, value: Any) -> list[int] | None: + target = _to_serial(value) + try: + fuzz = abs(target) * _FUZZ + lo, hi = target - fuzz, target + fuzz + except OverflowError: + return None + if not (math.isfinite(lo) and math.isfinite(hi)): + return None + return (self.at[bisect.bisect_left(self.serials, lo): + bisect.bisect_right(self.serials, hi)] + self.odd) + + # The text cells a pattern of ``text`` without wildcards can match. + def like(self, text: str) -> list[int] | None: + if any(ch in '*?~' for ch in text): + return None + key = _fold(text) + return None if key is None else self.keyed.get(key, []) + self.loose + + # Whether ``meeting`` finds the cells ``op comparand`` can meet: an + # equality, to anything but a blank, an error or a pattern. + @staticmethod + def answers(op: str, comparand: Any) -> bool: + if op != '=' or comparand is None: + return False + return not isinstance(comparand, str) or not ( + comparand == '' or is_error(comparand) or any(ch in '*?~' for ch in comparand)) + + # The cells ``op comparand`` can meet, given as ``text`` (see + # ``_matcher``); None when every cell is read. + def meeting(self, op: str, comparand: Any, text: str | None) -> list[int] | None: + if not self.answers(op, comparand): + return None + same = self.folded.get(text.casefold(), []) if text is not None else [] + if isinstance(comparand, bool): + return self.bools[comparand] + same + if isinstance(comparand, str): + return self.like(comparand) + near = self.near(comparand) + return None if near is None else near + same + + +# The index of a range read once, for an equality it answers — built for +# the second such equality over the range: the first reads every cell, as +# one alone over a range does. +def _index(rng: _Range) -> _Index | None: + shared = rng.shared + if shared is None: + return None + kept = shared.kept + index: _Index | None = kept.get('index') + if index is None: + if not kept.get('asked'): + kept['asked'] = True + return None + index = kept['index'] = _Index(rng.values) + return index + + +def _scalar_criteria(var: Variable, where: str) -> Any: + reject_axised(var, where) + raw = var._value + if _series_value(raw): + raise ValueError( + f"{where}(): a criteria is one value — a number, text, or a scalar " + f"line; {src(var)} is a series. A conditional sum per period is a " + f"call over that period's cells, with this period's value as the " + f"criteria." + ) + return _cells(var, raw)[0] + + +def _error_value(raw: Any) -> str | None: + """The error a cell holds, else None (text that only starts with ``#`` + is text).""" + return raw if isinstance(raw, str) and raw in _ERRORS else None + + +def _timed(value: Any) -> bool: + """A datetime with a time of day, which ``DATE(...)`` cannot write.""" + return isinstance(value, datetime) and value.time() != time() + + +def _never(value: Any) -> bool: + return False + + +_Parsed = tuple[str, Any, str | None] + + +def _test(criteria: Any, where: str, + line: Variable | None = None) -> tuple[Callable[[Any], bool], _Parsed]: + """The matcher of one criteria, with what it was read as; an entry + read as a number in some locales only is refused, naming the call (and + the line holding it).""" + try: + parsed = _parse(criteria) + return _matcher(*parsed), parsed + except ValueError as exc: + held = f" It is the value of {src(line)}." if line is not None else "" + raise ValueError(f"{where}(): {exc}{held}") from None + + +def _joined(op: str, raw: Any, where: str, + line: Variable) -> tuple[Callable[[Any], bool], _Parsed]: + """``op & cell``, joined as the spreadsheet joins them: a blank cell or + empty text leaves the operator alone, a boolean is ``TRUE`` / + ``FALSE``, text is read after the operator, and a number or a date + is its value.""" + if raw is None or (isinstance(raw, str) and raw == ''): + return _matcher(op, None), (op, None, None) + if isinstance(raw, bool): + raw = 'TRUE' if raw else 'FALSE' + if isinstance(raw, str): + return _test(op + raw, where, line) + return _matcher(op, raw), (op, raw, None) + + +def _entry_text(value: Any) -> str: + """A typed value joined to an operator, as the joined text spells it.""" + if value is None: + return '' + if isinstance(value, bool): + return 'TRUE' if value else 'FALSE' + if isinstance(value, datetime): + return repr(_to_serial(value)) + return str(value) + + +def _criterion(criteria: Any, where: str) -> _Criterion: + """A criteria argument: a number, text, date or boolean (a literal); a + scalar Variable (its cell); ``(operator, value)`` (the operator joined + to the value, ``">=" & B2``). A cell holding an error makes that + error the result.""" + if isinstance(criteria, Variable): + raw = _scalar_criteria(criteria, where) + node: Expr = VarRef(criteria) + text = src(criteria) + error = _error_value(raw) + if error is not None: + return _Criterion(_never, node, text, error) + test, parsed = _test(raw, where, criteria) + return _Criterion(test, node, text, parsed=parsed) + if isinstance(criteria, tuple): + if len(criteria) != 2 or criteria[0] not in _OPERATORS: + raise ValueError( + f"{where}(): a criteria pair is (operator, value) with the operator " + f"one of {', '.join(repr(op) for op in _OPERATORS)}; got {criteria!r}." + ) + op, value = criteria + if isinstance(value, Variable): + raw = _scalar_criteria(value, where) + node = FuncCall('CONCAT', [Literal(op), VarRef(value)]) + text = f"CONCAT({op!r}, {src(value)})" + error = _error_value(raw) + if error is not None: + return _Criterion(_never, node, text, error) + test, parsed = _joined(op, raw, where, value) + return _Criterion(test, node, text, parsed=parsed) + if isinstance(value, (date, timedelta)) and not _timed(value): + node = FuncCall('CONCAT', [Literal(op), Literal(value)]) + return _Criterion(_matcher(op, value), node, f"CONCAT({op!r}, {value!r})", + parsed=(op, value, None)) + joined = f"{op}{_entry_text(value)}" + test, parsed = _test(joined, where) + return _Criterion(test, Literal(joined), repr(joined), parsed=parsed) + if isinstance(criteria, (list, dict, set)): + raise ValueError( + f"{where}(): a criteria is one value — a number, text, a scalar line " + f"or (operator, value); got {type(criteria).__name__}." + ) + # A time of day is written as the serial: ``DATE(...)`` would drop it. + written = _to_serial(criteria) if _timed(criteria) else criteria + test, parsed = _test(criteria, where) + return _Criterion(test, Literal(written), repr(criteria), parsed=parsed) + + +def _same_length(where: str, first: _Range, other: _Range) -> None: + if len(first.values) != len(other.values): + raise ValueError( + f"{where}(): every range has the same number of cells — {first.text} " + f"has {len(first.values)}, {other.text} has {len(other.values)}." + ) + + +def _pairs(args: tuple[Any, ...], where: str, example: str) -> list[tuple[Any, Any]]: + if len(args) < 2 or len(args) % 2: + raise ValueError( + f"{where}() takes criteria ranges and criteria in pairs: {example}. " + f"Got {len(args)} argument{'s' if len(args) != 1 else ''} after the first." + ) + return [(args[i], args[i + 1]) for i in range(0, len(args), 2)] + + +def _mask(pairs: list[tuple[Any, Any]], where: str, like: _Range | None, + nodes: list[Expr], texts: list[str] + ) -> tuple[list[tuple[_Range, _Criterion]], str | None]: + """Every ``(range, criteria)`` pair read, and the error of the first + criteria cell holding one; the ranges must be as long as ``like`` (the + aggregated range) and as each other.""" + read: list[tuple[_Range, _Criterion]] = [] + error: str | None = None + for rng_arg, crit_arg in pairs: + rng = _range(rng_arg, where) + if like is not None: + _same_length(where, like, rng) + crit = _criterion(crit_arg, where) + error = error or crit.error + read.append((rng, crit)) + nodes.extend([rng.node, crit.node]) + texts.extend([rng.text, crit.text]) + like = like or rng + return read, error + + +# The positions every pair admits, in order: the cells an index finds for +# a pair are read by its test, and every other pair's test reads those +# alone. +def _admitted(read: list[tuple[_Range, _Criterion]]) -> list[int]: + found: list[int] | None = None + rest: list[tuple[_Range, _Criterion]] = [] + for rng, crit in read: + parsed = crit.parsed + index = (_index(rng) if parsed is not None and _Index.answers(parsed[0], parsed[1]) + else None) + near = index.meeting(*parsed) if index is not None and parsed is not None else None + if near is None: + rest.append((rng, crit)) + continue + values, test = rng.values, crit.test + own = [i for i in sorted(set(near)) if test(values[i])] + if found is None: + found = own + else: + kept = set(own) + found = [i for i in found if i in kept] + if found is None: + rng, crit = rest.pop(0) + values, test = rng.values, crit.test + found = [i for i, v in enumerate(values) if test(v)] + for rng, crit in rest: + values, test = rng.values, crit.test + found = [i for i in found if test(values[i])] + return found + + +def _picked(values: list[Any], at: list[int]) -> list[Any]: + return [values[i] for i in at] + + +def _numbers(values: list[Any]) -> list[Any]: + """The cells a sum, an average or a ranking reads: numbers, dates + and errors; a blank, text or boolean is skipped.""" + return [v for v in values + if is_error(v) or isinstance(v, (date, timedelta)) + or (isinstance(v, (int, float)) and not isinstance(v, bool))] + + +def _serials(values: list[Any]) -> list[Any]: + return [_to_serial(v) for v in values] + + +# A date reduces as its serial; the largest and the smallest come back +# as the cell holds them (a date stays a date). +_sum = _excel_reduce(lambda values: builtins.sum(_serials(values))) +_average = _excel_reduce(lambda values: _mean(_serials(values)), '#DIV/0!') +_largest = _excel_reduce(lambda values: builtins.max(values, key=_to_serial)) +_smallest = _excel_reduce(lambda values: builtins.min(values, key=_to_serial)) + + +def _mean_of(cells: list[Any]) -> Any: + """The mean of the picked cells; of dates, the mean date.""" + value = _average(cells) + if cells and isinstance(value, (int, float)) and all(isinstance(c, date) for c in cells): + return _from_serial(value) + return value + + +def _mean_type(value: Any) -> str: + return 'datetime' if isinstance(value, date) else 'float' + + +def _summed_type(rng: _Range) -> str: + """A sum is a number: of its range's type when that is a number's, a + float otherwise (a sum of dates is a number of days).""" + return rng.value_type if rng.value_type in ('int', 'float') else 'float' + + +def _ranked_type(rng: _Range) -> str: + """The largest or the smallest is one of the range's numbers or + dates: of its range's type when that is a number's or a date's, a + float otherwise.""" + return rng.value_type if rng.value_type in ('int', 'float', 'datetime') else 'float' + + +def _result(func: str, nodes: list[Expr], texts: list[str], value: Any, + value_type: str) -> Variable: + """One number with its formula; its unit is the function's law (see + Units below).""" + with _checked(nodes): + return make_func_var( + func, nodes, value, value_type, 'scalar', + source_code=f"{func}({', '.join(texts)})", + ) + + +# ─── The functions ────────────────────────────────────────────────── + + +def SUMIF(range: RangeArg, criteria: Any, sum_range: RangeArg | None = None) -> Variable: + """Sum of the cells of ``sum_range`` (else ``range``) where ``range`` + meets ``criteria`` (renders ``=SUMIF(...)``). + + Example:: + + north = mo.SUMIF(sales.region, "North", sales.amount) # =SUMIF(B2:F2, "North", B3:F3) + big = mo.SUMIF(sales.amount, ">=1000") # =SUMIF(B3:F3, ">=1000") + over = mo.SUMIF(sales.amount, (">=", inputs.floor)) # =SUMIF(B3:F3, ">=" & Inputs!B1) + per_deal = mo.SUMIF([d.a.region, d.b.region], "North", # =SUMIF(Deals!B3:B4, "North", + [d.a.amount, d.b.amount]) # Deals!C3:C4) + """ + where = "SUMIF" + rng = _range(range, where) + crit = _criterion(criteria, where) + agg = rng if sum_range is None else _range(sum_range, where) + _same_length(where, rng, agg) + value = crit.error or _sum(_numbers(_picked(agg.values, _admitted([(rng, crit)])))) + nodes = [rng.node, crit.node] + ([] if sum_range is None else [agg.node]) + texts = [rng.text, crit.text] + ([] if sum_range is None else [agg.text]) + return _result(where, nodes, texts, value, _summed_type(agg)) + + +def SUMIFS(sum_range: RangeArg, *criteria: Any) -> Variable: + """Sum of the cells of ``sum_range`` where every ``(range, criteria)`` + pair is met (renders ``=SUMIFS(...)``). + + Example:: + + mo.SUMIFS(sales.amount, sales.region, "North", sales.year, 2026) + # =SUMIFS(B3:F3, B2:F2, "North", B4:F4, 2026) + """ + where = "SUMIFS" + agg = _range(sum_range, where) + nodes: list[Expr] = [agg.node] + texts = [agg.text] + pairs = _pairs(criteria, where, "mo.SUMIFS(sum_range, range1, criteria1, range2, criteria2)") + read, error = _mask(pairs, where, agg, nodes, texts) + value = error or _sum(_numbers(_picked(agg.values, _admitted(read)))) + return _result(where, nodes, texts, value, _summed_type(agg)) + + +def COUNTIF(range: RangeArg, criteria: Any) -> Variable: + """Number of cells of ``range`` that meet ``criteria`` (renders + ``=COUNTIF(...)``): ``mo.COUNTIF(sales.region, "N*")``.""" + where = "COUNTIF" + rng = _range(range, where) + crit = _criterion(criteria, where) + value = crit.error or len(_admitted([(rng, crit)])) + return _result(where, [rng.node, crit.node], [rng.text, crit.text], value, 'int') + + +def COUNTIFS(*criteria: Any) -> Variable: + """Number of positions where every ``(range, criteria)`` pair is met + (renders ``=COUNTIFS(...)``): ``mo.COUNTIFS(region, "North", amount, ">0")``.""" + where = "COUNTIFS" + nodes: list[Expr] = [] + texts: list[str] = [] + pairs = _pairs(criteria, where, "mo.COUNTIFS(range1, criteria1, range2, criteria2)") + read, error = _mask(pairs, where, None, nodes, texts) + value = error or len(_admitted(read)) + return _result(where, nodes, texts, value, 'int') + + +def AVERAGEIF(range: RangeArg, criteria: Any, + average_range: RangeArg | None = None) -> Variable: + """Mean of the cells of ``average_range`` (else ``range``) where + ``range`` meets ``criteria`` (renders ``=AVERAGEIF(...)``); of dates, + the mean date; ``#DIV/0!`` when no number is picked.""" + where = "AVERAGEIF" + rng = _range(range, where) + crit = _criterion(criteria, where) + agg = rng if average_range is None else _range(average_range, where) + _same_length(where, rng, agg) + value = crit.error or _mean_of(_numbers(_picked(agg.values, _admitted([(rng, crit)])))) + nodes = [rng.node, crit.node] + ([] if average_range is None else [agg.node]) + texts = [rng.text, crit.text] + ([] if average_range is None else [agg.text]) + return _result(where, nodes, texts, value, _mean_type(value)) + + +def AVERAGEIFS(average_range: RangeArg, *criteria: Any) -> Variable: + """Mean of the cells of ``average_range`` where every ``(range, + criteria)`` pair is met (renders ``=AVERAGEIFS(...)``); ``#DIV/0!`` + when no number is picked.""" + where = "AVERAGEIFS" + agg = _range(average_range, where) + nodes: list[Expr] = [agg.node] + texts = [agg.text] + pairs = _pairs(criteria, where, + "mo.AVERAGEIFS(average_range, range1, criteria1, range2, criteria2)") + read, error = _mask(pairs, where, agg, nodes, texts) + value = error or _mean_of(_numbers(_picked(agg.values, _admitted(read)))) + return _result(where, nodes, texts, value, _mean_type(value)) + + +def MAXIFS(max_range: RangeArg, *criteria: Any) -> Variable: + """Largest cell of ``max_range`` where every ``(range, criteria)`` pair + is met (renders ``=MAXIFS(...)``); 0 when none is.""" + where = "MAXIFS" + agg = _range(max_range, where) + nodes: list[Expr] = [agg.node] + texts = [agg.text] + pairs = _pairs(criteria, where, "mo.MAXIFS(max_range, range1, criteria1, range2, criteria2)") + read, error = _mask(pairs, where, agg, nodes, texts) + value = error or _largest(_numbers(_picked(agg.values, _admitted(read)))) + return _result(where, nodes, texts, value, _ranked_type(agg)) + + +def MINIFS(min_range: RangeArg, *criteria: Any) -> Variable: + """Smallest cell of ``min_range`` where every ``(range, criteria)`` pair + is met (renders ``=MINIFS(...)``); 0 when none is.""" + where = "MINIFS" + agg = _range(min_range, where) + nodes: list[Expr] = [agg.node] + texts = [agg.text] + pairs = _pairs(criteria, where, "mo.MINIFS(min_range, range1, criteria1, range2, criteria2)") + read, error = _mask(pairs, where, agg, nodes, texts) + value = error or _smallest(_numbers(_picked(agg.values, _admitted(read)))) + return _result(where, nodes, texts, value, _ranked_type(agg)) + + +# ─── The counts ───────────────────────────────────────────────────── + + +def _count_operands(args: tuple[Any, ...], where: str) -> tuple[list[tuple[Any, bool]], + list[Expr], list[str]]: + """``(value, typed)`` per cell — ``typed`` for a value given in the + call itself, not read from a cell or a list (an array) — with the + AST nodes and names.""" + if not args: + raise ValueError( + f"{where}() needs at least one argument. " + f"Syntax: mo.{where}(a, b, c) or mo.{where}(row). Got 0 arguments." + ) + cells: list[tuple[Any, bool]] = [] + nodes: list[Expr] = [] + texts: list[str] = [] + for a in args: + if isinstance(a, (Variable, list, tuple)): + rng = _range(a, where) + cells.extend((v, False) for v in rng.values) + nodes.append(rng.node) + texts.append(rng.text) + else: + cells.append((a, True)) + nodes.append(Literal(a)) + texts.append(repr(a)) + return cells, nodes, texts + + +def _counts_as_number(value: Any, typed: bool) -> bool: + """``COUNT``: a number, a date or a duration in a cell; typed into the + call, a boolean, text that reads as a number and ``None`` (written as + the 0 it stands for) count too; never an error.""" + if isinstance(value, bool) or value is None: + return typed + if isinstance(value, (int, float, date, timedelta)): + return True + return typed and isinstance(value, str) and _as_number(value) is not None + + +# Is a count one call per period: as many values as the registry's +# per_period_arity or more — each line of a list of lines one value — a +# series among them (a line holding a list or a row per track, a list +# typed in)? +def _counts_by_period(args: tuple[Any, ...], where: str) -> bool: + count, shaped = 0, False + for a in args: + if isinstance(a, (list, tuple)) and any(isinstance(i, Variable) for i in a): + count += len(a) + shaped = shaped or any(_series_value(i._value) for i in a if isinstance(i, Variable)) + else: + count += 1 + shaped = shaped or (_series_value(a._value) if isinstance(a, Variable) + else isinstance(a, list) and bool(a)) + return shaped and count >= (per_period_arity(where) or 2) + + +# A line's cells as the workbook holds them (``_cells``), one per period, +# or one per period of each track. +def _line_cells(var: Variable) -> Any: + from ..core.tracks import TrackValues + + def fiber(raw: Any) -> Any: + cells = _cells(var, raw) + return cells if isinstance(raw, list) else cells[0] + + raw = var._value + return TrackValues.lift(fiber, raw) if isinstance(raw, TrackValues) else fiber(raw) + + +# A count over two or more values with a series among them: one call per +# period over this period's cells, as ``a + b`` combines its operands — a +# line's cell of the period, a value typed in (a list typed in is one per +# period), the cells of a list of scalar lines in every period; scalars +# and one-value series broadcast, other lengths are refused, located +# lines on other windows are aligned by date (each continued by its own +# extend= rule, the cells the sheet holds for it), a line with tracks is +# counted track by track, lines along the same keys key by key. +def _count_each_period(where: str, args: tuple[Any, ...], + counts: Callable[[Any, bool], bool]) -> Variable: + from ..core.tracks import TrackValues + raws: list[Any] = [] + typed: list[bool] = [] + nodes: list[Expr] = [] + texts: list[str] = [] + for a in args: + if isinstance(a, Variable): + reject_axised(a, where) + raws.append(_line_cells(a)) + typed.append(False) + nodes.append(VarRef(a)) + texts.append(src(a)) + elif isinstance(a, tuple) or (isinstance(a, list) + and any(isinstance(i, Variable) for i in a)): + rng = _range(a, where) + raws.append(tuple(rng.values)) + typed.append(False) + nodes.append(rng.node) + texts.append(rng.text) + elif isinstance(a, list): + if not a: + raise ValueError(f"{where}(): a range needs at least one cell; got [].") + raws.append(list(a)) + typed.append(True) + nodes.append(Literal(list(a))) + texts.append(repr(a)) + else: + raws.append(a) + typed.append(True) + nodes.append(Literal(a)) + texts.append(repr(a)) + + def one(values: list[Any]) -> int: + return builtins.sum( + 1 for value, t in zip(values, typed) + for cell in (value if isinstance(value, tuple) else (value,)) if counts(cell, t) + ) + + raws, keys = align_keyed(where, args, raws) + raws, window = align_located(args, raws) + value: Any + if any(isinstance(raw, TrackValues) for raw in raws): + value = TrackValues.lift(lambda *fibers: _per_period(list(fibers), one), *raws) + var_type = 'list' if value.time_length else 'scalar' + else: + value = _per_period(raws, one) + var_type = 'list' if isinstance(value, list) else 'scalar' + with _checked(nodes): + out = make_func_var(where, nodes, value, 'int', var_type, + source_code=f"{where}({', '.join(texts)})") + return placed(out, window, keys) + + +def _counta_counts(value: Any, typed: bool) -> bool: + return typed or value is not None + + +def COUNT(*args: Any) -> Variable: + """Number of cells holding a number (renders ``=COUNT(...)``): a date + is one, a blank, text, a boolean or an error is not; a boolean, + numeric text or ``None`` (the 0 the formula holds) typed into the + call itself counts, as the spreadsheet counts it. Over ONE operand, + the count of the row's cells: ``mo.COUNT(row)`` → ``=COUNT(B2:F2)``. + Over two or more operands with a series among them, the count of + each period's cells: ``mo.COUNT(plan, actual)`` → ``=COUNT(B2, B3)`` in + every column. Over scalars only, one number.""" + _refuse_listed_series("COUNT", args) + if _counts_by_period(args, "COUNT"): + return _count_each_period("COUNT", args, _counts_as_number) + cells, nodes, texts = _count_operands(args, "COUNT") + value = builtins.sum(1 for v, typed in cells if _counts_as_number(v, typed)) + return _result("COUNT", nodes, texts, value, 'int') + + +def COUNTA(*args: Any) -> Variable: + """Number of cells that are not blank (renders ``=COUNTA(...)``): text, + numbers, booleans, errors and a formula's empty text all count, and + a value typed into the call always does; empty text typed into a + line is a blank cell. Over ONE operand, the row's cells; over two or + more operands with a series among them, each period's cells + (``=COUNTA(B2, B3)`` in every column); over scalars only, one + number.""" + _refuse_listed_series("COUNTA", args) + if _counts_by_period(args, "COUNTA"): + return _count_each_period("COUNTA", args, _counta_counts) + cells, nodes, texts = _count_operands(args, "COUNTA") + value = builtins.sum(1 for v, typed in cells if _counta_counts(v, typed)) + return _result("COUNTA", nodes, texts, value, 'int') + + +# ─── Units ────────────────────────────────────────────────────────── + + +def _lines_of(node: Any) -> list[Any]: + """A range's lines: a row is one, a list of lines each of its items — + in the call that has just checked them, the first line of each unit + among them (see ``_unit_items``).""" + return _unit_items(node, operand_unit) + + +def _criteria_unit(node: Any, unit_of: Callable[[Any], Any]) -> tuple[Any, Any]: + """The unit a criteria compares in, and the node carrying it: a line's + own; for an operator joined to a cell — ``('>=', cap)``, or a line + that is ``mo.CONCAT('>=', cap)`` — the cell's. A typed one has none.""" + if isinstance(node, VarRef): + unit = unit_of(node) + if unit is not None: + return unit, node + inner = getattr(node.var, '_expr', None) + if not (isinstance(inner, FuncCall) and inner.func == 'CONCAT'): + return None, None + node = inner + if isinstance(node, FuncCall) and node.func == 'CONCAT': + for arg in node.args: + unit = unit_of(arg) + if unit is not None: + return unit, arg + return None, None + + +def joined_criteria(node: Any) -> tuple[str, Any] | None: + """``(operator, value node)`` when ``node`` is an operator joined to a + value - the ``CONCAT('>=', cap)`` a ``('>=', cap)`` criteria is written + as - else None. A loop container runs the call again from its formula + and hands the pair back, so the criteria is read as the author gave it.""" + if (isinstance(node, FuncCall) and node.func == 'CONCAT' and len(node.args) == 2 + and isinstance(node.args[0], Literal) and node.args[0].value in _OPERATORS): + return node.args[0].value, node.args[1] + return None + + +def _named(node: Any) -> str: + from ..core.variable_ops import _line_name + return _line_name(node.var) if isinstance(node, VarRef) else "a formula" + + +def _conditional_unit(func: str, total: Any, pairs: list[tuple[Any, Any]], + unit_of: Callable[[Any], Any]) -> Any: + """Each criteria compared in its range's unit, then the unit the + totalled lines share (none for a count).""" + from ..core.variable_ops import _incompatible_units + for rng, criteria in pairs: + unit, held = _criteria_unit(criteria, unit_of) + if unit is None or unit is UNSETTLED: + continue + for line in _lines_of(rng): + own = unit_of(line) + if own is not None and own is not UNSETTLED and own != unit: + detail = _incompatible_units(own, unit).split(": ", 1)[1] + raise ValueError( + f"mo.{func} compares {_named(line)} with the criteria " + f"{_named(held)}; they are in {detail}" + ) + return None if total is None else same_unit(func, _lines_of(total), unit_of) + + +def _one_criteria_law(func: str) -> UnitLaw: + """``SUMIF`` / ``AVERAGEIF`` (range, criteria, the range totalled — + else the range itself); ``COUNTIF`` (range, criteria), a count.""" + def law(args: Any, unit_of: Callable[[Any], Any]) -> Any: + if len(args) < 2: + return None + total = None if func == "COUNTIF" else args[2] if len(args) > 2 else args[0] + return _conditional_unit(func, total, [(args[0], args[1])], unit_of) + return law + + +def _criteria_pairs_law(func: str) -> UnitLaw: + """``SUMIFS`` and its kin (the range totalled, then range and criteria + pairs); ``COUNTIFS`` (the pairs alone), a count.""" + def law(args: Any, unit_of: Callable[[Any], Any]) -> Any: + if not args: + return None + total, pairs = (None, list(args)) if func == "COUNTIFS" else (args[0], list(args[1:])) + return _conditional_unit(func, total, list(zip(pairs[0::2], pairs[1::2])), unit_of) + return law + + +register("SUMIF", shape="range") +register("SUMIFS", shape="range") +register("COUNTIF", shape="range") +register("COUNTIFS", shape="range") +register("AVERAGEIF", shape="range") +register("AVERAGEIFS", shape="range") +register("MAXIFS", shape="range", file_prefix="_xlfn.") +register("MINIFS", shape="range", file_prefix="_xlfn.") +register("COUNTA", shape="reduction", per_period_arity=2) +unit_law("SUMIF", _one_criteria_law("SUMIF")) +unit_law("AVERAGEIF", _one_criteria_law("AVERAGEIF")) +unit_law("COUNTIF", _one_criteria_law("COUNTIF")) +unit_law("SUMIFS", _criteria_pairs_law("SUMIFS")) +unit_law("AVERAGEIFS", _criteria_pairs_law("AVERAGEIFS")) +unit_law("MAXIFS", _criteria_pairs_law("MAXIFS")) +unit_law("MINIFS", _criteria_pairs_law("MINIFS")) +unit_law("COUNTIFS", _criteria_pairs_law("COUNTIFS")) +# ``COUNT`` is registered beside ``SUM`` (functions/aggregate.py). diff --git a/src/modeleon/functions/dates.py b/src/modeleon/functions/dates.py index 2ad3cea..fd994d5 100644 --- a/src/modeleon/functions/dates.py +++ b/src/modeleon/functions/dates.py @@ -1,15 +1,23 @@ # SPDX-License-Identifier: Apache-2.0 """Date helpers — ``YEAR``, ``MONTH``, ``DAY``, ``EDATE``, ``EOMONTH``, -``DATE``, ``DAYS360``, ``YEARFRAC``, ``DAYS``, ``TODAY``. +``DATE``, ``DAYS360``, ``YEARFRAC``, ``DAYS``, ``TODAY``, ``DATEDIF``, +``WEEKDAY``, ``WEEKNUM``, ``NETWORKDAYS``, ``WORKDAY``. **Invariant:** every call returns a :class:`Variable` with a tracked formula — even when inputs are plain ``date`` objects or ISO strings. The Excel cell is the product (``=EOMONTH(...)``), not just the value. -The day counts (``DAYS360``, ``YEARFRAC``, ``DAYS``) compute what Excel -computes, quirks included, so the workbook recalculates to the engine's -numbers. Dates from 1 March 1900 on: Excel's serial calendar counts a +The day counts (``DAYS360``, ``YEARFRAC``, ``DAYS``, ``DATEDIF``, +``NETWORKDAYS``) compute what Excel computes, quirks included, so the +workbook recalculates to the engine's numbers. Dates from 1 March 1900 on: Excel's serial calendar counts a 29 February 1900 that never existed, and nothing here imitates it. + +LibreOffice Calc recalculates three of these cells differently, and the +engine keeps Excel's numbers: ``WEEKNUM`` (every mode but 21) numbers +the last days of a year 1 where Excel reaches 53 or 54; ``DATEDIF``'s +``"YD"`` moves the start's month and day into the end's year where Excel +moves the end's into the start's; a ``DATEDIF`` with the start after the +end is ``#NUM!`` in Excel and ``#VALUE!`` in LibreOffice. """ from __future__ import annotations @@ -17,15 +25,20 @@ import math import numbers from calendar import isleap, monthrange -from datetime import date, datetime -from typing import Any, Callable, Union +from datetime import date, datetime, timedelta +from typing import Any, Callable, Sequence, Union from dateutil.relativedelta import relativedelta from ..core.variable import Variable -from ._helpers import make_func_var, operand, src +from ._helpers import ( + align_keyed, align_located, elementwise, is_error, is_series, make_func_var, operand, + placed, refuse_unpaired, src, +) +from .registry import register DateOperand = Union[Variable, date, datetime, str] +Holidays = Union[Variable, Sequence[DateOperand], date, datetime, str, None] _EXCEL_ONLY = frozenset({'excel'}) @@ -89,13 +102,25 @@ def _period_name(value: Any, i: int) -> str: return f"period {i + 1}" +def _by_date(func_name: str, lines: Sequence[Any], + raws: list[Any]) -> tuple[list[Any], Any, list[Any] | None]: + """The operands' values read as ``a + b`` reads them: lines on other + time windows by date, lines along keys key by key; with the combined + window and the keys the result runs on.""" + raws, keys = align_keyed(func_name, lines, raws) + raws, window = align_located(lines, raws) + refuse_unpaired(lines, raws, window) + return raws, window, keys + + def _broadcast(fn: Callable[..., Any], *raws: Any) -> tuple[Any, str]: """Apply ``fn`` over scalar or list operands, element-wise. Scalars repeat; lists shorter than the longest clamp to their last element (matches length-1 broadcast elsewhere). Returns ``(value, var_type)`` where ``var_type`` is ``'list'`` if any operand - is a list, else ``'scalar'``. + is a list, else ``'scalar'``. Lines are read by date first + (:func:`_by_date`). """ lists = [r for r in raws if isinstance(r, list)] if not lists: @@ -122,16 +147,18 @@ def _unary_date_func( def helper(value: DateOperand) -> Variable: _refuse_blanks(func_name, value) raw, expr = operand(value) + keys = align_keyed(func_name, [value], [raw])[1] if isinstance(raw, list): calc: Any = [py_fn(_extract_date(v)) for v in raw] var_type = 'list' else: calc = py_fn(_extract_date(raw)) var_type = 'scalar' - return make_func_var( + out = make_func_var( func_name, [expr], calc, 'int', var_type, source_code=f"{func_name}({src(value)})", ) + return placed(out, None, keys) return helper @@ -146,14 +173,30 @@ def helper(value: DateOperand) -> Variable: DAY.__doc__ = "Day of month 1-31 (renders ``=DAY(...)``). Always returns a Variable." +# One period of ``EDATE`` / ``EOMONTH`` whose months are a series: the +# months read whole, their fraction dropped, as the spreadsheet reads them. +def _edate(start: Any, months: Any) -> date: + return _extract_date(start) + relativedelta(months=int(months)) + + +def _eomonth(start: Any, months: Any) -> date: + shifted = _edate(start, months) + return date(shifted.year, shifted.month, monthrange(shifted.year, shifted.month)[1]) + + def EDATE(start_date: DateOperand, months: Union[Variable, int]) -> Variable: """Shift a date by ``months`` (renders ``=EDATE(...)``). Always returns a Variable. ``EDATE("2025-01-15", 3)`` → Variable with value ``date(2025, 4, 15)`` and formula ``EDATE("2025-01-15", 3)``. Negative values go backward. + ``months`` may vary by period: a series of months is one call per + period, ``=EDATE(C5, C9)`` in every column. """ _refuse_blanks("EDATE", start_date) _refuse_blanks("EDATE", months, needs="a number") + if is_series(months): + return elementwise("EDATE", _edate, [start_date, months], value_type='datetime', + exceptions='raise', render_backends=_EXCEL_ONLY) raw_start, start_expr = operand(start_date) raw_months, months_expr = operand(months) m = int(raw_months) @@ -168,11 +211,12 @@ def shift(d): calc = shift(raw_start) var_type = 'scalar' - return make_func_var( + out = make_func_var( "EDATE", [start_expr, months_expr], calc, 'datetime', var_type, source_code=f"EDATE({src(start_date)}, {m})", render_backends=_EXCEL_ONLY, ) + return placed(out, None, align_keyed("EDATE", [start_date], [raw_start])[1]) def EOMONTH(start_date: DateOperand, months: Union[Variable, int] = 0) -> Variable: @@ -180,9 +224,14 @@ def EOMONTH(start_date: DateOperand, months: Union[Variable, int] = 0) -> Variab ``EOMONTH("2025-01-15", 0)`` → Variable with value ``date(2025, 1, 31)``. ``months=1`` → ``date(2025, 2, 28)``. Always returns a Variable. + ``months`` may vary by period, one call per period as for + :func:`EDATE`. """ _refuse_blanks("EOMONTH", start_date) _refuse_blanks("EOMONTH", months, needs="a number") + if is_series(months): + return elementwise("EOMONTH", _eomonth, [start_date, months], value_type='datetime', + exceptions='raise', render_backends=_EXCEL_ONLY) raw_start, start_expr = operand(start_date) raw_months, months_expr = operand(months) m = int(raw_months) @@ -199,11 +248,12 @@ def end_of(d): calc = end_of(raw_start) var_type = 'scalar' - return make_func_var( + out = make_func_var( "EOMONTH", [start_expr, months_expr], calc, 'datetime', var_type, source_code=f"EOMONTH({src(start_date)}, {m})", render_backends=_EXCEL_ONLY, ) + return placed(out, None, align_keyed("EOMONTH", [start_date], [raw_start])[1]) def DATE( @@ -225,13 +275,23 @@ def DATE( d_raw, d_expr = operand(day) def build(y: Any, m: Any, d: Any) -> date: - return date(int(y), 1, 1) + relativedelta(months=int(m) - 1, days=int(d) - 1) + return _date_overflow(int(y), int(m), int(d)) + (y_raw, m_raw, d_raw), window, keys = _by_date( + "DATE", (year, month, day), [y_raw, m_raw, d_raw]) calc, var_type = _broadcast(build, y_raw, m_raw, d_raw) - return make_func_var( + out = make_func_var( "DATE", [y_expr, m_expr, d_expr], calc, 'datetime', var_type, source_code=f"DATE({src(year)}, {src(month)}, {src(day)})", ) + return placed(out, window, keys) + + +def _date_overflow(year: int, month: int, day: int) -> date: + """``DATE(year, month, day)`` as the spreadsheet reads it: a month or a + day past its range rolls over, so ``DATE(2025, 2, 29)`` is 1 March + 2025 and month 0 is the December before.""" + return date(year, 1, 1) + relativedelta(months=month - 1, days=day - 1) def _is_month_end(d: date) -> bool: @@ -300,22 +360,25 @@ def DAYS360( interest = balance * rate / 360 * days Always returns a Variable; scalar, or element-wise over list inputs - (e.g. ``DAYS360(dates.shift(1), dates, True)`` across a schedule). + (e.g. ``DAYS360(dates.shift(1), dates, True)`` across a schedule) — + the method too, each period's own. """ _refuse_blanks("DAYS360", start_date, end_date) start_raw, start_expr = operand(start_date) end_raw, end_expr = operand(end_date) method_raw, method_expr = operand(method) - european = bool(method_raw) - def count(s: Any, e: Any) -> int: - return _days360(_extract_date(s), _extract_date(e), european) + def count(s: Any, e: Any, european: Any) -> int: + return _days360(_extract_date(s), _extract_date(e), bool(european)) - calc, var_type = _broadcast(count, start_raw, end_raw) - return make_func_var( + (start_raw, end_raw, method_raw), window, keys = _by_date( + "DAYS360", (start_date, end_date, method), [start_raw, end_raw, method_raw]) + calc, var_type = _broadcast(count, start_raw, end_raw, method_raw) + out = make_func_var( "DAYS360", [start_expr, end_expr, method_expr], calc, 'int', var_type, source_code=f"DAYS360({src(start_date)}, {src(end_date)}, {src(method)})", ) + return placed(out, window, keys) def _yearfrac_30_360_us(d1: date, d2: date) -> int: @@ -441,18 +504,22 @@ def YEARFRAC( def fraction(s: Any, e: Any, b: Any) -> float: return _yearfrac(_extract_date(s), _extract_date(e), _yearfrac_basis(b)) + (start_raw, end_raw, basis_raw), window, keys = _by_date( + "YEARFRAC", (start_date, end_date, basis), [start_raw, end_raw, basis_raw]) calc, var_type = _broadcast(fraction, start_raw, end_raw, basis_raw) - return make_func_var( + out = make_func_var( "YEARFRAC", [start_expr, end_expr, basis_expr], calc, 'float', var_type, source_code=f"YEARFRAC({src(start_date)}, {src(end_date)}, {src(basis)})", render_backends=_EXCEL_ONLY, ) + return placed(out, window, keys) #: DAYS is an Excel 2013 function: in the file it must be written #: ``_xlfn.DAYS`` (bare, Excel shows ``#NAME?`` — measured in Excel 16.0, -#: build 20326). The Excel renderer adds the prefix (``_FILE_PREFIX``); the -#: function keeps its plain name everywhere a person reads the formula. +#: build 20326). The registry entry below carries the prefix +#: (``file_prefix='_xlfn.'``) for the Excel renderer; the function keeps +#: its plain name everywhere a person reads the formula. _DAYS_FUNC_NAME = "DAYS" @@ -471,12 +538,15 @@ def DAYS(end_date: DateOperand, start_date: DateOperand) -> Variable: def count(e: Any, s: Any) -> int: return (_extract_date(e) - _extract_date(s)).days + (end_raw, start_raw), window, keys = _by_date( + "DAYS", (end_date, start_date), [end_raw, start_raw]) calc, var_type = _broadcast(count, end_raw, start_raw) - return make_func_var( + out = make_func_var( _DAYS_FUNC_NAME, [end_expr, start_expr], calc, 'int', var_type, source_code=f"DAYS({src(end_date)}, {src(start_date)})", render_backends=_EXCEL_ONLY, ) + return placed(out, window, keys) def TODAY() -> Variable: @@ -489,3 +559,376 @@ def TODAY() -> Variable: "TODAY", [], date.today(), 'datetime', 'scalar', source_code="TODAY()", ) + + +# ─── Day counts and calendar positions ──────────────────────────────── +# +# ``DATEDIF``, ``WEEKDAY``, ``WEEKNUM``, ``NETWORKDAYS`` and ``WORKDAY`` are +# one call per period through the shared :func:`elementwise` helper over +# their date operands — a scalar date stands in every period, a series +# gives a series, tracks lift, an error cell passes through — under this +# module's law for what the function needs: a blank, or a value that is +# not a date, is refused by name at build time, never computed on as the +# year 1900. An error the DATA decides (``DATEDIF`` with the start after +# the end) is the period's ``#NUM!``, so an ``IF`` can guard it; a wrong +# constant (a unit, a return type) is refused at build time with Excel's +# error named. The holidays of ``NETWORKDAYS`` / ``WORKDAY`` are one range +# every period's call reads, never a series. No result is in an operand's +# unit: a day count, a weekday, a week number and a date carry none. + + +_DATEDIF_UNITS = ('Y', 'M', 'D', 'MD', 'YM', 'YD') + +#: The weekday a week begins on (Monday 0 … Sunday 6) for each return +#: type ``WEEKDAY`` and ``WEEKNUM`` share: 1 and 17 Sunday, 2, 3 and 11 +#: Monday, 12 to 16 Tuesday to Saturday. +_WEEK_STARTS = {1: 6, 2: 0, 3: 0, 11: 0, 12: 1, 13: 2, 14: 3, 15: 4, 16: 5, 17: 6} +_WEEKDAY_TYPES = frozenset(_WEEK_STARTS) +_WEEKNUM_TYPES = frozenset(_WEEK_STARTS) - {3} | {21} + + +def _no_unit(out: Variable) -> Variable: + out._unit = None + return out + + +def _whole_number(func_name: str, value: Any, what: str) -> int: + """A number the function reads whole, truncated as Excel truncates + it; TRUE/FALSE and text are refused (Excel returns ``#VALUE!``).""" + if isinstance(value, bool) or not isinstance(value, numbers.Real): + raise TypeError( + f"{func_name} {what} must be a number, got {value!r} (Excel returns #VALUE!)." + ) + return math.trunc(value) + + +def _return_type(func_name: str, value: Any, allowed: frozenset[int], spelled: str) -> int: + kind = _whole_number(func_name, value, "return type") + if kind not in allowed: + raise ValueError( + f"{func_name} return type must be {spelled}, got {value!r} " + f"(Excel returns #NUM!)." + ) + return kind + + +def _datedif_unit(value: Any) -> str: + if not isinstance(value, str) or value.upper() not in _DATEDIF_UNITS: + raise ValueError( + f"DATEDIF unit must be one of 'Y', 'M', 'D', 'MD', 'YM', 'YD' — whole " + f"years, whole months, days, days past the whole months, months past " + f"the whole years, days past the whole years — got {value!r} " + f"(Excel returns #NUM!)." + ) + return value.upper() + + +def _datedif(start: Any, end: Any, unit: Any) -> Any: + """Excel's ``DATEDIF(start, end, unit)``, its quirks included. + + ``"Y"`` and ``"M"`` are complete years and months — a year or a month + is complete only once the end's day is reached, so 31 January to 28 + February is 0 months and 29 February to the next 28 February 0 years. + ``"YM"`` is the months past the whole years. + + ``"MD"``, the days past the whole months, is the count Microsoft's + help flags as possibly negative: with the start's day past the end's, + the start's day is placed in the month before the end's and rolls + over as ``DATE()`` rolls it, so 31 January to 1 March of a common year + is 1 March minus "31 February" = 3 March, -2 (the rule LibreOffice + extracted from Excel by trial and error). + + ``"YD"``, the days past the whole years, places the END's month and + day in the START's year — or the year after, when they fall before + the start's — and counts from the start: 3 January 2015 to 28 July + 2016 is 206 through the 28-day February of 2015, where a count that + ends at the end date gives 207 (Excel's value is documented at + exceljet.net; LibreOffice counts the other way and notes in its + source that Excel answers 323 where it answers 324 for 22 June 1988 + to 11 May 2012). The rule rests on those measured values, none of + which has a 29 February at either end; where one does, it is + applied as written — 29 February 2024 to 1 March 2025 is 1 day past + the whole year (LibreOffice: 0), and an end on 29 February placed in + a common year rolls over to 1 March — and the day may differ in + Excel. + + A start after the end is ``#NUM!``, never swapped. + """ + s, e = _extract_date(start), _extract_date(end) + kind = _datedif_unit(unit) + if s > e: + return '#NUM!' + if kind == 'D': + return (e - s).days + years = e.year - s.year - int((e.month, e.day) < (s.month, s.day)) + if kind == 'Y': + return years + months = (e.year - s.year) * 12 + e.month - s.month - int(e.day < s.day) + if kind == 'M': + return months + if kind == 'YM': + return months - 12 * years + if kind == 'MD': + if e.day >= s.day: + return e.day - s.day + return (e - _date_overflow(e.year, e.month - 1, s.day)).days + later = int((e.month, e.day) < (s.month, s.day)) + return (_date_overflow(s.year + later, e.month, e.day) - s).days + + +def _weekday(value: Any, return_type: Any = 1) -> int: + d = _extract_date(value) + kind = _return_type("WEEKDAY", return_type, _WEEKDAY_TYPES, "1, 2, 3 or 11-17") + number = (d.weekday() - _WEEK_STARTS[kind]) % 7 + return number if kind == 3 else number + 1 + + +def _weeknum(value: Any, return_type: Any = 1) -> int: + d = _extract_date(value) + kind = _return_type("WEEKNUM", return_type, _WEEKNUM_TYPES, "1, 2, 11-17 or 21") + if kind == 21: + return d.isocalendar()[1] + first = date(d.year, 1, 1) + # Week 1 is the week that holds 1 January, however few of its days + # fall in the year. + into_week = (first.weekday() - _WEEK_STARTS[kind]) % 7 + return ((d - first).days + into_week) // 7 + 1 + + +def _networkdays(start: Any, end: Any, holidays: frozenset[date]) -> int: + s, e = _extract_date(start), _extract_date(end) + sign = 1 + if s > e: + s, e, sign = e, s, -1 + weeks, rest = divmod((e - s).days + 1, 7) + count = weeks * 5 + sum(1 for i in range(rest) if (s + timedelta(days=i)).weekday() < 5) + count -= sum(1 for h in holidays if s <= h <= e and h.weekday() < 5) + return sign * count + + +def _workday(start: Any, days: Any, holidays: frozenset[date]) -> date: + d = _extract_date(start) + left = _whole_number("WORKDAY", days, "days") + step = timedelta(days=1 if left > 0 else -1) + left = abs(left) + while left: + d += step + if d.weekday() < 5 and d not in holidays: + left -= 1 + return d + + +def _holidays(func_name: str, holidays: Holidays) -> tuple[frozenset[date], Any, Any]: + """The holidays as a set of dates, their AST node (None when none were + given) and the first error marker among them (None when none). A + blank among them is skipped, as an empty cell of the range is, and + so is a 0 — what a line extended with ``mo.zero()`` fills with, a + serial before every date, which the spreadsheet never counts.""" + if holidays is None: + return frozenset(), None, None + if isinstance(holidays, Variable): + from ..core.shape import reject_axised + from ..core.tracks import TrackValues + reject_axised(holidays, func_name) + if isinstance(holidays._value, TrackValues): + name = holidays._display_name or holidays.python_name or "the holidays" + raise ValueError( + f"{func_name}: {name!r} carries tracks ({', '.join(holidays._value.roles)}) — " + f"the holidays are one list of dates. Read one track first (.at(track=...))." + ) + elif isinstance(holidays, (list, tuple)): + if any(isinstance(h, Variable) for h in holidays): + raise TypeError( + f"{func_name} holidays: a list of lines is not a range — put the dates in " + f"one line (mo.Variable([...])) and pass that line." + ) + holidays = list(holidays) + raw, node = operand(holidays) + values = raw if isinstance(raw, list) else [raw] + error = next((v for v in values if is_error(v)), None) + dates = set() + for i, v in enumerate(values): + if v is None or is_error(v) or _is_zero(v): + continue + try: + dates.add(_extract_date(v)) + except (TypeError, ValueError): + if isinstance(holidays, Variable): + label = holidays._display_name or holidays.python_name + where = f"'{label}'" if label else "the holidays line" + at = f" in {_period_name(holidays, i)}" if isinstance(raw, list) else "" + elif isinstance(raw, list): + where, at = "the list", f" at position {i + 1}" + else: + where, at = "the holidays", "" + raise TypeError( + f"{func_name} holidays must be dates: {where} holds {v!r}{at}." + ) from None + return frozenset(dates), node, error + + +def _is_zero(v: Any) -> bool: + return isinstance(v, numbers.Real) and not isinstance(v, bool) and v == 0 + + +def _with_holidays( + func_name: str, + kernel: Callable[..., Any], + cell_args: list[Any], + holidays: Holidays, + value_type: str, +) -> Variable: + """One call per period over ``cell_args`` with the holidays read + whole by every call — the range position of the written formula, + not a series the helper would spread over the periods.""" + dates, node, error = _holidays(func_name, holidays) + + def call(*cells: Any) -> Any: + return error if error is not None else kernel(*cells, dates) + + args = cell_args if node is None else [*cell_args, holidays] + source_code = f"{func_name}({', '.join(src(a) for a in args)})" + out = elementwise(func_name, call, cell_args, value_type=value_type, exceptions='raise', + source_code=source_code, render_backends=_EXCEL_ONLY) + if node is not None: + # The same call with the holidays among its operands: on the window + # and along the keys the per-period call's lines were read on. + window = out.time if getattr(out, '_grain', None) is not None else None + out = placed(make_func_var( + func_name, [operand(a)[1] for a in cell_args] + [node], out._value, value_type, + out.var_type, source_code=source_code, render_backends=_EXCEL_ONLY, + ), window, out._keys) + return _no_unit(out) + + +def DATEDIF(start_date: DateOperand, end_date: DateOperand, unit: Union[Variable, str]) -> Variable: + """Whole years, months or days between two dates (renders ``=DATEDIF(...)``). + + ``unit`` as in Excel, in either case: ``"Y"`` complete years, ``"M"`` + complete months, ``"D"`` days, ``"YM"`` months past the whole years, + ``"MD"`` days past the whole months, ``"YD"`` days past the whole + years. Excel's own arithmetic, quirks included — ``"MD"`` can be + negative (31 January to 1 March is -1 in a leap year, -2 otherwise) + and ``"YD"`` counts from the start date through the start's own + year, see :func:`_datedif`. A start after the end is the period's + ``#NUM!``, so ``mo.IF(end >= start, mo.DATEDIF(start, end, "M"), 0)`` + guards it. Months of service to a leaving date:: + + months = mo.DATEDIF(joined, mo.IF(mo.ISBLANK(left), mo.TODAY(), left), "M") + + Always returns a Variable; scalar, or one call per period over list + inputs. A blank date is refused by name. + """ + _refuse_blanks("DATEDIF", start_date, end_date) + _refuse_blanks("DATEDIF", unit, needs="a unit") + return _no_unit(elementwise( + "DATEDIF", _datedif, [start_date, end_date, unit], value_type='int', + exceptions='raise', render_backends=_EXCEL_ONLY, + )) + + +def WEEKDAY(value: DateOperand, return_type: Union[Variable, int, None] = None) -> Variable: + """Day of the week as a number (renders ``=WEEKDAY(...)``). + + ``return_type`` as in Excel: 1 (default) Sunday is 1 to Saturday 7, 2 + Monday is 1 to Sunday 7, 3 Monday is 0 to Sunday 6, 11 to 17 a week + beginning on Monday to Sunday with that day 1. Written without the + type when none is given. Always returns a Variable; scalar, or one + call per period over a list of dates. + """ + _refuse_blanks("WEEKDAY", value) + args: list[Any] = [value] + if return_type is not None: + _refuse_blanks("WEEKDAY", return_type, needs="a number") + args.append(return_type) + return _no_unit(elementwise( + "WEEKDAY", _weekday, args, value_type='int', exceptions='raise', + render_backends=_EXCEL_ONLY, + )) + + +def WEEKNUM(value: DateOperand, return_type: Union[Variable, int, None] = None) -> Variable: + """Week of the year (renders ``=WEEKNUM(...)``). + + ``return_type`` as in Excel: 1 (default) weeks begin on Sunday, 2 on + Monday, 11 to 17 on Monday to Sunday — week 1 is the week that holds + 1 January, so the last days of a year reach week 53 — and 21 the ISO + week, where the first days of a year can belong to week 52 or 53 of + the year before. Written without the type when none is given. Always + returns a Variable; scalar, or one call per period over a list of + dates. LibreOffice Calc gives the last days of a year week 1 in every + mode but 21 - it counts the week that runs into the next year as that + year's first; the engine computes Excel's 53 or 54. + """ + _refuse_blanks("WEEKNUM", value) + args: list[Any] = [value] + if return_type is not None: + _refuse_blanks("WEEKNUM", return_type, needs="a number") + args.append(return_type) + return _no_unit(elementwise( + "WEEKNUM", _weeknum, args, value_type='int', exceptions='raise', + render_backends=_EXCEL_ONLY, + )) + + +def NETWORKDAYS( + start_date: DateOperand, + end_date: DateOperand, + holidays: Holidays = None, +) -> Variable: + """Working days from ``start_date`` to ``end_date``, both included + (renders ``=NETWORKDAYS(...)``). + + Saturdays and Sundays are not working days, nor are the ``holidays``: + a line of dates in the model (``inputs.holidays``, written as the + range the formula reads; a line shorter than the model's window keeps + its own length with ``extend=mo.none()``), a plain list of dates, or + one date. A holiday on a weekend or outside the span changes nothing; + a blank or a 0 among the holidays is skipped, as an empty cell is. A + start after the end counts negative. Always returns a Variable; + scalar, or one call per period over list dates with the same holidays + throughout. A plain list of holidays has no cells of its own, so the + workbook carries the computed value in its place (the renderer warns + when it does); name the holidays as a line to keep the formula live. + """ + _refuse_blanks("NETWORKDAYS", start_date, end_date) + return _with_holidays("NETWORKDAYS", _networkdays, [start_date, end_date], holidays, 'int') + + +def WORKDAY( + start_date: DateOperand, + days: Union[Variable, int, float], + holidays: Holidays = None, +) -> Variable: + """The date ``days`` working days after ``start_date`` (renders ``=WORKDAY(...)``). + + Saturdays, Sundays and the ``holidays`` (as for :func:`NETWORKDAYS`) + are stepped over; negative ``days`` go backward; a fraction is + truncated; 0 is the start date itself, weekend or not. Always + returns a Variable holding a date; scalar, or one call per period + over a list start or a list of day counts. + """ + _refuse_blanks("WORKDAY", start_date) + _refuse_blanks("WORKDAY", days, needs="a number") + return _with_holidays("WORKDAY", _workday, [start_date, days], holidays, 'datetime') + + +register("YEAR", shape="elementwise") +register("MONTH", shape="elementwise") +register("DAY", shape="elementwise") +register("DATE", shape="elementwise") +register("EDATE", shape="elementwise") +register("EOMONTH", shape="elementwise") +register("DAYS360", shape="elementwise") +register("TODAY", shape="elementwise") +# Computed over a series by their helpers; not run period by period +# inside a loop container. +register("YEARFRAC", shape="scalar") +register(_DAYS_FUNC_NAME, shape="scalar", file_prefix="_xlfn.") +register("DATEDIF", shape="elementwise") +register("WEEKDAY", shape="elementwise") +register("WEEKNUM", shape="elementwise") +# One call per period over the dates (or the day count), the holidays a +# range every period's call reads: ``=NETWORKDAYS(B2, C2, Inputs!B5:D5)``. +register("NETWORKDAYS", shape="range", range_args=[2]) +register("WORKDAY", shape="range", range_args=[2]) diff --git a/src/modeleon/functions/financial.py b/src/modeleon/functions/financial.py index 1eff947..62e68e0 100644 --- a/src/modeleon/functions/financial.py +++ b/src/modeleon/functions/financial.py @@ -1,15 +1,27 @@ # SPDX-License-Identifier: Apache-2.0 -"""Financial formula helpers — IRR, NPV, XIRR, XNPV, PMT, FV, PV. +"""Financial formula helpers — IRR, NPV, XIRR, XNPV, PMT, FV, PV; the +loan schedule NPER, IPMT, PPMT, RATE, CUMIPMT, CUMPRINC; depreciation +SLN, DB, DDB, SYD. Every function builds a live Excel formula (``=IRR(B2:B6)``, ``=PMT(...)``) alongside the Python-side value. Pass a Variable (including list-valued ones) to get the formula; pass a raw list to get a value-only result — the latter is useful for ad-hoc calculations outside the model tree. + +Units: the amounts a function reads together — a loan's present and +future value, a payment and a balance, a cost and its salvage — are in +one unit, and two are refused, as ``+`` refuses them; the result is in +it (``NPV`` and ``XNPV`` in their flows'). A rate, a term, a period and +a type are never the result's unit, and a rate (``IRR``, ``RATE``) or a +count of periods (``NPER``) carries none. """ from __future__ import annotations +import decimal +import functools import logging +import math from datetime import date as _date_type from typing import Any, List, Optional, Union @@ -17,13 +29,20 @@ from ..core.expr import FuncCall, Literal, VarRef from ..core.variable import Variable -from ._helpers import val +from ._helpers import ( + elementwise, function_unit, is_series, operand_unit, same_unit, unit_law, val, + waits_on_loop) +from .registry import register from .dates import _extract_date +from .mathfn import _spreadsheet_round +from .text import _number_of logger = logging.getLogger(__name__) -__all__ = ['IRR', 'NPV', 'XIRR', 'XNPV', 'PMT', 'FV', 'PV'] +__all__ = ['IRR', 'NPV', 'XIRR', 'XNPV', 'PMT', 'FV', 'PV', + 'NPER', 'IPMT', 'PPMT', 'RATE', 'CUMIPMT', 'CUMPRINC', + 'SLN', 'DB', 'DDB', 'SYD'] # ─── Shared helpers ─────────────────────────────────────────────── @@ -116,7 +135,8 @@ def _extract_cashflow_values(cash_flows, func: str) -> List[float]: def _wrap_with_formula(value: float, func_name: str, args: list) -> Variable: - """Build a scalar-valued result Variable with an Excel FuncCall formula.""" + """Build a scalar-valued result Variable with an Excel FuncCall formula, + in the unit the function's law gives.""" result = Variable(value_type='float') result._set_expr(FuncCall( func_name, args, @@ -125,6 +145,14 @@ def _wrap_with_formula(value: float, func_name: str, args: list) -> Variable: result._value = float(value) result.var_type = 'scalar' result.value_type = 'float' + try: + unit = function_unit(func_name, args, operand_unit) + except ValueError: + if not waits_on_loop(result): # provisional until its loop closes + raise + unit = None + if unit is not None: + result._unit = unit return result @@ -133,6 +161,17 @@ def _formula_arg(value) -> "Literal | VarRef": return VarRef(value) if isinstance(value, Variable) else Literal(value) +def _range_arg(value: Any) -> Any: + """AST node for a flows or dates argument: a row is its range; a list + holding lines is the list of their cells, each its own node, as a + criteria range is - the book writes their cells and the unit law + reads each flow's unit; a list of numbers typed in is a literal.""" + from ..core.expr import ListExpr + if isinstance(value, (list, tuple)) and any(isinstance(x, Variable) for x in value): + return ListExpr([_formula_arg(x) for x in value]) + return _formula_arg(value) + + # ─── Core algorithms ────────────────────────────────────────────── @@ -331,6 +370,15 @@ def _years_from_first(dates, count: int, func: str, *, earlier_allowed: bool) -> # ─── Public functions ───────────────────────────────────────────── +# The rate a search starts from is one number for the whole row. +def _one_guess(guess: Any, func: str) -> None: + if isinstance(val(guess), (list, tuple)): + raise TypeError( + f"{func}'s guess is one number — the rate the search starts from — " + f"got a series. Give one value, or leave it out (0.1)." + ) + + def IRR( cash_flows: Union[Variable, List[float]], guess: float = 0.1, @@ -349,6 +397,7 @@ def IRR( cf = mo.Variable([-100_000, 30_000, 40_000, 50_000, 60_000]) project_irr = mo.IRR(cf) # Excel: =IRR(B2:F2, 0.1) """ + _one_guess(guess, "IRR") cf_values = _present(_extract_cashflow_values(cash_flows, "IRR")) if len(cf_values) < 2: raise ValueError( @@ -364,11 +413,9 @@ def IRR( irr_value = _irr_solve(cf_array, guess) - if isinstance(cash_flows, Variable): - return _wrap_with_formula( - irr_value, 'IRR', - [VarRef(cash_flows), Literal(guess)], - ) + flows = _range_arg(cash_flows) + if not isinstance(flows, Literal): + return _wrap_with_formula(irr_value, 'IRR', [flows, Literal(guess)]) # Raw list input — no cell range to reference; return value-only. return Variable(irr_value, value_type='float') @@ -392,14 +439,17 @@ def NPV( # Excel: =B1 + NPV(B2, C3:E3) """ rate_value = _arg(rate) + if isinstance(rate_value, (list, tuple)): + raise TypeError( + "NPV takes a single discount rate — got a series. For a rate that " + "changes over time, discount each flow explicitly." + ) cf_values = _present(_extract_cashflow_values(cash_flows, "NPV")) npv_value = _npv(rate_value, np.array(cf_values, dtype=float)) - if isinstance(rate, Variable) or isinstance(cash_flows, Variable): - return _wrap_with_formula( - npv_value, 'NPV', - [_formula_arg(rate), _formula_arg(cash_flows)], - ) + flows = _range_arg(cash_flows) + if isinstance(rate, Variable) or not isinstance(flows, Literal): + return _wrap_with_formula(npv_value, 'NPV', [_formula_arg(rate), flows]) return Variable(npv_value, value_type='float') @@ -443,6 +493,7 @@ def XIRR( d = mo.Variable([date(2025,1,1), date(2025,3,15), date(2025,7,10), date(2025,12,31)]) project_xirr = mo.XIRR(cf, d) # Excel: =XIRR(B2:E2, B3:E3, 0.1) """ + _one_guess(guess, "XIRR") cf_array = np.array(_zero_blanks(_extract_cashflow_values(cash_flows, "XIRR")), dtype=float) years = _years_from_first(dates, len(cf_array), "XIRR", earlier_allowed=True) if not (np.any(cf_array < 0) and np.any(cf_array > 0)): @@ -471,11 +522,9 @@ def XIRR( "cash flows and their dates." ) from exc - if isinstance(cash_flows, Variable) and isinstance(dates, Variable): - return _wrap_with_formula( - rate, 'XIRR', - [VarRef(cash_flows), VarRef(dates), _formula_arg(guess)], - ) + flows, when = _range_arg(cash_flows), _range_arg(dates) + if not isinstance(flows, Literal) and not isinstance(when, Literal): + return _wrap_with_formula(rate, 'XIRR', [flows, when, _formula_arg(guess)]) return Variable(float(rate), value_type='float') @@ -524,14 +573,46 @@ def XNPV( years = _years_from_first(dates, len(cf_array), "XNPV", earlier_allowed=False) xnpv_value = float(np.sum(cf_array / (1 + rate_value) ** years)) - if isinstance(cash_flows, Variable) and isinstance(dates, Variable): - return _wrap_with_formula( - xnpv_value, 'XNPV', - [_formula_arg(rate), VarRef(cash_flows), VarRef(dates)], - ) + flows, when = _range_arg(cash_flows), _range_arg(dates) + if not isinstance(flows, Literal) and not isinstance(when, Literal): + return _wrap_with_formula(xnpv_value, 'XNPV', [_formula_arg(rate), flows, when]) return Variable(xnpv_value, value_type='float') +# The payment, future value and present value of one period's numbers — +# the arithmetic of a call over scalars, and of each period's cell of a +# call over a series. +def _pmt(r: Any, n: Any, v: Any, f: Any, w: Any) -> Any: + if r == 0: + return -(v + f) / n + return -(f + v * (1 + r) ** n) * r / ((1 + r * w) * ((1 + r) ** n - 1)) + + +def _fv(r: Any, n: Any, p: Any, v: Any, w: Any) -> Any: + if r == 0: + return -(v + p * n) + return -(v * (1 + r) ** n + p * (1 + r * w) * ((1 + r) ** n - 1) / r) + + +def _pv(r: Any, n: Any, p: Any, f: Any, w: Any) -> Any: + if r == 0: + return -(f + p * n) + return -(f + p * (1 + r * w) * ((1 + r) ** n - 1) / r) / (1 + r) ** n + + +def _series_among(*args: Any) -> bool: + return any(is_series(a) for a in args) + + +# A call with a series among its arguments: one per period, every argument +# written as the call over scalars writes it, read as ``a + b`` reads its +# operands (a scalar in every period, lines on other windows by date, +# lines along keys key by key). +def _each_period(func: str, kernel: Any, *args: Any) -> Variable: + return elementwise(func, kernel, list(args), blank=0, value_type='float', + render_backends=_EXCEL_ONLY) + + def PMT( rate: Union[float, Variable], nper: Union[int, float, Variable], @@ -549,12 +630,14 @@ def PMT( # Monthly payment on a $300k 30-year loan at 6% annual monthly = mo.PMT(0.06 / 12, 30 * 12, 300_000) # ≈ -1,798.65 + + Any argument may vary by period: a series among them makes the call + one per period, ``=PMT(C9 / 100, 12, 1000, 0, 0)`` in every column. """ + if _series_among(rate, nper, pv, fv, when): + return _each_period('PMT', _pmt, rate, nper, pv, fv, when) r, n, pv_v, fv_v, w = _arg(rate), _arg(nper), _arg(pv), _arg(fv), _arg(when) - if r == 0: - pmt_val = -(pv_v + fv_v) / n - else: - pmt_val = -(fv_v + pv_v * (1 + r) ** n) * r / ((1 + r * w) * ((1 + r) ** n - 1)) + pmt_val = _pmt(r, n, pv_v, fv_v, w) result = _wrap_with_formula( pmt_val, 'PMT', @@ -577,12 +660,14 @@ def FV( # $500/month into 7%-return account for 30 years, starting from $0 retirement = mo.FV(0.07 / 12, 30 * 12, -500) # ≈ 611,729 + + Any argument may vary by period, one call per period as for + :func:`PMT`. """ + if _series_among(rate, nper, pmt, pv, when): + return _each_period('FV', _fv, rate, nper, pmt, pv, when) r, n, p, pv_v, w = _arg(rate), _arg(nper), _arg(pmt), _arg(pv), _arg(when) - if r == 0: - fv_val = -(pv_v + p * n) - else: - fv_val = -(pv_v * (1 + r) ** n + p * (1 + r * w) * ((1 + r) ** n - 1) / r) + fv_val = _fv(r, n, p, pv_v, w) result = _wrap_with_formula( fv_val, 'FV', @@ -605,12 +690,14 @@ def PV( # PV of $1000/yr for 20 years at 5% discount pv = mo.PV(0.05, 20, -1000) # ≈ 12,462 + + Any argument may vary by period, one call per period as for + :func:`PMT`. """ + if _series_among(rate, nper, pmt, fv, when): + return _each_period('PV', _pv, rate, nper, pmt, fv, when) r, n, p, fv_v, w = _arg(rate), _arg(nper), _arg(pmt), _arg(fv), _arg(when) - if r == 0: - pv_val = -(fv_v + p * n) - else: - pv_val = -(fv_v + p * (1 + r * w) * ((1 + r) ** n - 1) / r) / (1 + r) ** n + pv_val = _pv(r, n, p, fv_v, w) result = _wrap_with_formula( pv_val, 'PV', @@ -618,3 +705,574 @@ def PV( ) result._source_code = f"PV({r}, {n}, {p}, {fv_v}, {w})" return result + + +# ─── Loan schedules and depreciation ────────────────────────────── +# +# ``NPER``, ``IPMT``, ``PPMT``, ``RATE``, ``CUMIPMT``, ``CUMPRINC`` and +# the depreciation functions ``SLN``, ``DB``, ``DDB``, ``SYD`` are each +# one call per period through the shared :func:`elementwise` helper, +# over a kernel that takes one period's numbers: scalars give one +# number, a period-shaped operand (``per=mo.time.index``) a schedule +# row. A kernel reads a cell as the spreadsheet does (``_number``) — a +# blank is 0, a boolean 1 or 0, numeric text its number, other text +# ``#VALUE!``, an error cell passes through — and a number outside the +# function's domain is ``#NUM!``. An optional argument is written only +# when given. + +Numeric = Variable | int | float + + +def _number(v: Any) -> int | float: + """A cell's number as the spreadsheet reads it, as a plain Python + number: numpy's float warns and gives ``inf`` where Python raises + the division by zero or the overflow the helper reads as an error.""" + x = _number_of(v) + return x if type(x) is int else float(x) + + +def _whole(v: Any) -> int: + """A period, a month or a type read whole: the number as a cell shows + it (15 significant digits), its fraction dropped — 12.999999999999998 + is period 13, as the spreadsheet counts it.""" + x = _number(v) + if not math.isfinite(x): + raise OverflowError(f"not a finite number: {v!r}") # the helper's #NUM! + return int(_spreadsheet_round(x, 0, decimal.ROUND_DOWN)) + + +def _advance(when: Any) -> int: + """Excel's ``type``: 1 for a payment at the start of the period, 0 + at its end. Any other number reads as 1, as the spreadsheet reads it.""" + return 1 if _number(when) else 0 + + +def _written(*args: Any) -> list[Any]: + """The arguments as the cell writes them: an optional one that is + omitted (None) is dropped from the end; one before a given one is + its default, 0.""" + out = list(args) + while out and out[-1] is None: + out.pop() + return [0 if a is None else a for a in out] + + +def _payment(r: Any, n: Any, v: Any, f: Any, w: int) -> Any: + """The payment ``PMT`` computes — the kernel the schedule functions + share. ``(1 + r) ** n - 1`` is ``expm1``, as the spreadsheet computes + it: at a rate of 1e-12 the subtraction would keep four digits.""" + if r == 0: + return -(v + f) / n + if r == -1: + t, t_less_one = 0.0, -1.0 # every amount gone after a period + else: + growth = n * math.log1p(r) + t, t_less_one = math.exp(growth), math.expm1(growth) + return -(f + v * t) * r / ((1 + r * w) * t_less_one) + + +def _balance(r: Any, k: Any, payment: Any, v: Any, w: int) -> Any: + """``FV`` after ``k`` payments: minus the balance the loan still carries.""" + if r == 0: + return -(v + payment * k) + t = (1 + r) ** k + return -(v * t + payment * (1 + r * w) * (t - 1) / r) + + +def _nper(rate: Any, pmt: Any, pv: Any, fv: Any = 0, when: Any = 0) -> Any: + r, p, v, f = (_number(x) for x in (rate, pmt, pv, fv)) + w = _advance(when) + if v + f == 0: + return 0.0 # nothing to pay off + if r == 0: + return -(v + f) / p + 0.0 if p else '#NUM!' + if r <= -1: + return '#NUM!' + z = p * (1 + r * w) / r + if v + z == 0: + return '#NUM!' # the payment only ever covers the interest + ratio = (z - f) / (v + z) + # ``+ 0.0`` throughout: -0.0 + 0.0 is 0.0, and a cell shows 0, not -0. + return math.log(ratio) / math.log1p(r) + 0.0 if ratio > 0 else '#NUM!' + + +def _ipmt(rate: Any, per: Any, nper: Any, pv: Any, fv: Any = 0, when: Any = 0) -> Any: + r, p, n, v, f = (_number(x) for x in (rate, per, nper, pv, fv)) + w = _advance(when) + if p < 1 or p > n or r < -1: + return '#NUM!' + if r == 0 or (w and p == 1): + return 0.0 + if p == 1: + # The interest on the loan itself, the payment not needed: a term + # too long for (1 + rate) ** nper still has a first period. + return -v * r + 0.0 + interest = _balance(r, p - 1, _payment(r, n, v, f, w), v, w) * r + return (interest / (1 + r) if w else interest) + 0.0 + + +def _ppmt(rate: Any, per: Any, nper: Any, pv: Any, fv: Any = 0, when: Any = 0) -> Any: + interest = _ipmt(rate, per, nper, pv, fv, when) + if isinstance(interest, str): + return interest + r, n, v, f = (_number(x) for x in (rate, nper, pv, fv)) + return _payment(r, n, v, f, _advance(when)) - interest + 0.0 + + +#: ``RATE`` iterates as LibreOffice's source does: Newton's method on the +#: balance from the guess, at most 150 steps, done when a step is under +#: 1e-7 or the balance under 1e-14. With no guess given, 0.1 is tried +#: and then, in turn, 0.2, 0.05, 0.3, 0.1/3, ... 1.0, 0.01, each where +#: the last found no rate. An end that is no rate (-100%, or a point +#: that does not balance the flows) is a start that found none: the +#: spreadsheet stops there and shows it. +_RATE_STEPS = 150 +_RATE_STEP_DONE = 1e-7 +_RATE_BALANCE_DONE = 1e-14 +_RATE_GUESSES = (0.1,) + tuple(g for k in range(2, 11) for g in (0.1 * k, 0.1 / k)) + + +def _rate_iteration(n: float, p: float, v: float, f: float, x: float) -> float | None: + """The rate Newton's method reaches from ``x`` on the end-of-period + balance ``f + v (1+x)^n + p ((1+x)^n - 1) / x``, or None: LibreOffice's + arithmetic and stopping rules, step for step, so it settles where the + spreadsheet settles and fails where it fails. A whole ``n`` may pass + below -100% on the way (a whole power of a negative number is real); + a fractional one may not.""" + whole = float(n).is_integer() + if not whole: + x = max(x, -1.0) + try: + for _ in range(_RATE_STEPS): + if whole: + t1 = (1.0 + x) ** (n - 1.0) + t = t1 * (1.0 + x) + else: + t, t1 = (1.0 + x) ** n, (1.0 + x) ** (n - 1.0) + if x == 0: + series, slope = float(n), n * (n - 1.0) / 2.0 + else: + series = (t - 1.0) / x + slope = n * t1 / x - series / x + balance = f + v * t + p * series + if abs(balance) < _RATE_BALANCE_DONE: + break + d_balance = v * n * t1 + p * slope + x_new = x - balance / d_balance if d_balance else x + 1.1 * _RATE_STEP_DONE + done = abs(x_new - x) < _RATE_STEP_DONE + x = x_new + if not whole and not x >= -1.0: + return None + if done: + break + else: + return None + except (OverflowError, ZeroDivisionError): + return None # the spreadsheet's inf / nan: no root + return x if not whole or x > -1.0 else None + + +def _rate_gap(n: float, p: float, v: float, f: float, r: float) -> tuple[float, float, float]: + """The balance ``f + v (1+r)^n + p ((1+r)^n - 1) / r`` a rate leaves + (zero at the rate sought), its slope, and the size of its terms; + infinite where they overflow. log1p / expm1 keep the series exact at + a small rate, where the slope is read at its limit, n (n-1) / 2.""" + try: + lg = math.log1p(r) + t1, t2 = math.exp(n * lg), math.exp((n - 1) * lg) + series = n if r == 0 else math.expm1(n * lg) / r + except OverflowError: + return math.inf, math.inf, math.inf + slope = (n * t2 - series) / r if abs(r) > 1e-8 else n * (n - 1) / 2 + size = abs(f) + abs(v * t1) + abs(p * series) + return f + v * t1 + p * series, v * n * t2 + p * slope, size + + +def _settled_rate(n: float, p: float, v: float, f: float, x: float, + at_minus_one: bool) -> float | None: + """The rate the iteration ended on, polished on the exact series, or + None where it is no rate: -100% (every amount gone — a root of the + flows ``at_minus_one``), or a point that does not balance them (an + iteration that stalled far out).""" + if not 1.0 + x > (1e-5 if at_minus_one else 0.0): + return None + g, dg, size = _rate_gap(n, p, v, f, x) + if not (math.isfinite(size) and abs(g) <= 1e-6 * size): + return None + # The spreadsheet stops within 1e-7; a few exact steps settle the + # last digits (a rate of 0 is 0, not 1e-11), never past -100%. + for _ in range(3): + step = g / dg if dg else 0.0 + if not (abs(step) <= 1e-6 * (1 + abs(x)) and 1.0 + (x - step) > 0.0): + break + x -= step + if abs(step) <= 1e-15 * (1 + abs(x)): + break + g, dg, size = _rate_gap(n, p, v, f, x) + # A rate too small to move 1 + rate is the rate 0, as the cell shows it. + return x + 0.0 if 1.0 + x != 1.0 else 0.0 + + +def _rate(nper: Any, pmt: Any, pv: Any, fv: Any = 0, when: Any = 0, guess: Any = None) -> Any: + n, p, v, f = (_number(x) for x in (nper, pmt, pv, fv)) + if n <= 0: + return '#NUM!' + w = _advance(when) + if v + p * w == 0 and f + p * (1 - w) == 0 and (n == 1 or p == 0): + # Every rate balances flows that are not there, or a single period + # repaid as it opens: the iteration stops where it starts. + start = 0.1 if guess is None else _number(guess) + return start + 0.0 if start > -1 else '#NUM!' + at_minus_one = f + p * (1 - w) == 0 + # A payment at the start of the period is one made at time zero: it + # moves from the annuity into the present value and out of the + # future value, and the iteration is the end-of-period one. + v, f = v + p * w, f - p * w + for start in _RATE_GUESSES if guess is None else (_number(guess),): + x = _rate_iteration(n, p, v, f, start) + rate = None if x is None else _settled_rate(n, p, v, f, x, at_minus_one) + if rate is not None: + return rate + return '#NUM!' + + +def _cumulative(kernel: Any, rate: Any, nper: Any, pv: Any, start: Any, end: Any, + when: Any) -> Any: + r, n, v = (_number(x) for x in (rate, nper, pv)) + # The periods and the type are read whole, as Excel documents them. + s, e, w = (_whole(x) for x in (start, end, when)) + if r <= 0 or n <= 0 or v <= 0 or s < 1 or e < s or e > n or w not in (0, 1): + return '#NUM!' + return sum(kernel(r, i, n, v, 0, w) for i in range(s, e + 1)) + 0.0 + + +def _sln(cost: Any, salvage: Any, life: Any) -> Any: + c, s, n = (_number(x) for x in (cost, salvage, life)) + return (c - s) / n + 0.0 # a life of 0 is the helper's #DIV/0! + + +def _db(cost: Any, salvage: Any, life: Any, period: Any, month: Any = 12) -> Any: + c, s, n, p = (_number(x) for x in (cost, salvage, life, period)) + m = _whole(month) + if c <= 0 or s < 0 or s > c or n <= 0 or p < 1 or p > n + 1 or m < 1 or m > 12: + return '#NUM!' + # Excel's documented formula: the rate rounded to three decimals as + # ROUND rounds it; the first period's charge over ``month`` months; + # each next period's on the balance; and in a period past ``life`` + # the stub over the months the first year left. The period is read + # whole: 1.5 is the first period, even past a life shorter than that. + rate = _spreadsheet_round(1 - (s / c) ** (1 / n), 3) + first = c * rate * m / 12 + if _whole(p) == 1: + return first + 0.0 + charge, total = 0.0, first + for _ in range(2, _whole(min(n, p)) + 1): + charge = (c - total) * rate + total += charge + if p > n: + charge = (c - total) * rate * (12 - m) / 12 + return charge + 0.0 + + +def _ddb(cost: Any, salvage: Any, life: Any, period: Any, factor: Any = 2) -> Any: + c, s, n, p, k = (_number(x) for x in (cost, salvage, life, period, factor)) + if c < 0 or s < 0 or s > c or k <= 0 or p < 1 or p > n: + return '#NUM!' + # The book value before and after the period at the declining rate, + # the charge their difference, stopped at the salvage value. + rate = k / n + if rate >= 1: + rate, before = 1.0, (c if p == 1 else 0.0) + else: + before = c * (1 - rate) ** (p - 1) + after = c * (1 - rate) ** p + charge = before - s if after < s else before - after + return max(charge, 0.0) + 0.0 + + +def _syd(cost: Any, salvage: Any, life: Any, per: Any) -> Any: + c, s, n, p = (_number(x) for x in (cost, salvage, life, per)) + if n <= 0 or p < 1 or p > n: + return '#NUM!' + return (c - s) * (n - p + 1) * 2 / (n * (n + 1)) + 0.0 + + +def NPER( + rate: Numeric, + pmt: Numeric, + pv: Numeric, + fv: Numeric | None = None, + when: Numeric | None = None, +) -> Variable: + """Number of periods of a loan or an annuity (renders + ``=NPER(rate, pmt, pv, [fv], [type])``). + + ``NPER(0.01, -100, -1000, 10000)`` is 60.08; a rate of 0 gives + ``-(pv + fv) / pmt``; ``when=1`` (Excel's ``type``) puts the payment + at the start of each period. A present and a future value that + cancel need 0 periods; a payment that never pays the balance down + (``NPER(0.1, 100, -1000)``), a zero payment at a zero rate, or a + rate of -100% or below is ``#NUM!``. The result carries no unit; the + payment and the values are in one. Element-wise over a period-shaped + operand. + """ + return elementwise("NPER", _nper, _written(rate, pmt, pv, fv, when), + blank=0, value_type='float') + + +def IPMT( + rate: Numeric, + per: Numeric, + nper: Numeric, + pv: Numeric, + fv: Numeric | None = None, + when: Numeric | None = None, +) -> Variable: + """Interest paid in period ``per`` (renders + ``=IPMT(rate, per, nper, pv, [fv], [type])``). + + Signs as ``PMT``: a positive ``pv`` gives a negative payment. + ``IPMT(0.1, 3, 3, 8000)`` is -292.45. A rate of 0 is 0 interest; + with ``when=1`` the first period's interest is 0 and every later + one is discounted a period. ``per`` below 1 or above ``nper``, or a + rate below -100%, is ``#NUM!``. A series ``per`` — ``mo.time.index`` + — makes the call a schedule row:: + + interest = mo.IPMT(inp.rate, mo.time.index, inp.term, inp.loan) + """ + return elementwise("IPMT", _ipmt, _written(rate, per, nper, pv, fv, when), + blank=0, value_type='float') + + +def PPMT( + rate: Numeric, + per: Numeric, + nper: Numeric, + pv: Numeric, + fv: Numeric | None = None, + when: Numeric | None = None, +) -> Variable: + """Principal paid in period ``per`` (renders + ``=PPMT(rate, per, nper, pv, [fv], [type])``). + + The payment less :func:`IPMT`'s interest, with its rules: + ``PPMT(0.08, 10, 10, 200000)`` is -27,598.05; at a rate of 0 the + whole payment ``-(pv + fv) / nper``; ``per`` outside 1 .. ``nper`` + is ``#NUM!``. A series ``per`` makes the call a schedule row. + """ + return elementwise("PPMT", _ppmt, _written(rate, per, nper, pv, fv, when), + blank=0, value_type='float') + + +def RATE( + nper: Numeric, + pmt: Numeric, + pv: Numeric, + fv: Numeric | None = None, + when: Numeric | None = None, + guess: Numeric | None = None, +) -> Variable: + """The rate per period of a loan or an annuity (renders + ``=RATE(nper, pmt, pv, [fv], [type], [guess])``). + + ``RATE(48, -200, 8000)`` is 0.77% a month. Found as LibreOffice + finds it: Newton's iteration from ``guess``, at most 150 steps, + settled when a step is under 1e-7; with no guess it starts at 0.1 + (Excel's default) and, should that find no rate, at 0.2, 0.05, 0.3, + 0.033, ... 1.0, 0.01 in turn, so of several roots it finds the + spreadsheet's (``RATE(5, -1000, 1000)`` is 96.6%). The rate settled + on is refined to full precision; payments that repay the loan exactly + have a rate of 0. An end at -100% or at a point that does not + balance the flows is no rate: the spreadsheet shows it, the engine + tries the next start. ``#NUM!`` when ``nper`` is 0 or below, or when + no start finds a rate (``RATE(10, 100, 1000)``: every flow the same + way; a guess the iteration runs away from). With no flow at all + every rate balances, and the guess is the answer. Excel documents 20 + steps, so a rate that takes more may be ``#NUM!`` in Excel. The + result carries no unit. Element-wise over a period-shaped operand. + """ + return elementwise("RATE", _rate, _written(nper, pmt, pv, fv, when, guess), + blank=0, value_type='float') + + +def CUMIPMT( + rate: Numeric, + nper: Numeric, + pv: Numeric, + start_period: Numeric, + end_period: Numeric, + when: Numeric = 0, +) -> Variable: + """Interest paid between two periods, both included (renders + ``=CUMIPMT(rate, nper, pv, start_period, end_period, type)``). + + The sum of :func:`IPMT` over the periods — ``CUMIPMT(0.09 / 12, + 360, 125000, 13, 24, 0)`` is -11,135.23, the second year's interest. + The periods and ``when`` are read whole, as a cell shows them + (12.999999999999998 is period 13) and as Excel documents it, their + fractions dropped; LibreOffice refuses a fractional ``when``. + ``when`` is written always, as Excel requires it. A rate, term or + present value of 0 or below, a start before period 1 or after the + end, an end after ``nper``, or a ``when`` other than 0 and 1 is + ``#NUM!``. + """ + return elementwise( + "CUMIPMT", functools.partial(_cumulative, _ipmt), + [rate, nper, pv, start_period, end_period, 0 if when is None else when], + blank=0, value_type='float', + ) + + +def CUMPRINC( + rate: Numeric, + nper: Numeric, + pv: Numeric, + start_period: Numeric, + end_period: Numeric, + when: Numeric = 0, +) -> Variable: + """Principal paid between two periods, both included (renders + ``=CUMPRINC(rate, nper, pv, start_period, end_period, type)``). + + The sum of :func:`PPMT` over the periods, with :func:`CUMIPMT`'s + rules — ``CUMPRINC(0.09 / 12, 360, 125000, 13, 24, 0)`` is -934.11. + """ + return elementwise( + "CUMPRINC", functools.partial(_cumulative, _ppmt), + [rate, nper, pv, start_period, end_period, 0 if when is None else when], + blank=0, value_type='float', + ) + + +def SLN(cost: Numeric, salvage: Numeric, life: Numeric) -> Variable: + """Straight-line depreciation for one period (renders ``=SLN(cost, salvage, life)``). + + ``(cost - salvage) / life``: ``SLN(30000, 7500, 10)`` is 2,250. A + life of 0 is ``#DIV/0!``; a negative cost or a salvage above the + cost is computed as written. Element-wise over a period-shaped + operand; the result keeps the unit the amounts share. + """ + return elementwise("SLN", _sln, [cost, salvage, life], blank=0, value_type='float') + + +def DB( + cost: Numeric, + salvage: Numeric, + life: Numeric, + period: Numeric, + month: Numeric | None = None, +) -> Variable: + """Fixed-declining-balance depreciation for ``period`` (renders + ``=DB(cost, salvage, life, period, [month])``). + + Excel's formula: the rate ``1 - (salvage / cost) ** (1 / life)`` + rounded to three decimals, the first period's charge over ``month`` + months (12 when omitted), each next period's on the remaining + balance, and in a period past ``life`` the stub over the months the + first year left. ``DB(1000000, 100000, 6, 1, 7)`` is 186,083.33. + ``period`` and ``month`` are read whole, as a cell shows them: 1.5 is + the first period, whatever the life. A cost of 0 or below, a salvage + below 0 or above the cost, a life of 0 or below, a period below 1 or + above ``life + 1``, or a month outside 1 .. 12 is ``#NUM!`` (a period + between 0 and 1 is outside Excel's domain; LibreOffice computes it as + 0). A series ``period`` — ``mo.time.index`` — makes the call a + schedule row. + """ + return elementwise("DB", _db, _written(cost, salvage, life, period, month), + blank=0, value_type='float') + + +def DDB( + cost: Numeric, + salvage: Numeric, + life: Numeric, + period: Numeric, + factor: Numeric | None = None, +) -> Variable: + """Double-declining-balance depreciation for ``period`` (renders + ``=DDB(cost, salvage, life, period, [factor])``). + + The charge at the rate ``factor / life`` (``factor`` 2 when omitted) + on the book value, never past the salvage value: + ``DDB(2400, 300, 10, 1)`` is 480, ``DDB(2400, 300, 10, 10)`` is + 22.12 (the balance stops at 300). A fractional period is charged + along the declining curve, as the spreadsheet charges it. A cost or + salvage below 0, a salvage above the cost, a factor of 0 or below, + or a period below 1 or above ``life`` is ``#NUM!``. A series + ``period`` makes the call a schedule row. + """ + return elementwise("DDB", _ddb, _written(cost, salvage, life, period, factor), + blank=0, value_type='float') + + +def SYD(cost: Numeric, salvage: Numeric, life: Numeric, per: Numeric) -> Variable: + """Sum-of-years'-digits depreciation for ``per`` (renders + ``=SYD(cost, salvage, life, per)``). + + ``(cost - salvage) * (life - per + 1) * 2 / (life * (life + 1))``: + ``SYD(30000, 7500, 10, 1)`` is 4,090.91, the tenth year 409.09. A + life of 0 or below, or a period below 1 or above ``life``, is + ``#NUM!`` — Excel's domain; LibreOffice computes the formula as + written there (0 the period after the life, negative after that). A + series ``per`` makes the call a schedule row. + """ + return elementwise("SYD", _syd, [cost, salvage, life, per], blank=0, value_type='float') + + +# The cash-flow functions take their flows as a range; a rate or a date +# argument that is one value is written as its cell. +register("IRR", shape="range") +register("NPV", shape="range") +register("XIRR", shape="range") +register("XNPV", shape="range") +register("PMT", shape="scalar") +register("FV", shape="scalar") +register("PV", shape="scalar") +# The schedule and depreciation functions are one call per period. +register("NPER", shape="elementwise") +register("IPMT", shape="elementwise") +register("PPMT", shape="elementwise") +register("RATE", shape="elementwise") +register("CUMIPMT", shape="elementwise") +register("CUMPRINC", shape="elementwise") +register("SLN", shape="elementwise") +register("DB", shape="elementwise") +register("DDB", shape="elementwise") +register("SYD", shape="elementwise") + + +# ─── Units ────────────────────────────────────────────────────────── + + +def _amounts(func: str, first: int, stop: int, *, gives: bool = True) -> Any: + """The law of a function reading amounts at ``first:stop`` of its + written arguments: one unit among them, the result's — none for a + rate or a count (``gives=False``).""" + from ..core.expr import ListExpr + + def law(args: Any, unit_of: Any) -> Any: + # A list of lines (``NPV(rate, [y1, y2])``) is read line by line. + nodes = [n for a in args[first:stop] + for n in (a.items if isinstance(a, ListExpr) else (a,))] + unit = same_unit(func, nodes, unit_of) + return unit if gives else None + return law + + +unit_law("IRR", _amounts("IRR", 0, 1, gives=False)) # (flows, guess) +unit_law("XIRR", _amounts("XIRR", 0, 1, gives=False)) # (flows, dates, guess) +unit_law("NPV", _amounts("NPV", 1, 2)) # (rate, flows) +unit_law("XNPV", _amounts("XNPV", 1, 2)) # (rate, flows, dates) +unit_law("PMT", _amounts("PMT", 2, 4)) # (rate, nper, pv, fv, type) +unit_law("FV", _amounts("FV", 2, 4)) # (rate, nper, pmt, pv, type) +unit_law("PV", _amounts("PV", 2, 4)) # (rate, nper, pmt, fv, type) +unit_law("NPER", _amounts("NPER", 1, 4, gives=False)) # (rate, pmt, pv, fv, type) +unit_law("RATE", _amounts("RATE", 1, 4, gives=False)) # (nper, pmt, pv, fv, type, guess) +unit_law("IPMT", _amounts("IPMT", 3, 5)) # (rate, per, nper, pv, fv, type) +unit_law("PPMT", _amounts("PPMT", 3, 5)) +unit_law("CUMIPMT", _amounts("CUMIPMT", 2, 3)) # (rate, nper, pv, start, end, type) +unit_law("CUMPRINC", _amounts("CUMPRINC", 2, 3)) +unit_law("SLN", _amounts("SLN", 0, 2)) # (cost, salvage, life) +unit_law("DB", _amounts("DB", 0, 2)) # (cost, salvage, life, period, month) +unit_law("DDB", _amounts("DDB", 0, 2)) # (cost, salvage, life, period, factor) +unit_law("SYD", _amounts("SYD", 0, 2)) # (cost, salvage, life, per) diff --git a/src/modeleon/functions/lag.py b/src/modeleon/functions/lag.py index b78f5f8..d253c02 100644 --- a/src/modeleon/functions/lag.py +++ b/src/modeleon/functions/lag.py @@ -22,6 +22,48 @@ from ..core.variable import Variable +def check_fill(fill: Any, spelled: str) -> None: + """A lag's fill that is a line is kept as a reference to its cell, so it + is one value assigned above the lag: a fill assigned further down, or a + series, is refused (``spelled`` is how the call is written).""" + if not isinstance(fill, Variable): + return + if fill._forward_of is not None: + raise TypeError( + f"{spelled}{fill._forward_of[1]}): the fill is assigned " + f"further down — assign it above this line." + ) + if isinstance(fill._value, list): + raise TypeError( + f"{spelled}) must be a number or a SCALAR Variable — " + f"a list fill has no single head value." + ) + + +def lag_unit(source: Variable, fill: Any, result: Variable) -> Any: + """The unit of a lag of ``source`` filled by ``fill`` — ``mo.lag``'s and + ``.shift``'s, by the law a loop container derives the row by again + (``core.loops._unit_of``). The source's unit; the fill is the series' + first cells, so a fill in a unit is in the source's, as the two values of + an ``IF`` share one (:func:`~modeleon.functions._helpers.same_unit`): + another dimension is + refused, two labels of a pure number name neither. A source read + before its line takes the fill's, and its mismatch waits for the loop + to close.""" + from ._helpers import operand_unit, same_unit, waits_on_loop + waiting = waits_on_loop(source) or waits_on_loop(result) + unit = getattr(source, "_unit", None) + if unit is None and not waiting: + return None + fill_node = [VarRef(fill)] if isinstance(fill, Variable) else [] + try: + return same_unit("lag", [VarRef(source), *fill_node], operand_unit) + except ValueError: + if not waiting: + raise + return None + + def lag( source: Variable, periods: int = 1, @@ -37,7 +79,9 @@ def lag( fill: Value for the positions the shift exposes (the first ``periods`` cells of a lag, the last of a lead). A number or a SCALAR Variable — a Variable keeps the dependency - live, so editing the fill assumption reflows the model. + live, so editing the fill assumption reflows the model. The + result is in the source's unit, and a fill in a unit is too: + one in another is refused. display_name: Optional display label for the result. Excel: each cell references the source row's cell ``periods`` @@ -64,16 +108,7 @@ def lag( raise TypeError("lag(..., periods=) must be an int") from None fill_is_var = isinstance(fill, Variable) - if fill_is_var and fill._forward_of is not None: - raise TypeError( - f"lag(..., fill={fill._forward_of[1]}): the fill is assigned " - f"further down — assign it above this line." - ) - if fill_is_var and isinstance(fill._value, list): - raise TypeError( - "lag(..., fill=) must be a number or a SCALAR Variable — " - "a list fill has no single head value." - ) + check_fill(fill, "lag(..., fill=") fill_value = fill._value if fill_is_var else fill from ..core.regrain import hole_value @@ -137,7 +172,7 @@ def _shift(values: list) -> list: result.value_type = "float" else: result.value_type = "int" - unit = getattr(source, "_unit", None) + unit = lag_unit(source, fill, result) if unit is not None: result._unit = unit diff --git a/src/modeleon/functions/logical.py b/src/modeleon/functions/logical.py new file mode 100644 index 0000000..f852178 --- /dev/null +++ b/src/modeleon/functions/logical.py @@ -0,0 +1,402 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Logical and error helpers — ``IFERROR``, ``ISERROR``, ``ISNUMBER``, +``XOR``, ``IFS``, ``SWITCH``. + +Each is one call per period through the shared :func:`elementwise` +helper: a series operand gives a series, scalars and a one-value series +broadcast as ``a + b`` does, other length mismatches are refused, tracked +operands lift per coordinate, and the cell per period is the call over +that period's cells (``=IFERROR(C4, 0)``). ``IF``, ``AND``, ``OR``, +``NOT``, ``ISBLANK`` and ``CHOOSE`` live in :mod:`conditional`. + +An operand is read as the cell the workbook holds for it: ``None`` is a +blank cell, and so is an empty text typed into a line (the workbook +writes it as an empty cell); a formula's empty text, or one given to the +call, is text. + +The error functions read the engine's error markers — ``'#DIV/0!'``, +``'#VALUE!'``, ``'#N/A'``, …, any text starting with ``#``, as +arithmetic carries one: ``IFERROR`` replaces a marker with its fallback, +``ISERROR`` names one, ``ISNUMBER`` is FALSE on one. A branch not taken +is never read, as the spreadsheet never reads it: ``IFS(a == 0, 0, True, +1 / a)`` is 0 where ``a`` is 0. ``XOR``, ``IFS`` and ``SWITCH`` came to +Excel after 2007: the file stores them as ``_xlfn.XOR`` / ``_xlfn.IFS`` +/ ``_xlfn.SWITCH``, shown without the prefix. +""" + +from __future__ import annotations + +import math +import numbers +from datetime import date, timedelta +from typing import Any, Callable, Iterable, Sequence + +from ..core.variable import Variable +from ..core.variable_ops import _to_serial +from ._helpers import elementwise, is_error, same_unit, shared_unit, unit_law +from .conditional import _blank0 +from .registry import register + +#: ``SWITCH`` without a default, when no case matches. +_NO_DEFAULT = object() + + +# ─── Reading the operands ─────────────────────────────────────────── + + +def _typed_line(arg: Any) -> bool: + # Values typed in, not computed: the workbook writes the line's + # empty text as an empty cell. + return isinstance(arg, Variable) and arg._expr is None + + +def _as_cells(pyfunc: Callable[..., Any], args: Sequence[Any]) -> Callable[..., Any]: + # ``pyfunc`` over one period's operands as their cells read: an empty + # text typed into a line is the blank cell the workbook writes. + typed = [_typed_line(a) for a in args] + if not any(typed): + return pyfunc + + def read(*values: Any) -> Any: + return pyfunc(*[None if t and v == '' else v for t, v in zip(typed, values)]) + return read + + +def _values(arg: Any) -> list[Any]: + # Every value an operand gives — a scalar, a series, a series per + # track — read as its cells. + from ..core.tracks import TrackValues + raw = arg._value if isinstance(arg, Variable) else arg + parts = list(raw.values()) if isinstance(raw, TrackValues) else [raw] + flat = [v for part in parts for v in (part if isinstance(part, list) else [part])] + return [None if v == '' else v for v in flat] if _typed_line(arg) else flat + + +def _type_of(values: Iterable[Any]) -> str: + # Typed by the first value that is neither a blank nor an error marker. + sample = next((v for v in values if v is not None and not is_error(v)), None) + if isinstance(sample, bool): + return 'bool' + if isinstance(sample, int): + return 'int' + if isinstance(sample, str): + return 'string' + if isinstance(sample, date): + return 'datetime' + return 'float' + + +def _shown(value: Any) -> Any: + # A formula that lands on a blank cell shows 0. + from ..core.tracks import TrackValues + if isinstance(value, TrackValues): + return TrackValues.lift(_shown, value) + if isinstance(value, list): + return [_blank0(v) for v in value] + return _blank0(value) + + +def _boolean(out: Variable) -> Variable: + # A TRUE / FALSE carries no unit; the formula is the readable source. + out._source_from_formula() + out._unit = None + return out + + +def _picked(out: Variable, takes: Sequence[Any], sample: list[Any]) -> Variable: + # A value picked among ``takes``, the operands the result may be: typed + # by ``sample``'s first value that is not blank and not an error, a + # picked blank shows 0, and the unit is the one ``takes`` share — a + # condition or a case compared is never the result. Two units among + # ``takes`` were refused by the function's law, as it was built. + out._source_from_formula() + out.value_type = _type_of(sample) + out._value = _shown(out._value) + out._unit = shared_unit(takes) if out.value_type in ('int', 'float') else None + return out + + +# ─── Errors ───────────────────────────────────────────────────────── + + +def _iferror(value: Any, fallback: Any) -> Any: + return fallback if is_error(value) else value + + +def IFERROR(value: Any, fallback: Any) -> Variable: + """``value``, or ``fallback`` where ``value`` is an error (renders ``=IFERROR(...)``). + + The way to catch a ``#DIV/0!`` before it cascades: ``mo.IFERROR(profit + / revenue, 0)`` is the margin where revenue is nonzero and 0 where it + is not. Both operands are read in every period (both are the cell's + precedents); a blank shows 0, as a formula landing on a blank cell + does. The result is typed by the value where it is not an error — a + text fallback does not make a row of numbers text — and keeps the + unit the operands share: ``IFERROR(cost, 0)`` is still in dollars. + """ + args = [value, fallback] + out = elementwise("IFERROR", _as_cells(_iferror, args), args, propagate_errors=False) + return _picked(out, args, _values(value) + _values(fallback)) + + +def ISERROR(value: Any) -> Variable: + """TRUE where ``value`` is an error marker (renders ``=ISERROR(...)``). + + Any error — ``#DIV/0!``, ``#VALUE!``, ``#N/A``, ``#NUM!``, … — as the + engine carries it: a text value starting with ``#``. Never an error + itself. Element-wise over a series. + """ + return _boolean(elementwise("ISERROR", is_error, [value], value_type='bool', + propagate_errors=False)) + + +def _isnumber(value: Any) -> bool: + if isinstance(value, bool): + return False + return isinstance(value, (numbers.Real, date, timedelta)) + + +def ISNUMBER(value: Any) -> Variable: + """TRUE where ``value`` is a number (renders ``=ISNUMBER(...)``). + + A date is a number (the spreadsheet stores it as a serial number), a + duration is its day count; a boolean, text (a number written as text + included), a blank and an error are not. Never an error itself. + Element-wise over a series. + """ + return _boolean(elementwise("ISNUMBER", _isnumber, [value], value_type='bool', + propagate_errors=False)) + + +# ─── XOR ──────────────────────────────────────────────────────────── + + +def _xor(*values: Any) -> Any: + # A blank or an empty text is skipped, as the spreadsheet skips such a + # cell; a number is TRUE when nonzero; nothing left is #VALUE!. + counted = [v for v in values if v is not None and v != ''] + if not counted: + return '#VALUE!' + return sum(bool(v) for v in counted) % 2 == 1 + + +def XOR(*conditions: Any) -> Variable: + """Exclusive OR — TRUE when an odd number of conditions hold (renders ``=XOR(...)``). + + Element-wise over series operands. A blank or an empty text is + skipped, as the spreadsheet skips such a cell; a number counts as TRUE + when nonzero; the first error among a period's operands is the + period's result; nothing to count is ``#VALUE!``. Text is refused + (``TypeError``): the spreadsheet skips text in a cell it references + but reads text typed into the formula as ``#VALUE!``, so no value + would match the workbook — compare it instead, ``XOR(answer == "yes", + …)``. Stored in the file as ``_xlfn.XOR(...)``. + """ + if not conditions: + raise TypeError("XOR takes at least one condition.") + for condition in conditions: + text = next((v for v in _values(condition) + if isinstance(v, str) and v and not is_error(v)), None) + if text is not None: + raise TypeError( + f"XOR counts conditions, and {text!r} is text: the spreadsheet skips " + f"text in a cell it references but reads text typed into the formula " + f"as #VALUE!, so no value would match the workbook. Compare the text " + f"instead — XOR(answer == 'yes', …)." + ) + return _boolean(elementwise("XOR", _xor, list(conditions), value_type='bool')) + + +# ─── IFS and SWITCH ───────────────────────────────────────────────── + + +def _truth(value: Any) -> Any: + # A condition as the spreadsheet reads it: an error stays one, a blank + # is FALSE, a number is TRUE when nonzero, the text TRUE or FALSE (in + # any case) is that boolean, other text — an empty one too — #VALUE!. + if is_error(value): + return value + if value is None: + return False + if isinstance(value, str): + word = value.casefold() + return word == 'true' if word in ('true', 'false') else '#VALUE!' + return bool(value) + + +def _ifs(*operands: Any) -> Any: + for condition, value in zip(operands[0::2], operands[1::2]): + truth = _truth(condition) + if is_error(truth): + return truth + if truth: + return value + return '#N/A' + + +def IFS(*pairs: Any) -> Variable: + """The value of the first condition that holds (renders ``=IFS(...)``). + + ``mo.IFS(score >= 90, "A", score >= 80, "B", True, "C")`` — condition + and value pairs, read in order; a final ``True`` is the catch-all. + No condition holding is ``#N/A``, as in the spreadsheet. A condition + that is an error makes the result that error; a value or a condition + after the first that holds is never read. A blank condition is FALSE, + a number is TRUE when nonzero, the text ``"TRUE"`` or ``"FALSE"`` is + that boolean, other text is ``#VALUE!``; a picked blank shows 0. + Element-wise over series operands. The result is typed by the first + value taken that is not blank and not an error, and keeps a unit the + values share — a condition's unit is not the result's. Stored in the + file as ``_xlfn.IFS(...)``. + """ + if not pairs or len(pairs) % 2: + raise TypeError( + f"IFS takes condition and value pairs — IFS(cond1, value1, cond2, " + f"value2, …); got {len(pairs)} operand{'s' if len(pairs) != 1 else ''}." + ) + args = list(pairs) + out = elementwise("IFS", _as_cells(_ifs, args), args, propagate_errors=False) + return _picked(out, args[1::2], _values(out)) + + +#: Numbers closer than this share of both are equal, as the spreadsheet +#: compares them — about 15 significant digits. +_EPSILON = 2.0 ** -48 +#: A double holds every whole number up to this one exactly. +_WHOLE = 2.0 ** 53 - 1 + + +def _equal_numbers(a: float, b: float) -> bool: + # Within 2**-48 of both, so a computed 0.1 * 3 is the 0.3 typed into a + # case; zero equals only zero, and two whole numbers only each other. + if a == b: + return True + if a == 0.0 or b == 0.0: + return False + d, a, b = abs(a - b), abs(a), abs(b) + if not math.isfinite(d): + return False + if all(x <= _WHOLE and x.is_integer() for x in (d, a, b)): + return False + return d < a * _EPSILON and d < b * _EPSILON + + +def _key(value: Any) -> tuple[str, Any]: + # What a cell compares as: text regardless of case, a boolean only with + # a boolean, a date or a duration by its serial number. + if isinstance(value, bool): + return ('bool', value) + if isinstance(value, str): + return ('text', value.casefold()) + serial = _to_serial(value) + if isinstance(serial, numbers.Real): + return ('number', serial) + return ('other', value) + + +_BLANK_MATCHES = (('number', 0), ('text', ''), ('bool', False)) + + +def _matches(expression: Any, case: Any) -> bool: + if expression is None or case is None: + other = case if expression is None else expression + return other is None or _key(other) in _BLANK_MATCHES + kind, left = _key(expression) + case_kind, right = _key(case) + if kind != case_kind: + return False + if kind == 'number': + return _equal_numbers(float(left), float(right)) + return bool(left == right) + + +def _switch(expression: Any, *rest: Any) -> Any: + cases, default = (rest[:-1], rest[-1]) if len(rest) % 2 else (rest, _NO_DEFAULT) + if is_error(expression): + return expression + for case, value in zip(cases[0::2], cases[1::2]): + if is_error(case): + return case + if _matches(expression, case): + return value + return '#N/A' if default is _NO_DEFAULT else default + + +def SWITCH(expression: Any, *cases: Any) -> Variable: + """The value of the first case equal to ``expression`` (renders ``=SWITCH(...)``). + + ``mo.SWITCH(inputs.scenario, "base", 0.05, "bull", 0.08, 0.03)`` — + the expression, then case and value pairs, then an optional default + (an odd number of operands after the expression); no case matching + and no default is ``#N/A``, as in the spreadsheet. A case is equal as + cells are equal: numbers to about 15 significant digits (a computed + ``0.1 * 3`` is ``0.3``), text regardless of case, a date by its serial + number, a blank as 0, as empty text and as FALSE, TRUE/FALSE only + with each other, and never across kinds (1 is not "1"). An expression + or a case that is an error makes the result that error; a case after + the match is never read; a picked blank shows 0. Element-wise over + series operands. The result is typed by the first value taken that is + not blank and not an error, and keeps a unit the values share — the + expression and the cases are compared, never taken. Stored in the + file as ``_xlfn.SWITCH(...)``. + """ + if len(cases) < 2: + raise TypeError( + f"SWITCH takes an expression, then case and value pairs — " + f"SWITCH(expr, case1, value1, …, [default]); got " + f"{len(cases)} operand{'s' if len(cases) != 1 else ''} after the expression." + ) + args = [expression, *cases] + takes = [*cases[1::2], *(cases[-1:] if len(cases) % 2 else ())] + out = elementwise("SWITCH", _as_cells(_switch, args), args, propagate_errors=False) + return _picked(out, takes, _values(out)) + + +register("IFERROR", shape="elementwise") +register("ISERROR", shape="elementwise") +register("ISNUMBER", shape="elementwise") +register("XOR", shape="elementwise", file_prefix="_xlfn.") +register("IFS", shape="elementwise", file_prefix="_xlfn.") +register("SWITCH", shape="elementwise", file_prefix="_xlfn.") + + +def _ifs_unit(args: Any, unit_of: Callable[[Any], Any]) -> Any: + """``IFS`` takes the value after each condition.""" + return same_unit("IFS", args[1::2], unit_of) + + +def _switch_unit(args: Any, unit_of: Callable[[Any], Any]) -> Any: + """``SWITCH`` compares its expression with each case, as ``==`` does, + and takes the value after each case, or the default.""" + pairs = (len(args) - 1) // 2 + _refuse_a_case_in_another_unit(args[0], args[1:1 + 2 * pairs:2], unit_of) + default = list(args[-1:]) if (len(args) - 1) % 2 else [] + return same_unit("SWITCH", [*args[2::2], *default], unit_of) + + +def _refuse_a_case_in_another_unit(expression: Any, cases: Sequence[Any], + unit_of: Callable[[Any], Any]) -> None: + """A case in another unit than the expression is refused, as comparing + the two lines is; a number or an unlabelled line goes with any.""" + from ..core.expr import VarRef + from ..core.unit import UNSETTLED + from ..core.variable_ops import _incompatible_units, _line_name + + def named(node: Any) -> str: + return _line_name(node.var) if isinstance(node, VarRef) else "a formula" + + own = unit_of(expression) + if own is None or own is UNSETTLED: + return + for case in cases: + unit = unit_of(case) + if unit is not None and unit is not UNSETTLED and unit != own: + detail = _incompatible_units(own, unit).split(": ", 1)[1] + raise ValueError( + f"mo.SWITCH compares {named(expression)} with the case " + f"{named(case)}; they are in {detail}" + ) + + +unit_law("IFS", _ifs_unit) +unit_law("SWITCH", _switch_unit) diff --git a/src/modeleon/functions/lookup.py b/src/modeleon/functions/lookup.py new file mode 100644 index 0000000..3c3736d --- /dev/null +++ b/src/modeleon/functions/lookup.py @@ -0,0 +1,962 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Lookup and reference — ``MATCH``, ``INDEX``, ``XLOOKUP``, ``VLOOKUP``, +``HLOOKUP``. + +Each finds a cell in a range or a block and is written as the +spreadsheet writes it: ``=INDEX(Inputs!B3:D3, MATCH(C5, Inputs!B2:D2, +0))``, ``=_xlfn.XLOOKUP(C5, Inputs!B2:D2, Inputs!B3:D3)``, +``=VLOOKUP("Beta", Deals!A3:C5, 3, FALSE)``. A range argument is what +the conditional aggregates take (:mod:`criteria`): a row, written as +the range of its cells, or a list of scalar lines whose cells stand +one under another, written as a vertical range. A block argument is +two-dimensional: a container of records laid out as a table +(``orient='records'``) — one row per record, the record's label first +and its scalar fields after it, in the order the table has them, +``Deals!A3:C5`` — or a list of rows laid out one under another +(``[years, inflation]`` → ``Inputs!B2:D3``). Cells that form no +rectangle (records laid out as sections, a table with a unit column, +rows with a totals column between their cells) cannot be one range: +the cell carries the computed value and ``check_compat`` lists it. + +The key may be a series: the result is a series, one lookup per +period, and the cell per period is the call over that period's key +cell and the whole range. A position (``INDEX``'s, ``VLOOKUP``'s +column) may be a series too, and so may ``XLOOKUP``'s ``if_not_found``; +series on other time windows are read by date, as ``a + b`` reads them +(each continued by its own ``extend=`` rule, the result on the +combined window), and lines along keys key by key. A mode +(``match_type``, ``range_lookup``, ``match_mode``, ``search_mode``) is +one value, typed in. + +The edges, as the spreadsheet reads them. Text matches case- +insensitively, the whole cell, with the wildcards ``*`` and ``?`` +(``~`` taking the next character literally) only where Excel reads +them: an exact ``MATCH`` (``match_type`` 0), an exact ``VLOOKUP`` / +``HLOOKUP`` (``range_lookup`` FALSE) and ``XLOOKUP`` with +``match_mode`` 2. A number never matches text (``'10'`` is not 10 +here, unlike in a ``SUMIF`` criterion), a boolean matches a boolean only, a date +is its serial. A blank cell matches nothing, and a key read from a +blank cell finds nothing (``#N/A``); ``None`` typed as the key is +refused — it has no cell, and the formula would write it as 0. A key +that is an error marker gives that error; an error among the cells +searched is skipped; the cell found carries its own content — an +error marker stays one, a blank reads as 0, as a formula on a blank +cell does. The approximate modes — ``MATCH`` with ``match_type`` 1 or +-1, ``VLOOKUP`` / ``HLOOKUP`` with ``range_lookup`` TRUE (the default, +as in Excel) — need the cells sorted, ascending for 1 / TRUE and +descending for -1, as Excel requires: the engine takes the last cell +not above (not below) the key, which is Excel's answer over sorted +cells and nobody's over unsorted ones. ``XLOOKUP``'s ``match_mode`` +-1 / 1 need no order: the nearest cell below / above the key, the +first among equals (``search_mode`` -1: the last); its binary searches, +``search_mode`` 2 and -2, are refused — the engine reads every cell. + +A position is read as the spreadsheet reads a number: a boolean is 1 +or 0, a date its serial, text that reads as a number that number, a +fraction is truncated. A position 0 — the spreadsheet's whole row or +column — is ``#VALUE!``, as is a negative one, a blank or other text; +past the range, ``#REF!``. One case differs: in a line of periods, +for a 0 read from a cell the sheet takes the range's cell in the +period's own column (it intersects the whole row with the column), +where the engine gives ``#VALUE!``. ``VLOOKUP`` and ``HLOOKUP`` check +the column or row number before they search, so its error wins over +the key's: past the block is ``#REF!`` whether or not the key is +found. + +The result is typed by the line the cell comes from and carries its +unit — never the keys' — and ``MATCH``, a position, has none; a result +that can come from cells of different lines is typed by them when +they agree, else by the value found, and carries a unit only when +every one of them has it. Cells it can come from in two units are +refused, as ``CHOOSE`` refuses them, and so is an ``if_not_found`` in +another unit than the cells returned; the key is compared with the +cells searched as ``==`` compares two lines, so a key in another unit +is refused too. A number or a line without a unit goes with any. +Empty text typed into a line is a blank cell, as the workbook holds it. +``XLOOKUP`` is in Excel 2021 and Microsoft 365, not in Excel 2019: the +file stores it as +``_xlfn.XLOOKUP``; a mode given without ``if_not_found`` is written +after ``NA()``, Excel's own not-found value, as the argument before +it. + +``XLOOKUP(key, line)`` over a line that runs along keys of its own +(``mo.Variable({2026: 0.05, 2027: 0.04})``) looks the key up among the +line's keys: the keys are typed into the call (``XLOOKUP(year, [2026, +2027], inflation)``) and have no cells the workbook can name, so the +cell carries the computed value. + +LibreOffice departs from Excel in three places, and the engine keeps +Excel's rule: there a boolean is a number (``MATCH(TRUE, …)`` finds a +1), text that reads as a number matches the number (``MATCH("2027", +years, 0)`` finds 2027), and a column or row number past the block is +``#VALUE!``, not ``#REF!``. + +Not covered: a block with a record that lacks a field (the table +leaves the cell empty; no cell can be named for it), ``INDEX`` over a +one-dimensional range with two positions, a key with tracks (slice a +coordinate first), and ``MATCH`` or ``XLOOKUP`` over a block (one- +dimensional in Excel too). +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from datetime import date, timedelta +from typing import Any, Callable, Sequence + +from ..core.expr import Expr, FuncCall, ListExpr, Literal, VarRef +from ..core.multi_variable import MultiVariableBase +from ..core.shape import keys_along, reject_axised +from ..core.unit import unit_key +from ..core.variable import Variable +from ..core.variable_ops import _to_serial +from ._helpers import ( + _per_period, align_keyed, align_located, is_error, make_func_var, placed, src, +) +from .conditional import _blank0, _value_type_of +from .criteria import ( + _Range, _as_number, _cells, _checked, _index, _label, _range, _same_length, _wildcard, +) +from .registry import register + +#: An optional argument the call did not give — left out of the formula too. +_OMITTED: Any = object() + +# ─── One cell against the key ─────────────────────────────────────── + + +def _kind(value: Any) -> str | None: + """What a cell can be matched as: ``'number'`` (a date and a + duration among them), ``'text'``, ``'bool'``; None for a blank or an + error marker, which match nothing.""" + if isinstance(value, bool): + return 'bool' + if isinstance(value, (int, float, date, timedelta)): + return 'number' + if isinstance(value, str): + return None if is_error(value) else 'text' + return None + + +def _ordered(value: Any) -> Any: + """The value as it compares within its kind: text case-insensitively, + a date as its serial.""" + if isinstance(value, str): + return value.casefold() + if isinstance(value, (date, timedelta)): + return _to_serial(value) + return value + + +def _exact_test(key: Any, wildcard: bool) -> Callable[[Any], bool]: + kind = _kind(key) + if kind is None: + return lambda v: False + if kind == 'text': + if wildcard: + pattern = _wildcard(key) + return lambda v: _kind(v) == 'text' and pattern.fullmatch(v) is not None + folded = key.casefold() + return lambda v: _kind(v) == 'text' and v.casefold() == folded + target = _ordered(key) + return lambda v: _kind(v) == kind and _ordered(v) == target + + +def _exact(cells: list[Any], key: Any, *, wildcard: bool, last: bool = False) -> int | None: + """The position of the cell equal to the key — the first, or the + last — or None.""" + test = _exact_test(key, wildcard) + hits = [i for i, v in enumerate(cells) if test(v)] + if not hits: + return None + return hits[-1] if last else hits[0] + + +# ``_exact`` over a range read once: the cells its index finds for the key, +# each read by the same test; every cell where the index finds none, or for +# a pattern, which no index answers. +def _exact_in(rng: _Range, key: Any, *, wildcard: bool, last: bool = False) -> int | None: + kind = _kind(key) + if kind is None: + return None + pattern = wildcard and kind == 'text' and any(ch in '*?~' for ch in key) + index = None if pattern else _index(rng) + near: list[int] | None = None + if index is not None: + if kind == 'bool': + near = index.bools[key] + elif kind == 'number': + near = index.exact.get(_ordered(key), []) + elif wildcard: + near = index.like(key) + else: + near = index.folded.get(key.casefold(), []) + if near is None: + return _exact(rng.values, key, wildcard=wildcard, last=last) + test = _exact_test(key, wildcard) + hits = [i for i in sorted(set(near)) if test(rng.values[i])] + if not hits: + return None + return hits[-1] if last else hits[0] + + +def _nearest(cells: list[Any], key: Any, *, below: bool, last: bool = False) -> int | None: + """The position of the cell nearest the key on one side of it — the + largest not above it, or the smallest not below it — among the + cells of the key's kind, the first among equals or the last; None + when no cell is on that side.""" + kind = _kind(key) + target = _ordered(key) + best: Any = None + best_at: int | None = None + for i, v in enumerate(cells): + if kind is None or _kind(v) != kind: + continue + o = _ordered(v) + if (o > target) if below else (o < target): + continue + closer = best is None or ((o > best) if below else (o < best)) + if closer or (o == best and last): + best, best_at = o, i + return best_at + + +def _sorted_match(cells: list[Any], key: Any, *, ascending: bool) -> int | None: + """``MATCH`` 1 / -1 and the approximate ``VLOOKUP`` / ``HLOOKUP``: over + cells sorted ascending (descending), the last cell not above (not + below) the key, among the cells of the key's kind; None when the + first such cell is already past it.""" + kind = _kind(key) + target = _ordered(key) + found = None + for i, v in enumerate(cells): + if kind is None or _kind(v) != kind: + continue + o = _ordered(v) + if (o <= target) if ascending else (o >= target): + found = i + return found + + +def _position(n: Any, size: int) -> int | str: + """A 1-based position typed or read from a cell as a 0-based index, + read as the spreadsheet reads a number: an error marker stays one; a + boolean is 1 or 0, a date its serial, text that reads as a number + that number; 0, a negative number, a blank or other text is + ``#VALUE!``; past ``size`` is ``#REF!``; a fraction is truncated.""" + if is_error(n): + return n + if isinstance(n, bool): + n = int(n) + elif isinstance(n, (date, timedelta)): + n = _to_serial(n) + elif isinstance(n, str): + n = _as_number(n) + if not isinstance(n, (int, float)) or (isinstance(n, float) and not math.isfinite(n)): + return '#VALUE!' + k = int(n) + if k < 1: + return '#VALUE!' + if k > size: + return '#REF!' + return k - 1 + + +# ─── The arguments ────────────────────────────────────────────────── + + +@dataclass +class _Block: + """A two-dimensional range: the cells' values by row, the line each + cell comes from (None for a record's label or a typed value), the + AST node and the name in the source. ``labelled`` when the first + column holds the records' labels.""" + + rows: list[list[Any]] + lines: list[list[Variable | None]] + node: Expr + text: str + labelled: bool = False + + @property + def height(self) -> int: + return len(self.rows) + + @property + def width(self) -> int: + return len(self.rows[0]) + + +def _series_value(raw: Any) -> bool: + return isinstance(raw, list) or hasattr(raw, 'roles') + + +def _refuse_tracks(var: Variable, where: str) -> None: + if hasattr(var._value, 'roles'): + raise ValueError( + f"{where}(): {src(var)} has tracks — slice a coordinate first " + f"(.at(track='...'))." + ) + + +def _is_block(arg: Any) -> bool: + """A container of records, or a list with a row among its items + (list-valued lines, or typed lists), is a block; a line or a list of + scalar lines is a range.""" + if isinstance(arg, MultiVariableBase): + return True + if isinstance(arg, (list, tuple)) and arg: + return any( + isinstance(item, list) + or (isinstance(item, Variable) and _series_value(item._value)) + for item in arg + ) + return False + + +def _records_block(mv: MultiVariableBase, where: str) -> _Block: + """The table a container of records is laid out as: one row per + record — its label, then its scalar fields in first-appearance order + across the records, a field a record lacks left blank; a per-period + field is a block of its own below the table, not a column of it.""" + from ..compile.excel.view import ExcelView + records = [ + rec for rec in mv._components.values() + if isinstance(rec, MultiVariableBase) and not isinstance(rec, ExcelView) + ] + name = getattr(mv, 'python_name', None) or mv.display_name + if not records: + raise ValueError( + f"{where}(): {name} holds no records — a block is a container of " + f"records laid out as a table (orient='records'), or a list of rows." + ) + fields: list[str] = [] + seen: set[str] = set() + for rec in records: + for fname, f in rec._components.items(): + if not isinstance(f, Variable) or fname in seen: + continue + seen.add(fname) + if not _series_value(f._value): + fields.append(fname) + if not fields: + raise ValueError( + f"{where}(): the records of {name} have no scalar fields — a " + f"per-period field is a block of its own below the table." + ) + rows: list[list[Any]] = [] + lines: list[list[Variable | None]] = [] + items: list[Expr] = [] + for rec in records: + row: list[Any] = [rec.display_name] + row_lines: list[Variable | None] = [None] + nodes: list[Expr] = [Literal(rec.display_name)] + for fname in fields: + f = rec._components.get(fname) + if isinstance(f, Variable): + reject_axised(f, where) + row.append(_cells(f, f._value)[0]) + row_lines.append(f) + nodes.append(VarRef(f)) + else: + row.append(None) + row_lines.append(None) + nodes.append(Literal(None)) + rows.append(row) + lines.append(row_lines) + items.append(ListExpr(nodes)) + return _Block(rows, lines, ListExpr(items), name, labelled=True) + + +def _rows_block(arg: Sequence[Any], where: str) -> _Block: + """A list of rows — list-valued lines, or typed lists — of one length.""" + rows: list[list[Any]] = [] + lines: list[list[Variable | None]] = [] + items: list[Expr] = [] + labels: list[str] = [] + for item in arg: + if isinstance(item, Variable): + reject_axised(item, where) + _refuse_tracks(item, where) + if not isinstance(item._value, list): + raise ValueError( + f"{where}(): a block is a list of rows; {_label(item)} is one " + f"cell. A list of scalar lines is a range — one position " + f"addresses it (mo.INDEX(lines, k))." + ) + cells = _cells(item, item._value) + rows.append(cells) + lines.append([item] * len(cells)) + items.append(VarRef(item)) + labels.append(_label(item)) + elif isinstance(item, list): + rows.append(list(item)) + lines.append([None] * len(item)) + items.append(Literal(list(item))) + labels.append(repr(item)) + else: + raise TypeError( + f"{where}(): a block is a list of rows (list-valued lines); got " + f"{type(item).__name__} among them." + ) + widths = {len(row) for row in rows} + if len(widths) != 1: + raise ValueError( + f"{where}(): the rows of a block have the same number of cells; got " + f"{', '.join(f'{label} with {len(row)}' for label, row in zip(labels, rows))}." + ) + return _Block(rows, lines, ListExpr(items), f"[{', '.join(labels)}]") + + +def _block(arg: Any, where: str) -> _Block: + if isinstance(arg, MultiVariableBase): + return _records_block(arg, where) + if isinstance(arg, (list, tuple)) and arg: + return _rows_block(arg, where) + raise TypeError( + f"{where}(): a block is a container of records laid out as a table, or a " + f"list of rows; got {type(arg).__name__}." + ) + + +def _key(value: Any, where: str) -> tuple[Any, Expr]: + """The key, or a position: a scalar line, a series, or a typed value.""" + if isinstance(value, Variable): + reject_axised(value, where) + _refuse_tracks(value, where) + return value._value, VarRef(value) + if isinstance(value, (dict, set, tuple, MultiVariableBase)): + raise TypeError( + f"{where}(): a key is one value, or a line (one lookup per period); " + f"got {type(value).__name__}." + ) + return value, Literal(value) + + +def _lookup_value(value: Any, where: str) -> tuple[Any, Expr]: + """The value looked up — never None typed into the call, which has no + cell: the formula would write it as 0 and look 0 up.""" + raw, node = _key(value, where) + if isinstance(value, Variable): + return raw, node + if any(v is None for v in (raw if isinstance(raw, list) else [raw])): + what = "is None" if raw is None else f"{src(value)} holds None" + raise ValueError( + f"{where}(): the key {what}, and None has no cell — the formula would " + f"write it as 0 and look 0 up. Type the value to look up, or pass a " + f"line whose cells may be left blank: a blank cell finds nothing (#N/A)." + ) + return raw, node + + +def _mode(value: Any, where: str, name: str, allowed: tuple[int, ...], default: int) -> int: + """A mode typed into the call, or its default when omitted.""" + if value is _OMITTED: + return default + if isinstance(value, bool) or not isinstance(value, int) or value not in allowed: + raise ValueError( + f"{where}(): {name} is one of {', '.join(str(a) for a in allowed)}, typed " + f"in; got {value!r}." + ) + return value + + +def _flag(value: Any, where: str) -> bool: + """``range_lookup``: TRUE (approximate) unless given as FALSE / 0.""" + if value is _OMITTED: + return True + if isinstance(value, (bool, int)): + return bool(value) + raise ValueError( + f"{where}(): range_lookup is TRUE or FALSE, typed in; got {value!r}." + ) + + +def _span(n: Any, size: int) -> list[int]: + """The indexes a position can address: the one it holds, or every one + for a series, a position not given (None) or one the range lacks.""" + at = None if isinstance(n, list) else _position(n, size) + return [at] if isinstance(at, int) else list(range(size)) + + +def _type_of(value: Any) -> str: + """The value type a cell's content gives; a blank or an error marker + reads as a number.""" + if isinstance(value, date): + return 'datetime' + if value is None or is_error(value): + return 'float' + return _value_type_of(value) + + +def _picked(block: _Block, r: Any, c: Any, value: Any) -> tuple[str, Any]: + """The value type and unit of a block pick: those of the cells it can + come from — the one cell when both positions are typed in, a column's + or a row's when one is, every cell otherwise — when the cells share + them (a record's label is text, with no unit; a field a record lacks + counts for neither); else the type of the value found, a series as a + number, and no unit.""" + types: set[str] = set() + units: set[Any] = set() + unit: Any = None + for i in _span(r, block.height): + for j in _span(c, block.width): + line, cell = block.lines[i][j], block.rows[i][j] + if line is None and cell is None: + continue + if line is None: + types.add('string' if block.labelled and j == 0 else _type_of(cell)) + units.add(None) + continue + types.add(line.value_type) + units.add(None if line._unit is None else unit_key(line._unit)) + unit = line._unit if unit is None else unit + if len(types) == 1: + value_type = types.pop() + else: + value_type = 'float' if isinstance(value, list) else _type_of(value) + return value_type, unit if len(units) == 1 else None + + +def _read(where: str, lines: Sequence[Any], + raws: list[Any]) -> tuple[list[Any], Any, list[Any] | None]: + """The per-period operands' values read as ``a + b`` reads them — + series on other time windows by date, lines along keys key by key — + with the combined window and the keys the result runs on.""" + raws, keys = align_keyed(where, lines, raws) + raws, window = align_located(lines, raws) + return raws, window, keys + + +def _result(func: str, nodes: list[Expr], texts: list[str], value: Any, + value_type: str, unit: Any) -> Variable: + """The cell with its formula: a series when the key or a position is + one; the unit is the returned range's, never the keys' (a position + has none).""" + with _checked(nodes): + out = make_func_var( + func, nodes, value, value_type, 'list' if isinstance(value, list) else 'scalar', + source_code=f"{func}({', '.join(texts)})", + ) + out._unit = unit + return out + + +# ─── The functions ────────────────────────────────────────────────── + + +def MATCH(lookup_value: Any, lookup_range: Any, match_type: Any = _OMITTED) -> Variable: + """The 1-based position of ``lookup_value`` in ``lookup_range`` + (renders ``=MATCH(...)``): ``match_type`` 0 is the first exact match + (text with wildcards), 1 (the default) the last cell not above the + key over cells sorted ascending, -1 the last cell not below it over + cells sorted descending; ``#N/A`` when none. + + Example:: + + at = mo.MATCH(year, inputs.years, 0) # =MATCH(C5, Inputs!B2:D2, 0) + rate = mo.INDEX(inputs.inflation, at) # =INDEX(Inputs!B3:D3, MATCH(...)) + """ + where = "MATCH" + key_raw, key_node = _lookup_value(lookup_value, where) + rng = _range(lookup_range, where) + mode = _mode(match_type, where, "match_type", (1, 0, -1), 1) + + def one(values: list[Any]) -> Any: + key = values[0] + if is_error(key): + return key + if key is None: + return '#N/A' + if mode == 0: + at = _exact_in(rng, key, wildcard=True) + else: + at = _sorted_match(rng.values, key, ascending=mode == 1) + return '#N/A' if at is None else at + 1 + + nodes = [key_node, rng.node] + texts = [src(lookup_value), rng.text] + if match_type is not _OMITTED: + nodes.append(Literal(mode)) + texts.append(str(mode)) + (key_raw,), window, keys = _read(where, [lookup_value], [key_raw]) + return placed(_result(where, nodes, texts, _per_period([key_raw], one), 'int', None), + window, keys) + + +def INDEX(range_or_block: Any, row_num: Any, column_num: Any = _OMITTED) -> Variable: + """The cell at a 1-based position of a range, or at a row and a + column of a block (renders ``=INDEX(...)``): ``mo.INDEX(rates, 2)``, + ``mo.INDEX(deals, 2, 3)`` — the third column of the second record, + the label column first; ``mo.INDEX([years, rates], 2, k)``.""" + where = "INDEX" + if _is_block(range_or_block): + if column_num is _OMITTED: + raise ValueError( + f"{where}() over a block takes a row and a column number: " + f"mo.INDEX(table, 2, 3)." + ) + block = _block(range_or_block, where) + r_raw, r_node = _key(row_num, where) + c_raw, c_node = _key(column_num, where) + + def pick(values: list[Any]) -> Any: + r, c = values + at = _position(r, block.height) + if isinstance(at, str): + return at + col = _position(c, block.width) + if isinstance(col, str): + return col + return _blank0(block.rows[at][col]) + + (r_raw, c_raw), window, keys = _read(where, [row_num, column_num], [r_raw, c_raw]) + value = _per_period([r_raw, c_raw], pick) + value_type, unit = _picked(block, r_raw, c_raw, value) + return placed(_result(where, [block.node, r_node, c_node], + [block.text, src(row_num), src(column_num)], value, value_type, + unit), window, keys) + if column_num is not _OMITTED: + raise ValueError( + f"{where}() over a range takes one position (mo.INDEX(row, k)); a row " + f"and a column number address a block — a table of records, or a " + f"list of rows." + ) + rng = _range(range_or_block, where) + k_raw, k_node = _key(row_num, where) + + def one(values: list[Any]) -> Any: + at = _position(values[0], len(rng.values)) + return at if isinstance(at, str) else _blank0(rng.values[at]) + + (k_raw,), window, keys = _read(where, [row_num], [k_raw]) + return placed(_result(where, [rng.node, k_node], [rng.text, src(row_num)], + _per_period([k_raw], one), rng.value_type or 'float', rng.unit), + window, keys) + + +def XLOOKUP(lookup_value: Any, lookup_range: Any, return_range: Any = None, + if_not_found: Any = _OMITTED, match_mode: Any = _OMITTED, + search_mode: Any = _OMITTED) -> Variable: + """The cell of ``return_range`` at the position of ``lookup_value`` + in ``lookup_range`` (renders ``=XLOOKUP(...)``, stored as + ``_xlfn.XLOOKUP``): an exact match unless ``match_mode`` is -1 (or + the next smaller), 1 (or the next larger) or 2 (wildcards); the + first found, or the last with ``search_mode`` -1; ``if_not_found`` + in place of ``#N/A``. Over a line that runs along keys of its own, + ``mo.XLOOKUP(year, inflation)`` looks the key up among its keys + (``if_not_found`` and the modes by keyword there). + + Example:: + + rate = mo.XLOOKUP(year, inputs.years, inputs.inflation) + # =_xlfn.XLOOKUP(C5, Inputs!B2:D2, Inputs!B3:D3) + """ + where = "XLOOKUP" + key_raw, key_node = _lookup_value(lookup_value, where) + if return_range is None: + keys = keys_along(lookup_range) if isinstance(lookup_range, Variable) else None + if keys is None: + raise ValueError( + f"{where}() takes the key, the cells to search and the cells to " + f"return: mo.XLOOKUP(year, years, rates). Over one line, that line " + f"runs along keys of its own (mo.Variable({{2026: 0.05, ...}})) and " + f"its keys are searched." + ) + search = _Range(list(keys), Literal(list(keys)), repr(list(keys)), None, None) + back = _Range(_cells(lookup_range, lookup_range._value), VarRef(lookup_range), + src(lookup_range), getattr(lookup_range, '_unit', None), + lookup_range.value_type) + else: + search = _range(lookup_range, where) + back = _range(return_range, where) + _same_length(where, search, back) + mode = _mode(match_mode, where, "match_mode", (0, -1, 1, 2), 0) + if type(search_mode) is int and abs(search_mode) == 2: + raise ValueError( + f"{where}(): search_mode {search_mode} is a binary search over sorted " + f"cells, which the engine does not do — it reads every cell. Use 1 for " + f"the first match or -1 for the last." + ) + order = _mode(search_mode, where, "search_mode", (1, -1), 1) + last = order < 0 + fallback = None if if_not_found is _OMITTED else _key(if_not_found, where) + + def one(values: list[Any]) -> Any: + key = values[0] + if is_error(key): + return key + at = None + if key is not None: + if mode in (0, 2): + at = _exact_in(search, key, wildcard=mode == 2, last=last) + else: + at = _nearest(search.values, key, below=mode == -1, last=last) + if at is None: + return '#N/A' if fallback is None else _blank0(values[1]) + return _blank0(back.values[at]) + + raws = [key_raw] if fallback is None else [key_raw, fallback[0]] + lines = [lookup_value] if fallback is None else [lookup_value, if_not_found] + raws, window, keys = _read(where, lines, raws) + nodes = [key_node, search.node, back.node] + texts = [src(lookup_value), search.text, back.text] + if fallback is not None: + nodes.append(fallback[1]) + texts.append(src(if_not_found)) + elif match_mode is not _OMITTED or search_mode is not _OMITTED: + nodes.append(FuncCall('NA', [])) + if match_mode is not _OMITTED or search_mode is not _OMITTED: + nodes.append(Literal(mode)) + texts.append(f"match_mode={mode}") + if search_mode is not _OMITTED: + nodes.append(Literal(order)) + texts.append(f"search_mode={order}") + return placed(_result(where, nodes, texts, _per_period(raws, one), + back.value_type or 'float', back.unit), window, keys) + + +def VLOOKUP(lookup_value: Any, block: Any, col_index_num: Any, + range_lookup: Any = _OMITTED) -> Variable: + """The cell in column ``col_index_num`` of the row of ``block`` whose + first column holds ``lookup_value`` (renders ``=VLOOKUP(...)``): over + a table of records the first column is the record's label; an + approximate match over a sorted first column unless ``range_lookup`` + is FALSE, as in Excel. + + Example:: + + amount = mo.VLOOKUP("Beta", deals, 3, False) # =VLOOKUP("Beta", Deals!A3:C5, 3, FALSE) + """ + where = "VLOOKUP" + key_raw, key_node = _lookup_value(lookup_value, where) + table = _block(block, where) + c_raw, c_node = _key(col_index_num, where) + approx = _flag(range_lookup, where) + first = [row[0] for row in table.rows] + + def one(values: list[Any]) -> Any: + key, c = values + # The column number is checked before the key is searched, as the + # spreadsheet checks an argument: its error wins over the key's. + col = _position(c, table.width) + if isinstance(col, str): + return col + if is_error(key): + return key + if key is None: + return '#N/A' + at = (_sorted_match(first, key, ascending=True) if approx + else _exact(first, key, wildcard=True)) + return '#N/A' if at is None else _blank0(table.rows[at][col]) + + (key_raw, c_raw), window, keys = _read(where, [lookup_value, col_index_num], + [key_raw, c_raw]) + value = _per_period([key_raw, c_raw], one) + value_type, unit = _picked(table, None, c_raw, value) + nodes = [key_node, table.node, c_node] + texts = [src(lookup_value), table.text, src(col_index_num)] + if range_lookup is not _OMITTED: + nodes.append(Literal(approx)) + texts.append(str(approx)) + return placed(_result(where, nodes, texts, value, value_type, unit), window, keys) + + +def HLOOKUP(lookup_value: Any, block: Any, row_index_num: Any, + range_lookup: Any = _OMITTED) -> Variable: + """The cell in row ``row_index_num`` of the column of ``block`` whose + first row holds ``lookup_value`` (renders ``=HLOOKUP(...)``): over a + list of rows, ``mo.HLOOKUP(year, [years, inflation], 2)``; an + approximate match over a sorted first row unless ``range_lookup`` + is FALSE, as in Excel.""" + where = "HLOOKUP" + key_raw, key_node = _lookup_value(lookup_value, where) + table = _block(block, where) + r_raw, r_node = _key(row_index_num, where) + approx = _flag(range_lookup, where) + first = table.rows[0] + + def one(values: list[Any]) -> Any: + key, r = values + # The row number is checked before the key is searched, as the + # spreadsheet checks an argument: its error wins over the key's. + row = _position(r, table.height) + if isinstance(row, str): + return row + if is_error(key): + return key + if key is None: + return '#N/A' + at = (_sorted_match(first, key, ascending=True) if approx + else _exact(first, key, wildcard=True)) + return '#N/A' if at is None else _blank0(table.rows[row][at]) + + (key_raw, r_raw), window, keys = _read(where, [lookup_value, row_index_num], + [key_raw, r_raw]) + value = _per_period([key_raw, r_raw], one) + value_type, unit = _picked(table, r_raw, None, value) + nodes = [key_node, table.node, r_node] + texts = [src(lookup_value), table.text, src(row_index_num)] + if range_lookup is not _OMITTED: + nodes.append(Literal(approx)) + texts.append(str(approx)) + return placed(_result(where, nodes, texts, value, value_type, unit), window, keys) + + +register("MATCH", shape="range", range_args=(1,)) +register("INDEX", shape="range", range_args=(0,)) +register("XLOOKUP", shape="range", range_args=(1, 2), file_prefix="_xlfn.") +register("VLOOKUP", shape="range", range_args=(1,)) +register("HLOOKUP", shape="range", range_args=(1,)) +# Written by the renderer as given: ``NA()``, the not-found value an +# ``XLOOKUP`` with a mode and no ``if_not_found`` is written with. +register("NA", shape="scalar", bare=True) + + +# ─── Units ────────────────────────────────────────────────────────── + +# A lookup compares its key with the cells it searches, as ``==`` compares +# two lines, and takes one of the cells it returns, as ``CHOOSE`` takes one +# of its values: a line searched in another unit than the key is refused, +# and so are cells to return in two units, or an ``if_not_found`` in +# another unit than theirs; a number or an unlabelled line goes with any. +# Each law gives the unit the call is built with (``_result``) — the +# returned cells', never the key's; ``MATCH``, a position, has none. + +from ._helpers import same_unit, unit_law # noqa: E402 - the laws below + + +def _key_against(func: str, key: Any, searched: Sequence[Any], + unit_of: Callable[[Any], Any]) -> None: + """The key compared with each line among the cells searched: a line in + another unit is refused, as comparing the two lines is.""" + from ..core.unit import UNSETTLED + from ..core.variable_ops import _incompatible_units + from .criteria import _named + + own = unit_of(key) + if own is None or own is UNSETTLED: + return + for line in searched: + unit = unit_of(line) + if unit is not None and unit is not UNSETTLED and unit != own: + detail = _incompatible_units(own, unit).split(": ", 1)[1] + raise ValueError( + f"mo.{func} compares the key {_named(key)} with {_named(line)}; " + f"they are in {detail}" + ) + + +def _rows_of(block: Any) -> list[Any]: + """A block's rows as its node holds them: a record (its label, then its + fields), a row line, or a list typed in.""" + return list(block.items) if isinstance(block, ListExpr) else [] + + +def _pick_unit(func: str, block: Any, r: Any, c: Any, unit_of: Callable[[Any], Any]) -> Any: + """The unit of a pick from a block, as ``_picked`` gives it — that of the + cells the positions can address (``r`` / ``c`` a position's node, None + for any) when every one of them has it, else none — with cells in two + units refused, as ``CHOOSE`` refuses them; unless a position that is + one value is one the block lacks, when the call is an error in every + period and takes no cell.""" + from ..core.unit import UNSETTLED + + def held(node: Any) -> Any: + return (node.value if isinstance(node, Literal) + else node.var._value if isinstance(node, VarRef) else []) + + rows = _rows_of(block) + if not rows: + return None + top = rows[0].items if isinstance(rows[0], ListExpr) else held(rows[0]) + spans: list[Any] = [] + erring = False + for node, size in ((r, len(rows)), (c, len(top) if isinstance(top, list) else 1)): + n = [] if node is None else held(node) + spans.append(_span(n, size)) + erring = erring or not (isinstance(n, list) or isinstance(_position(n, size), int)) + lines: list[Any] = [] + seen: set[int] = set() + bare = False # a label or a value typed in: no unit + for i in spans[0]: + for j in spans[1]: + cell = rows[i].items[j] if isinstance(rows[i], ListExpr) else rows[i] + if isinstance(cell, VarRef): + if id(cell.var) not in seen: + seen.add(id(cell.var)) + lines.append(cell) + elif isinstance(cell, Literal): + value = cell.value[j] if isinstance(cell.value, list) else cell.value + bare = bare or value is not None + if not erring: + same_unit(func, lines, unit_of) + units = [unit_of(n) for n in lines] + if any(u is UNSETTLED for u in units): + return UNSETTLED + keys = {None if u is None else unit_key(u) for u in units} | ({None} if bare else set()) + return next((u for u in units if u is not None), None) if len(keys) == 1 else None + + +def _match_unit(args: Sequence[Any], unit_of: Callable[[Any], Any]) -> Any: + """``MATCH`` compares the key with the lines of the range; a position + has no unit.""" + from .criteria import _lines_of + if len(args) >= 2: + _key_against("MATCH", args[0], _lines_of(args[1]), unit_of) + return None + + +def _index_unit(args: Sequence[Any], unit_of: Callable[[Any], Any]) -> Any: + """``INDEX`` takes a cell of a range — the unit its lines share — or of a + block, at the rows and columns its positions can address.""" + from .criteria import _lines_of + if len(args) == 2: + return same_unit("INDEX", _lines_of(args[0]), unit_of) + if len(args) == 3: + return _pick_unit("INDEX", args[0], args[1], args[2], unit_of) + return None + + +def _xlookup_unit(args: Sequence[Any], unit_of: Callable[[Any], Any]) -> Any: + """``XLOOKUP`` compares the key with the lines searched and takes a cell + of the lines returned, or ``if_not_found`` in their unit: theirs.""" + from .criteria import _lines_of + if len(args) < 3: + return None + _key_against("XLOOKUP", args[0], _lines_of(args[1]), unit_of) + returned = _lines_of(args[2]) + unit = same_unit("XLOOKUP", returned, unit_of) + if len(args) > 3: # if_not_found, or NA() before a mode + same_unit("XLOOKUP", [*returned, args[3]], unit_of) + return unit + + +def _vlookup_unit(args: Sequence[Any], unit_of: Callable[[Any], Any]) -> Any: + """``VLOOKUP`` compares the key with the block's first column and takes a + cell of the column its number addresses, in any row.""" + if len(args) < 3: + return None + first = [row.items[0] if isinstance(row, ListExpr) else row for row in _rows_of(args[1])] + _key_against("VLOOKUP", args[0], first, unit_of) + return _pick_unit("VLOOKUP", args[1], None, args[2], unit_of) + + +def _hlookup_unit(args: Sequence[Any], unit_of: Callable[[Any], Any]) -> Any: + """``HLOOKUP`` compares the key with the block's first row and takes a + cell of the row its number addresses, in any column.""" + from .criteria import _lines_of + if len(args) < 3: + return None + rows = _rows_of(args[1]) + if rows: + _key_against("HLOOKUP", args[0], _lines_of(rows[0]), unit_of) + return _pick_unit("HLOOKUP", args[1], args[2], None, unit_of) + + +unit_law("MATCH", _match_unit) +unit_law("INDEX", _index_unit) +unit_law("XLOOKUP", _xlookup_unit) +unit_law("VLOOKUP", _vlookup_unit) +unit_law("HLOOKUP", _hlookup_unit) diff --git a/src/modeleon/functions/mathfn.py b/src/modeleon/functions/mathfn.py index 5cd7880..a351bda 100644 --- a/src/modeleon/functions/mathfn.py +++ b/src/modeleon/functions/mathfn.py @@ -1,11 +1,28 @@ # SPDX-License-Identifier: Apache-2.0 -"""Element-wise math helpers — ``ABS``, ``ROUND``, ``INT``, ``MOD``. +"""Element-wise math helpers — ``ABS``, ``ROUND``, ``INT``, ``MOD``; the +rounding family ``ROUNDUP``, ``ROUNDDOWN``, ``TRUNC``, ``CEILING``, +``FLOOR``; the powers and logarithms ``POWER``, ``SQRT``, ``EXP``, ``LN``, +``LOG``, ``LOG10``; ``SIGN``; and ``PRODUCT``, a reduction over one +series and one call per period over several. Named ``mathfn`` (not ``math``) to avoid shadowing Python's stdlib ``math``. **Invariant:** every call returns a :class:`Variable` with a tracked formula. ``mo.ABS(-5)`` produces ``=ABS(-5)`` in Excel, not a bare ``5`` — the formula is the product. + +The functions after ``MOD`` read a cell as the spreadsheet does: a blank +is 0 (``ROUNDUP`` of a blank is 0, ``EXP`` of one is 1), a boolean is 1 +or 0, a date its serial number (``FLOOR(date, 7)`` is a serial on a +multiple of 7), numeric text is its number (``'4'`` is 4) and other text +``#VALUE!``, a number outside the function's domain is ``#NUM!`` +(``SQRT`` of a negative, ``LN`` of zero), one too large for a cell is +``#NUM!`` too, and an error cell passes through. A rounded amount keeps +its unit, a remainder and a multiple share one unit with what they divide +by; ``POWER`` and ``SQRT`` raise the unit as ``**`` does (``SQRT`` of +``USD^2`` is ``USD``; a power that is not whole in every dimension, or +that is a line, leaves none); ``PRODUCT`` multiplies its operands' +units; an exponential, a logarithm and a sign carry none. """ from __future__ import annotations @@ -13,10 +30,21 @@ import builtins import decimal import math +import sys +from datetime import date, timedelta from typing import Any, Union +from ..core.expr import Literal, VarRef +from ..core.unit import UNSETTLED from ..core.variable import Variable -from ._helpers import make_func_var, operand, src +from ._helpers import ( + align_keyed, align_located, elementwise, is_series, make_func_var, operand, placed, + series_node, src, unit_law, +) +from .aggregate import ( + _by_period, _elementwise, _excel_reduce, _flatten_args, _lifted_reduce, _listed, +) +from .registry import per_period_arity, register Number = Union[int, float] Numeric = Union[Variable, Number] @@ -59,10 +87,11 @@ def ABS(value: Numeric) -> Variable: raw, expr = operand(value) calc, var_type = _apply(builtins.abs, raw) value_type = value.value_type if isinstance(value, Variable) else 'float' - return make_func_var( + out = make_func_var( "ABS", [expr], calc, value_type, var_type, source_code=f"ABS({src(value)})", ) + return placed(out, None, align_keyed("ABS", [value], [raw])[1]) #: No limit on digits or exponent, so a very long whole number or an @@ -76,7 +105,7 @@ def ABS(value: Numeric) -> Variable: _SHOWN_CONTEXT = decimal.Context(prec=15, rounding=decimal.ROUND_HALF_UP) -def _spreadsheet_round(v, digits: int): +def _spreadsheet_round(v, digits: int, rounding: str = decimal.ROUND_HALF_UP): """Round ``v`` the way the spreadsheet's ``ROUND`` does. Two differences from Python's ``round``: a half goes AWAY FROM ZERO @@ -88,7 +117,9 @@ def _spreadsheet_round(v, digits: int): away from zero.) A number of 16 or more whole digits has no decimals left to show and is rounded as held, so its whole digits never move. Anything that is not a finite number goes to Python's ``round`` as - before. + before. ``rounding`` is the :mod:`decimal` mode: ``ROUND_HALF_UP`` is + ``ROUND``, ``ROUND_UP`` (away from zero) ``ROUNDUP``, ``ROUND_DOWN`` + (toward zero) ``ROUNDDOWN`` and ``TRUNC`` — each on the shown number. """ if isinstance(v, bool) or not isinstance(v, (int, float)): return builtins.round(v, digits) @@ -106,7 +137,7 @@ def _spreadsheet_round(v, digits: int): rounded = shown # no digit past the cut-off else: step = decimal.Decimal(1).scaleb(-digits, context=_ROUND_CONTEXT) - rounded = shown.quantize(step, context=_ROUND_CONTEXT) + rounded = shown.quantize(step, rounding=rounding, context=_ROUND_CONTEXT) return int(rounded) if isinstance(v, int) else float(rounded) @@ -116,17 +147,22 @@ def ROUND(value: Numeric, digits: Numeric = 0) -> Variable: Computes what the spreadsheet computes: halves round away from zero (``ROUND(2.5, 0) == 3``), on the number as a cell shows it (``ROUND(1.005, 2) == 1.01``). Negative ``digits`` round to tens, - hundreds, … (``ROUND(1250, -2) == 1300``). + hundreds, … (``ROUND(1250, -2) == 1300``). ``digits`` may vary by + period: a series of digits is one call per period, ``=ROUND(C5, C9)`` + in every column. """ + value_type = value.value_type if isinstance(value, Variable) else 'float' + if is_series(digits): + return elementwise("ROUND", _round_to, [value, digits], blank=0, value_type=value_type) raw, expr = operand(value) raw_digits, digits_expr = operand(digits) d = int(raw_digits or 0) # blank digits read as 0 calc, var_type = _apply(lambda v: _spreadsheet_round(v, d), raw) - value_type = value.value_type if isinstance(value, Variable) else 'float' - return make_func_var( + out = make_func_var( "ROUND", [expr, digits_expr], calc, value_type, var_type, source_code=f"ROUND({src(value)}, {d})", ) + return placed(out, None, align_keyed("ROUND", [value], [raw])[1]) def INT(value: Numeric) -> Variable: @@ -137,10 +173,11 @@ def INT(value: Numeric) -> Variable: """ raw, expr = operand(value) calc, var_type = _apply(math.floor, raw) - return make_func_var( + out = make_func_var( "INT", [expr], calc, 'int', var_type, source_code=f"INT({src(value)})", ) + return placed(out, None, align_keyed("INT", [value], [raw])[1]) def MOD(number: Numeric, divisor: Numeric) -> Variable: @@ -148,12 +185,16 @@ def MOD(number: Numeric, divisor: Numeric) -> Variable: For the usual case, the ``%`` operator already compiles to ``MOD(...)``. Use this when you want the explicit function call form in the Excel output. + Element-wise as ``%``: lines on other time windows are read by date, + lines along keys key by key. """ from ..core.tracks import TrackValues from ..core.variable_ops import _VariableArithmetic num_val, num_expr = operand(number) div_val, div_expr = operand(divisor) + raws, keys = align_keyed("MOD", (number, divisor), [num_val, div_val]) + (num_val, div_val), window = align_located((number, divisor), raws) def mod(a: Any, b: Any) -> Any: # Element-wise with broadcasting, as the ``%`` operator: a blank is @@ -167,7 +208,474 @@ def mod(a: Any, b: Any) -> Any: else: calc = mod(num_val, div_val) var_type = 'list' if isinstance(calc, list) else 'scalar' - return make_func_var( + out = make_func_var( "MOD", [num_expr, div_expr], calc, 'float', var_type, source_code=f"MOD({src(number)}, {src(divisor)})", ) + return placed(out, window, keys) + + +# ─── The rounding family, the powers and logarithms, the sign ───────── +# +# Each is one call per period through the shared :func:`elementwise` +# helper, over a kernel that takes one period's numbers. The recurrence +# template evaluator reuses the kernels, so ``"ROUNDUP({prev}, 0)"`` +# computes what the cell computes. + + +def _number(v: Any) -> int | float: + """A cell's number, as the spreadsheet reads an operand: a boolean is + 1 or 0, a date its serial number, numeric text its number; other text + raises ``TypeError`` (the helper's ``#VALUE!``), a number no cell + holds ``OverflowError`` (its ``#NUM!``).""" + if type(v) is int or (type(v) is float and math.isfinite(v)): + return v # the common case, without the reader's checks + from .text import _number_of # late, so neither module needs the other first + return _number_of(v) + + +def _in_a_cell(out: int | float) -> Any: + """``out`` as a cell holds it: a negative zero is 0, a number too + large for a cell is ``#NUM!``.""" + if isinstance(out, int): + return out if abs(out) <= sys.float_info.max else '#NUM!' + return out + 0.0 if math.isfinite(out) else '#NUM!' + + +def _rounded(value: Any, digits: Any, rounding: str) -> Any: + return _in_a_cell(_spreadsheet_round(_number(value), int(_number(digits)), rounding)) + + +# ``ROUND`` of one period whose digits are a series: the value read as +# ``ROUND`` reads it over one number of digits, the digits as the +# spreadsheet reads a number, their fraction dropped. +def _round_to(value: Any, digits: Any) -> Any: + return _spreadsheet_round(value, int(_number(digits))) + + +def _roundup(value: Any, digits: Any) -> Any: + return _rounded(value, digits, decimal.ROUND_UP) + + +def _rounddown(value: Any, digits: Any) -> Any: + return _rounded(value, digits, decimal.ROUND_DOWN) + + +def _trunc(value: Any, digits: Any = 0) -> Any: + return _rounddown(value, digits) + + +def _multiple(x: int | float, significance: int | float, rounding: str) -> Any: + """``x`` moved to a multiple of ``significance`` the way ``rounding`` + says. The quotient is the double the sheet divides to, read as a cell + shows it: ``1.5 / 0.1`` is held as 15.000000000000002 and shown as 15, + so ``CEILING(1.5, 0.1)`` stays 1.5. One of 16 or more whole digits has + no decimals left to show and is taken as held, as ``ROUND`` takes such + a number, so an exact multiple stays (``CEILING(1000000000000001, 1)``); + one too large for a cell is ``#NUM!``, one too small for it 0.""" + held = x / significance + if not math.isfinite(held): + return '#NUM!' + quotient = decimal.Decimal(held) + if abs(held) < 1e15: + quotient = _SHOWN_CONTEXT.plus(quotient) + steps = quotient.to_integral_value(rounding=rounding) + if isinstance(x, int) and isinstance(significance, int): + return _in_a_cell(int(steps) * significance) + return _in_a_cell(float(steps * _SHOWN_CONTEXT.plus(decimal.Decimal(significance)))) + + +def _ceiling(value: Any, significance: Any) -> Any: + x, step = _number(value), _number(significance) + if step == 0: + return 0 if isinstance(x, int) and isinstance(step, int) else 0.0 + if x > 0 and step < 0: + return '#NUM!' + return _multiple(x, step, decimal.ROUND_CEILING) + + +def _floor(value: Any, significance: Any) -> Any: + x, step = _number(value), _number(significance) + if step == 0 and x == 0: + return 0 if isinstance(x, int) and isinstance(step, int) else 0.0 + if step == 0: + raise ZeroDivisionError # the helper's #DIV/0! + if x > 0 and step < 0: + return '#NUM!' + return _multiple(x, step, decimal.ROUND_FLOOR) + + +#: A whole number of more bits than this is past the largest a cell +#: holds (2 ** 1024): ``POWER`` of whole numbers beyond it is ``#NUM!`` +#: without computing the number. +_POWER_BITS = 1100 + + +def _approx_equal(a: float, b: float) -> bool: + """The spreadsheet's test for the same number: equal to within the + last four bits of the smaller, or half its 15th significant digit.""" + if a == b: + return True + if a == 0 or b == 0 or (a < 0) != (b < 0): + return False + least = min(abs(a), abs(b)) + return abs(a - b) < max(least * 2.0 ** -48, 10.0 ** math.floor(math.log10(least)) * 5e-15) + + +def _odd_root(x: int | float, y: float) -> Any: + """A negative number to a fractional power, as the spreadsheet takes + it: real only as an odd root, ``y`` the inverse of an odd whole + number to within the 15th digit (``POWER(-8, 1/3)`` is -2, + ``POWER(-32, -0.2)`` -0.5); any other fraction is ``#NUM!``.""" + nearest = 1 / y + (0.5 if y > 0 else -0.5) + if not abs(nearest) < 2.0 ** 63: + return '#NUM!' + n = int(nearest) # toward zero + if n % 2 == 0 or not _approx_equal(1 / n, y): + return '#NUM!' + return _in_a_cell(-((-x) ** y)) + + +def _power(base: Any, exponent: Any) -> Any: + x, y = _number(base), _number(exponent) + if x == 0 and y == 0: + return '#NUM!' + if x < 0 and not (isinstance(y, int) or y.is_integer()): + return _odd_root(x, y) + if (isinstance(x, int) and isinstance(y, int) and y > 0 and abs(x) > 1 + and y * math.log2(abs(x)) > _POWER_BITS): + return '#NUM!' # a whole number no cell can hold + return _in_a_cell(x ** y) # 0 ** -1 is the helper's #DIV/0!, a float too large its #NUM! + + +def _sqrt(value: Any) -> Any: + x = _number(value) + return '#NUM!' if x < 0 else math.sqrt(x) + + +def _exp(value: Any) -> Any: + return math.exp(_number(value)) # too large is the helper's #NUM! + + +def _ln(value: Any) -> Any: + x = _number(value) + return '#NUM!' if x <= 0 else math.log(x) + + +def _log10(value: Any) -> Any: + x = _number(value) + return '#NUM!' if x <= 0 else math.log10(x) + + +def _log(value: Any, base: Any = 10) -> Any: + x, b = _number(value), _number(base) + if x <= 0 or b <= 0: + return '#NUM!' + if b == 1: + return '#DIV/0!' + if b == 10: + return math.log10(x) # exact at the powers of ten + if b == 2: + return math.log2(x) + return math.log(x) / math.log(b) + + +def _sign(value: Any) -> int: + x = _number(value) + return (x > 0) - (x < 0) + + +def _whole(x: Any) -> bool: + """An operand that is whole in every period: a plain whole number, or + a Variable of them.""" + if isinstance(x, Variable): + return x.value_type == 'int' + return isinstance(x, int) and not isinstance(x, bool) + + +def _whole_or_float(*args: Any) -> str: + """``'int'`` when the result is whole by construction — every operand + is — else ``'float'``.""" + return 'int' if all(_whole(a) for a in args) else 'float' + + +def ROUNDUP(value: Numeric, digits: Numeric) -> Variable: + """Round away from zero to ``digits`` decimal places (renders ``=ROUNDUP(...)``). + + ``ROUNDUP(3.2, 0) == 4``, ``ROUNDUP(-3.2, 0) == -4``, + ``ROUNDUP(31415.92654, -2) == 31500``; on the number as a cell shows + it, so ``ROUNDUP(0.1 + 0.2, 1) == 0.3``. Scalar or element-wise on + lists; ``digits`` may vary by period. A whole number stays whole. + """ + return elementwise("ROUNDUP", _roundup, [value, digits], blank=0, + value_type=_whole_or_float(value)) + + +def ROUNDDOWN(value: Numeric, digits: Numeric) -> Variable: + """Round toward zero to ``digits`` decimal places (renders ``=ROUNDDOWN(...)``). + + ``ROUNDDOWN(3.9, 0) == 3``, ``ROUNDDOWN(-3.9, 0) == -3``, + ``ROUNDDOWN(31415.92654, -2) == 31400``; on the number as a cell shows + it, so ``ROUNDDOWN(4.35 * 100, 0) == 435``, not 434. + """ + return elementwise("ROUNDDOWN", _rounddown, [value, digits], blank=0, + value_type=_whole_or_float(value)) + + +def TRUNC(value: Numeric, digits: Numeric | None = None) -> Variable: + """Drop the decimals past ``digits`` (default 0), toward zero (renders ``=TRUNC(...)``). + + ``TRUNC(8.9) == 8``, ``TRUNC(-8.9) == -8``, ``TRUNC(1234.5678, 2) == + 1234.56``: ``ROUNDDOWN`` with an optional second argument, written + ``=TRUNC(x)`` when none is given. + """ + args = [value] if digits is None else [value, digits] + return elementwise("TRUNC", _trunc, args, blank=0, value_type=_whole_or_float(value)) + + +def CEILING(value: Numeric, significance: Numeric) -> Variable: + """Round away from zero to a multiple of ``significance`` (renders ``=CEILING(...)``). + + The classic function's rules: ``CEILING(2.5, 1) == 3``, + ``CEILING(0.234, 0.01) == 0.24``, an exact multiple stays; a negative + number with a negative significance rounds away from zero + (``CEILING(-2.5, -2) == -4``), with a positive one toward zero + (``CEILING(-2.5, 2) == -2``); a positive number with a negative + significance is ``#NUM!``; a significance of 0 gives 0. The quotient + is read as a cell shows it (``CEILING(1.5, 0.1) == 1.5``); one too + large for a cell is ``#NUM!``. A date is its serial number. + """ + return elementwise("CEILING", _ceiling, [value, significance], blank=0, + value_type=_whole_or_float(value, significance)) + + +def FLOOR(value: Numeric, significance: Numeric) -> Variable: + """Round toward zero to a multiple of ``significance`` (renders ``=FLOOR(...)``). + + The classic function's rules: ``FLOOR(3.7, 2) == 2``, ``FLOOR(0.234, + 0.01) == 0.23``, an exact multiple stays; a negative number with a + positive significance rounds away from zero (``FLOOR(-2.5, 2) == + -4``), with a negative one toward zero (``FLOOR(-2.5, -2) == -2``); a + positive number with a negative significance is ``#NUM!``; a + significance of 0 is ``#DIV/0!``, except that ``FLOOR(0, 0)`` is 0. + """ + return elementwise("FLOOR", _floor, [value, significance], blank=0, + value_type=_whole_or_float(value, significance)) + + +def POWER(base: Numeric, exponent: Numeric) -> Variable: + """``base`` to the power ``exponent`` (renders ``=POWER(...)``). + + The ``**`` operator already writes ``^``; this is the function call in + the cell. ``POWER(5, 2) == 25``, ``POWER(4, 5 / 4) == 5.656854…``. A + negative base to a fractional power is real only as an odd root — + ``POWER(-8, 1 / 3) == -2``, the exponent the inverse of an odd whole + number to within the 15th digit — and ``#NUM!`` otherwise + (``POWER(-8, 0.5)``); ``0 ** 0`` is ``#NUM!``, ``0 ** -1`` is + ``#DIV/0!``, a result too large for a cell ``#NUM!``, one too small + for it 0. The unit follows ``**``: each dimension's power times the + exponent (``POWER(t, 2)`` in hr is hr^2, ``POWER(area, 0.5)`` in m^2 + is m); none when a power comes out fractional or the exponent is a + line. + """ + plain = not isinstance(exponent, Variable) + whole = _whole(base) and plain and _whole(exponent) and exponent >= 0 + return elementwise("POWER", _power, [base, exponent], blank=0, + value_type='int' if whole else 'float') + + +def SQRT(value: Numeric) -> Variable: + """Square root (renders ``=SQRT(...)``); a negative number is ``#NUM!``. + The unit is halved as ``** 0.5`` halves it: ``SQRT`` of a variance in + USD^2 is in USD; of a unit with an odd power, none.""" + return elementwise("SQRT", _sqrt, [value], blank=0, value_type='float') + + +def EXP(value: Numeric) -> Variable: + """``e`` to the power ``value`` (renders ``=EXP(...)``): ``EXP(0) == 1``; + a result too large for a cell is ``#NUM!``.""" + return elementwise("EXP", _exp, [value], blank=0, value_type='float') + + +def LN(value: Numeric) -> Variable: + """Natural logarithm (renders ``=LN(...)``); zero or a negative number is ``#NUM!``.""" + return elementwise("LN", _ln, [value], blank=0, value_type='float') + + +def LOG(value: Numeric, base: Numeric | None = None) -> Variable: + """Logarithm to ``base``, 10 when none is given (renders ``=LOG(x)`` / + ``=LOG(x, base)``). + + ``LOG(100) == 2``, ``LOG(8, 2) == 3``; zero or a negative number or + base is ``#NUM!``, a base of 1 is ``#DIV/0!``. + """ + args = [value] if base is None else [value, base] + return elementwise("LOG", _log, args, blank=0, value_type='float') + + +def LOG10(value: Numeric) -> Variable: + """Logarithm to base 10 (renders ``=LOG10(...)``); zero or a negative number is ``#NUM!``.""" + return elementwise("LOG10", _log10, [value], blank=0, value_type='float') + + +def SIGN(value: Numeric) -> Variable: + """The sign of a number, 1, 0 or -1 (renders ``=SIGN(...)``).""" + return elementwise("SIGN", _sign, [value], blank=0, value_type='int') + + +def _product(values: list[Any]) -> Any: + """``math.prod`` over a range's numbers, a date as its serial number; + one that is not a number is refused at build time, as ``SUM`` refuses + it.""" + factors = [] + for v in values: + if isinstance(v, (date, timedelta)): + v = _number(v) + elif not isinstance(v, (int, float)): + raise TypeError( + f"PRODUCT multiplies numbers; {v!r} is not one. The spreadsheet " + f"skips text in a range, so no error value would match the workbook — " + f"give it the number." + ) + factors.append(v) + return _in_a_cell(math.prod(factors)) + + +def PRODUCT(*args: Union[Variable, Number, list[Number]]) -> Variable: + """Product of Variables, scalars, or lists (renders ``=PRODUCT(...)``). + + Over ONE operand, the product of the row's cells over its periods: + ``PRODUCT(growth_factors)`` → ``=PRODUCT(B2:F2)``. Over two or more + operands with a series among them, the product PER PERIOD, as ``a * + b`` multiplies them — scalars broadcast, a line with tracks taken + track by track: ``PRODUCT(price, volume)`` → ``=PRODUCT(B2, B3)`` in + every column. Over scalars only, one number. The cells are read as + LibreOffice reads them: a blank is skipped, a boolean is 1 or 0, a + date its serial number, an error cell is the result, nothing left is + 0, a product too large for a cell is ``#NUM!``. Excel skips a boolean + held in a referenced cell and counts only one typed into the call; + the values here do not say where a boolean came from, so a referenced + FALSE makes 0 as ``SUM`` counts it, where Excel would skip it. Text is + refused at build time, as ``SUM`` refuses it. The result is a whole + number only over whole numbers (a product of dates is a number, not a + date). Its unit is the product of its operands' (``PRODUCT(price, + volume)`` in USD/t and t is USD); one row in a dimension alone is + refused, its cells' product being that dimension to the power of + their count — a row of pure numbers (growth factors) is what a + product over the periods runs along. + """ + args = _listed("PRODUCT", args) + if _by_period("PRODUCT", args): + return _elementwise("PRODUCT", args, _product, 0, _whole_or_float(*args)) + from ..core.tracks import TrackValues + product = _excel_reduce(_product) + if (len(args) == 1 and isinstance(args[0], Variable) + and isinstance(args[0]._value, TrackValues)): + out = _lifted_reduce("PRODUCT", args[0], product) + else: + flat, ast_args = _flatten_args(args, "PRODUCT") + out = make_func_var( + "PRODUCT", ast_args, product(flat), _whole_or_float(*args), + 'scalar', source_code=f"PRODUCT({', '.join(src(a) for a in args)})", + ) + return out + + +# ─── Units ────────────────────────────────────────────────────────── + + +def _rounded_unit(args: Any, unit_of: Any) -> Any: + """A rounded value keeps its unit; the digits are a count.""" + return unit_of(args[0]) if args else None + + +def _power_unit(args: Any, unit_of: Any) -> Any: + """The unit ``**`` gives: a constant exponent raises each dimension; + an exponent that is a line may differ by period, so none.""" + from ..core.variable_ops import _VariableArithmetic + if not args: + return None + base = unit_of(args[0]) + if base is UNSETTLED: + return UNSETTLED + exponent = args[1] if len(args) > 1 else None + return _VariableArithmetic._unit_pow( + base, exponent.value if isinstance(exponent, Literal) else None) + + +def _sqrt_unit(args: Any, unit_of: Any) -> Any: + from ..core.variable_ops import _VariableArithmetic + if not args: + return None + base = unit_of(args[0]) + return UNSETTLED if base is UNSETTLED else _VariableArithmetic._unit_pow(base, 0.5) + + +def _cell_count(var: Any) -> int: + from ..core.tracks import TrackValues + raw = var._value + if isinstance(raw, TrackValues): + return raw.time_length or 1 + return len(raw) if isinstance(raw, list) else 1 + + +def _product_unit(args: Any, unit_of: Any) -> Any: + """The product of the operands' units; a row in a dimension refused + where its cells are multiplied together — one call per period + multiplies each operand's unit once, as ``a * b`` does.""" + from ..core.variable_ops import _VariableArithmetic, _line_name + arity = per_period_arity("PRODUCT") or 2 + by_period = len(args) >= arity and any(series_node(node) for node in args) + unit = None + waiting = False + for node in args: + factor = unit_of(node) + if factor is UNSETTLED: + waiting = True + continue + if factor is None: + continue + words = factor._unit_components + if (words and not by_period and isinstance(node, VarRef) + and _cell_count(node.var) > 1): + hint = "" + if len(words) == 1 and next(iter(words.values())) == 1: + word = next(iter(words)) + name = word.upper() if word.isidentifier() else "NAME" + hint = (f" If '{word}' labels a pure number, declare it once - {name} = " + f"mo.Unit.dimensionless('{word}') - and set that object as the " + f"row's unit.") + raise ValueError( + f"mo.PRODUCT multiplies the cells of {_line_name(node.var)}, a row in " + f"{factor}: their product is in {factor} to the power of their count. " + f"A product runs along a row of pure numbers (growth factors, 1 + a " + f"rate); amounts multiply line by line: a * b.{hint}" + ) + unit = _VariableArithmetic._unit_mul(unit, factor) + return UNSETTLED if waiting else unit + + +register("ABS", shape="elementwise") +register("ROUND", shape="elementwise") +register("INT", shape="elementwise") +register("MOD", shape="elementwise") +register("ROUNDUP", shape="elementwise") +register("ROUNDDOWN", shape="elementwise") +register("TRUNC", shape="elementwise") +register("CEILING", shape="elementwise") +register("FLOOR", shape="elementwise") +register("POWER", shape="elementwise") +register("SQRT", shape="elementwise") +register("EXP", shape="elementwise") +register("LN", shape="elementwise") +register("LOG", shape="elementwise") +register("LOG10", shape="elementwise") +register("SIGN", shape="elementwise") +register("PRODUCT", shape="reduction", per_period_arity=2) +unit_law("ROUND", _rounded_unit) +unit_law("ROUNDUP", _rounded_unit) +unit_law("ROUNDDOWN", _rounded_unit) +unit_law("TRUNC", _rounded_unit) +unit_law("POWER", _power_unit) +unit_law("SQRT", _sqrt_unit) +unit_law("PRODUCT", _product_unit) diff --git a/src/modeleon/functions/recurrence.py b/src/modeleon/functions/recurrence.py index 72fd47f..6f26376 100644 --- a/src/modeleon/functions/recurrence.py +++ b/src/modeleon/functions/recurrence.py @@ -25,6 +25,21 @@ from ..core.expr import Expr, Literal, SelfRef, VarRef from ..core.variable import Variable +from ._helpers import align_keyed, align_located, placed, refuse_unpaired +from .mathfn import ( + _ceiling, + _exp, + _floor, + _ln, + _log, + _log10, + _power, + _rounddown, + _roundup, + _sign, + _sqrt, + _trunc, +) # ─── Safe formula evaluator ──────────────────────────────────────────── @@ -100,6 +115,22 @@ def _edate(d, months): 'ROUND': round, 'INT': math.floor, 'MOD': operator.mod, + # The rounding family, the powers and logarithms, the sign: the + # kernels behind the public helpers in ``mathfn``, so a template + # computes what the cell computes (``ROUNDUP`` away from zero on the + # number as shown, ``CEILING`` with its sign rules). + 'CEILING': _ceiling, + 'EXP': _exp, + 'FLOOR': _floor, + 'LN': _ln, + 'LOG': _log, + 'LOG10': _log10, + 'POWER': _power, + 'ROUNDDOWN': _rounddown, + 'ROUNDUP': _roundup, + 'SIGN': _sign, + 'SQRT': _sqrt, + 'TRUNC': _trunc, # Pure date arithmetic — safe to whitelist (no I/O, no attribute # access, deterministic on inputs). Mirror the public ``EDATE`` / # ``EOMONTH`` / ``YEAR`` / ``MONTH`` / ``DAY`` so a recurrence @@ -235,15 +266,19 @@ def _resolve_periods(periods, variables: Dict[str, Any]) -> Tuple[int, Optional[ return periods, None -def _per_period_values(variables: Dict[str, Any], periods: int) -> Dict[str, list]: +def _per_period_values(variables: Dict[str, Any], periods: int, + read: Optional[Dict[str, list]] = None) -> Dict[str, list]: """Align each variable to a per-period list of raw values. - List-valued Variables pass through (Variables-of-Variables unwrapped). + List-valued Variables pass through (Variables-of-Variables unwrapped), + as ``read`` gives them where they were read by date (:func:`_by_date`). Scalars get broadcast to ``periods`` copies. """ out: Dict[str, list] = {} for key, var in variables.items(): - if isinstance(var, Variable): + if read is not None and key in read: + out[key] = read[key] + elif isinstance(var, Variable): if var.var_type == 'list' and isinstance(var._value, list): out[key] = [v._value if isinstance(v, Variable) else v for v in var._value] else: @@ -253,6 +288,25 @@ def _per_period_values(variables: Dict[str, Any], periods: int) -> Dict[str, lis return out +def _by_date(variables: Dict[str, Any]) -> Tuple[Optional[Dict[str, list]], Any, Any]: + """The template variables' series read as ``a + b`` reads them: on + other time windows by date — each continued by its own ``extend=`` + rule, a window not covered refused — along keys key by key; lines of + other lengths refused (:func:`refuse_unpaired`). Returns + the series so read (None where nothing was), the combined window and + the keys the recurrence runs on.""" + names = list(variables) + lines = [variables[name] for name in names] + raws = [v._value if isinstance(v, Variable) else v for v in lines] + read, keys = align_keyed("recurrence", lines, raws) + read, window = align_located(lines, read) + refuse_unpaired(lines, read, window) + if window is None and keys is None: + return None, None, None + return ({name: value for name, value in zip(names, read) if isinstance(value, list)}, + window, keys) + + def _eval_lambda(formula, start_value, periods: int) -> list: """Evaluate ``lambda prev, t: ...`` for periods 1..N-1; period 0 = start.""" result = [start_value] @@ -315,6 +369,12 @@ def _eval_template(formula: str, start_value, periods: int, result.append('#DIV/0!') prev = '#DIV/0!' continue + except OverflowError: + # A number too large for a cell (EXP of a large one) is + # Excel's #NUM!, carried on the same way. + result.append('#NUM!') + prev = '#NUM!' + continue except TypeError as e: raise ValueError( f"Error evaluating formula {formula!r} at period {t}: {e}" @@ -547,10 +607,15 @@ def _slice_operand(v: Any, role: str) -> Any: ) if display_name: result._display_name = display_name - return _with_line(result, line) + return _with_line(result, line, start) is_lambda = callable(formula) and (inspect.isfunction(formula) or inspect.ismethod(formula)) - periods_resolved, periods_var = _resolve_periods(periods, variables) + read, window, keys = (None, None, None) if is_lambda else _by_date(variables) + if read is not None and periods is None: + # The combined window's periods, which every series read on it fills. + periods_resolved, periods_var = max(len(v) for v in read.values()), None + else: + periods_resolved, periods_var = _resolve_periods(periods, variables) start_value = start._value if isinstance(start, Variable) else start @@ -575,7 +640,7 @@ def _slice_operand(v: Any, role: str) -> Any: ) values = _eval_template( formula, start_value, periods_resolved, - _per_period_values(variables, periods_resolved), + _per_period_values(variables, periods_resolved, read), ) recurrence_expr = _build_selfref_expr( start, formula, variables, periods_var, periods_resolved, @@ -609,7 +674,7 @@ def _slice_operand(v: Any, role: str) -> Any: if display_name: result._display_name = display_name - return _with_line(result, line) + return _with_line(placed(result, window, keys), line, start) #: What a line takes beside its recurrence: its fold rule, unit, words and @@ -617,11 +682,23 @@ def _slice_operand(v: Any, role: str) -> Any: _LINE_KEYWORDS = ('regrain', 'unit', 'description', 'excel_props') -def _with_line(result: Variable, line: Dict[str, Any]) -> Variable: +def _with_line(result: Variable, line: Dict[str, Any], start: Any = None) -> Variable: if 'regrain' in line and line['regrain'] is not None: result.set_regrain(line['regrain']) - if 'unit' in line and line['unit'] is not None: - result.set_unit(line['unit']) + declared = line.get('unit') + expr = getattr(result, '_expr', None) + if isinstance(expr, SelfRef): + # The template is read for units as any formula is: a mix is + # refused, and a start without a unit takes the template's. + derived = _checked_template_unit(expr, start, declared) + if declared is None and derived is not None: + result._unit = derived + if declared is not None: + result.set_unit(declared) + elif getattr(result, '_unit', None) is None and getattr(start, '_unit', None) is not None: + # The line rolls its start forward, so it is in the start's unit: + # a balance that opens in USD stays in USD. + result._unit = start._unit if 'description' in line and line['description'] is not None: result.set_description(line['description']) if 'excel_props' in line and line['excel_props'] is not None: @@ -630,6 +707,190 @@ def _with_line(result: Variable, line: Dict[str, Any]) -> Variable: return result +_TEMPLATE_BIN = {ast.Add: '+', ast.Sub: '-', ast.Mult: '*', ast.Div: '/', ast.Mod: '%', + ast.Pow: '**', ast.FloorDiv: '//'} +_TEMPLATE_CMP = {ast.Eq: '==', ast.NotEq: '!=', ast.Lt: '<', ast.LtE: '<=', ast.Gt: '>', + ast.GtE: '>='} + + +def _template_expr(node: ast.AST, names: Dict[str, Expr]) -> Expr: + """The template's parsed tree as the engine's formula nodes, so the + operators' unit rules read it as they read any formula; ``names`` + maps each placeholder (and ``prev``) to its node.""" + from ..core.expr import BinOp, Compare, FuncCall, UnaryOp + if isinstance(node, ast.Constant): + return Literal(node.value) + if isinstance(node, ast.Name): + return names.get(node.id, Literal(None)) # a bare function name: no unit + if isinstance(node, ast.BinOp): + return BinOp(_TEMPLATE_BIN[type(node.op)], _template_expr(node.left, names), + _template_expr(node.right, names)) + if isinstance(node, ast.UnaryOp): + inner = _template_expr(node.operand, names) + return FuncCall('NOT', [inner]) if isinstance(node.op, ast.Not) else UnaryOp('-', inner) + if isinstance(node, ast.Compare): + left = _template_expr(node.left, names) + pairs: list[Expr] = [] + for op, rhs in zip(node.ops, node.comparators): + right = _template_expr(rhs, names) + pairs.append(Compare(_TEMPLATE_CMP[type(op)], left, right)) + left = right + return pairs[0] if len(pairs) == 1 else FuncCall('AND', pairs) + if isinstance(node, ast.IfExp): + return FuncCall('IF', [_template_expr(n, names) + for n in (node.test, node.body, node.orelse)]) + if isinstance(node, ast.BoolOp): + return FuncCall('AND' if isinstance(node.op, ast.And) else 'OR', + [_template_expr(v, names) for v in node.values]) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name): + name = node.func.id if node.func.id in _SAFE_FUNCTIONS else node.func.id.upper() + return FuncCall(name, [_template_expr(a, names) for a in node.args]) + raise ValueError(f"Expression not allowed in formulas: {type(node).__name__}") + + +def _compares(expr: Any) -> Any: + """Every comparison in a formula tree - a condition is read by no + function's law, so its two sides are checked here.""" + from ..core.expr import Compare + if isinstance(expr, Compare): + yield expr + for child in (getattr(expr, 'args', None) or ()): + yield from _compares(child) + for attr in ('left', 'right', 'operand', 'inner'): + child = getattr(expr, attr, None) + if isinstance(child, Expr): + yield from _compares(child) + + +def _template_tree(node: SelfRef) -> Tuple[Expr, Variable]: + """The template as the engine's formula nodes and the Variable that + stands for ``{prev}`` in it (``_display_name`` '{prev}').""" + parseable = _PLACEHOLDER_RE.sub(r'\1', node.template) + tree = ast.parse(parseable, mode='eval').body + prev = Variable() + prev._display_name = '{prev}' + return _template_expr(tree, {**node.variables, 'prev': VarRef(prev)}), prev + + +def template_unit(node: SelfRef, start_unit: Any, declared: Any, *, + unsettled: Any = frozenset(), check_dimension: bool = True) -> Any: + """The unit a recurrence line is in, and the check of its template - + the one rule ``mo.recurrence`` builds the line by and a loop container + derives it by again (``core.loops._unit_of``, which passes the rows + not settled yet as ``unsettled``): the declared unit, else the start's, + else the one the template derives with the previous value read as + unsettled (as a loop row is read while its loop settles); then, the + previous value read in that unit, the template may not mix two units, + and (``check_dimension``, where the declared unit is known) must give + the row's unit or none. A start or a variable not settled yet leaves + the answer :data:`~modeleon.core.unit.UNSETTLED`.""" + from ..core.loops import _dimension, _unit_of + from ..core.unit import UNSETTLED + + if not isinstance(node.template, str): # the lambda form: no template + return declared if declared is not None else start_unit + if start_unit is UNSETTLED: + return UNSETTLED + expr, prev = _template_tree(node) + + def derive(prev_unit: Any, ids: Any) -> Any: + prev._unit = prev_unit + for cmp in _compares(expr): + _unit_of(cmp, ids) + return _unit_of(expr, ids) + + ids = frozenset(unsettled) + row = declared if declared is not None else start_unit + if row is None: + u = derive(None, ids | {id(prev)}) + if u is UNSETTLED: + # Unsettled by the previous value alone - the template says + # nothing of the row's unit - or by a row of the loop? + if derive(None, ids) is UNSETTLED: + return UNSETTLED + u = None + row = u + given = derive(row, ids) + if given is UNSETTLED: + return UNSETTLED + if (check_dimension and declared is None and given is not None + and _dimension(given) != _dimension(row)): + if row is None or set(_dimension(given)) >= set(_dimension(row)): + raise ValueError( + f"the template gives {given} from a start in {row or 'no unit'}: a " + f"dimension multiplied in every period grows a power of it, so the " + f"line's unit does not settle. Give the factor no unit, or declare " + f"the line's unit (unit=...)." + ) + raise ValueError( + f"the template gives {given}, but the start is in {row}, so the line " + f"would hold two units. Give the start and the template the same unit, " + f"or declare the line's unit (unit=...)." + ) + return row + + +def template_count_pairs(node: SelfRef, start: Any) -> list: + """The pairs of lines the template's own ``+`` / ``-`` add - ``{prev}`` + read as ``start`` (the first period, the one the sum has not written + into), each variable as its line - for the check ``+`` makes of a bps + or pp count added to a % fraction; a loop container reads them again + when it closes (``core.loops._check_counts``).""" + from ..core.expr import BinOp + if not isinstance(node.template, str): + return [] + expr, prev = _template_tree(node) + pairs: list = [] + + def line(n: Any) -> Any: + if not isinstance(n, VarRef): + return None + return start if n.var is prev else n.var + + def walk(e: Any) -> None: + if isinstance(e, BinOp) and e.op in ('+', '-'): + a, b = line(e.left), line(e.right) + if isinstance(a, Variable) and isinstance(b, Variable): + pairs.append((a, b)) + for child in (getattr(e, 'args', None) or ()): + walk(child) + for attr in ('left', 'right', 'operand', 'inner'): + child = getattr(e, attr, None) + if isinstance(child, Expr): + walk(child) + + walk(expr) + return pairs + + +def _template_counts(node: SelfRef, start: Any) -> None: + """The check ``+`` makes, over the template's own ``+`` / ``-``.""" + from ..core.variable_ops import _refuse_a_count_added_to_a_share + for a, b in template_count_pairs(node, start): + _refuse_a_count_added_to_a_share(a, b) + + +def _checked_template_unit(node: SelfRef, start: Any, declared: Any) -> Any: + """:func:`template_unit` over a line as built. A refusal over a row + whose loop is open is left to the loop, which runs the recurrence + again from its formula when it closes and reports it under the row.""" + from ..core.unit import UNSETTLED, Unit + from ..core.variable_ops import _on_open_loop + if isinstance(declared, str): + declared = Unit.parse(declared) + start_unit = getattr(start, '_unit', None) + provisional = any(_on_open_loop(v) for v in (start, *[ + e.var for e in node.variables.values() if isinstance(e, VarRef)])) + try: + unit = template_unit(node, start_unit, declared) + _template_counts(node, start) + return None if unit is UNSETTLED else unit + except ValueError as exc: + if not provisional: + raise ValueError(f'recurrence("{node.template}"): {exc}') from exc + return declared if declared is not None else start_unit + + def recurrence_sum( start: Any, increment: Any, diff --git a/src/modeleon/functions/registry.py b/src/modeleon/functions/registry.py new file mode 100644 index 0000000..f8c1fb5 --- /dev/null +++ b/src/modeleon/functions/registry.py @@ -0,0 +1,235 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The function registry — what the engine knows about each function name. + +One entry per name a :class:`~modeleon.core.expr.FuncCall` can carry: how +the call is shaped, what the workbook stores it as, and whether a loop +container may run it one period at a time. The Excel renderer and the +loop container read the registry; a function registers itself from its +own module, next to its definition:: + + from .registry import register + + def IFERROR(value, fallback): + ... + + register("IFERROR", shape="elementwise") + +Shapes: + +- ``'elementwise'`` — one call per period: a series operand gives a + series, the workbook writes this period's cells, a loop container runs + it period by period (``IF``, ``ABS``, ``ROUND``, …). +- ``'reduction'`` — reads whole series and gives one number; the workbook + writes a list operand as a range (``SUM(row)`` is ``=SUM(B2:F2)``). A + ``per_period_arity`` makes a call over that many values or more, a + series among them, one call per period instead: ``SUM(a, b)`` is a + subtotal, ``MIN`` / ``MAX`` over two operands are a cap or a floor, + ``MEDIAN(a, b)`` the median of each period's two cells. +- ``'range'`` — fixed arguments, some of them ranges. ``range_args`` names + their positions; None means every list argument is a range and every + other one a cell (``NPV(rate, flows)``, ``SUMPRODUCT``). A + ``per_period_arity`` makes a call whose ranges list that many lines or + more, a series among them, one call per period over this period's + cells: ``PERCENTILE_INC([a, b, c], k)``. +- ``'scalar'`` — a plain call over its arguments, written like an + element-wise one, that a loop container does not run period by period + (``PMT``, ``FV``, ``YEARFRAC``). + +A name the workbook stores with a prefix (``_xlfn.`` for the functions +Excel added after 2007) registers it as ``file_prefix``; a name Excel +spells differently registers ``excel``. A name without a helper of its +own, which the renderer only writes as given, is ``bare``. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Literal, Sequence + +Shape = Literal['elementwise', 'reduction', 'range', 'scalar'] + +SHAPES: tuple[str, ...] = ('elementwise', 'reduction', 'range', 'scalar') + + +@dataclass(frozen=True) +class FunctionSpec: + """How one function name is shaped; see the module docstring.""" + + name: str + shape: Shape + excel: str + file_prefix: str = '' + range_args: tuple[int, ...] | None = None + per_period_arity: int | None = None + passthrough: bool = True + bare: bool = False + + +_SPECS: dict[str, FunctionSpec] = {} + + +def register( + name: str, + *, + shape: Shape, + excel: str | None = None, + file_prefix: str = '', + range_args: Sequence[int] | None = None, + per_period_arity: int | None = None, + passthrough: bool = True, + bare: bool = False, +) -> FunctionSpec: + """Register ``name`` (upper-case, as the call is written) with its shape. + + ``excel`` is the name the workbook writes when it differs; + ``file_prefix`` what the file stores before it (``'_xlfn.'``); + ``range_args`` the positions written as ranges (``'range'`` shape only; + None = every list argument); ``per_period_arity`` the count of values + — a ``'reduction'``'s operands, the lines a ``'range'`` function's + ranges list — from which a call with a series among them is one call + per period; ``passthrough`` + False when the renderer writes the call some other way (``CONCAT`` is + the ``&`` operator); ``bare`` when the name has no helper of its own. Registering + the same spec twice is one entry; a different spec for a registered + name is refused. + """ + if not name or name != name.upper() or not name.isidentifier(): + raise ValueError( + f"a function registers under its upper-case identifier, as the call " + f"is written (SUM, IFERROR): got {name!r}." + ) + if shape not in SHAPES: + raise ValueError( + f"{name}: shape must be one of {', '.join(SHAPES)}; got {shape!r}." + ) + positions: tuple[int, ...] | None = None + if range_args is not None: + if shape != 'range': + raise ValueError( + f"{name}: range_args names the range positions of a 'range' " + f"function; a '{shape}' function has none." + ) + positions = tuple(range_args) + if not positions or any(not isinstance(p, int) or p < 0 for p in positions): + raise ValueError( + f"{name}: range_args are argument positions, 0-based: got {range_args!r}." + ) + if per_period_arity is not None: + if shape not in ('reduction', 'range'): + raise ValueError( + f"{name}: per_period_arity is how many values make a 'reduction' or a " + f"'range' function one call per period; a '{shape}' function has none." + ) + if per_period_arity < 2: + raise ValueError( + f"{name}: per_period_arity is at least 2 (one value is the " + f"reduction itself); got {per_period_arity}." + ) + new = FunctionSpec( + name=name, shape=shape, excel=excel or name, file_prefix=file_prefix, + range_args=positions, per_period_arity=per_period_arity, + passthrough=passthrough, bare=bare, + ) + old = _SPECS.get(name) + if old is not None and old != new: + raise ValueError( + f"{name} is already registered as {old}; a second, different " + f"registration ({new}) would change how every {name} call is written." + ) + _SPECS[name] = new + return new + + +# ─── Queries ──────────────────────────────────────────────────────── + + +def spec(name: str) -> FunctionSpec | None: + """The spec registered under ``name``, or None.""" + return _SPECS.get(name) + + +def names() -> frozenset[str]: + """Every registered name.""" + return frozenset(_SPECS) + + +def is_passthrough(name: str) -> bool: + """Does the Excel renderer write ``=NAME(args)`` for this name?""" + s = _SPECS.get(name) + return s is not None and s.passthrough + + +def is_range_taking(name: str) -> bool: + """Is a list argument of this name written as a range (a reduction or + a range function)?""" + s = _SPECS.get(name) + return s is not None and s.shape in ('reduction', 'range') + + +def range_positions(name: str) -> tuple[int, ...] | None: + """The argument positions written as ranges, or None for every list + argument (and for a name that takes no range).""" + s = _SPECS.get(name) + return s.range_args if s is not None else None + + +def is_elementwise(name: str) -> bool: + """Is this name one call per period?""" + s = _SPECS.get(name) + return s is not None and s.shape == 'elementwise' + + +def per_period_arity(name: str) -> int | None: + """The count of values from which a call with a series among them is + one call per period (``MIN`` / ``MAX``, ``MEDIAN``, ``PERCENTILE_INC``: + 2), or None.""" + s = _SPECS.get(name) + return s.per_period_arity if s is not None else None + + +def admits_in_loop(name: str, n_args: int) -> bool: + """May a loop container run a call of ``name`` over ``n_args`` values + one period at a time? The element-wise names, and a reduction or a + range function over at least its ``per_period_arity`` values — a + reduction's operands, the lines its ranges list.""" + s = _SPECS.get(name) + if s is None: + return False + if s.shape == 'elementwise': + return True + return s.per_period_arity is not None and n_args >= s.per_period_arity + + +def file_prefix(name: str) -> str: + """What the workbook file stores before the name (``''`` or ``'_xlfn.'``).""" + s = _SPECS.get(name) + return s.file_prefix if s is not None else '' + + +def excel_name(name: str) -> str: + """The name the workbook writes — Excel's spelling when it differs.""" + s = _SPECS.get(name) + return s.excel if s is not None else name + + +# ─── Views, for the tests that pin them ───────────────────────────── + + +def passthrough_names() -> frozenset[str]: + """Every name the Excel renderer writes as ``=NAME(args)``.""" + return frozenset(n for n, s in _SPECS.items() if s.passthrough) + + +def range_taking_names() -> frozenset[str]: + """Every name whose list arguments are written as ranges.""" + return frozenset(n for n, s in _SPECS.items() if s.shape in ('reduction', 'range')) + + +def elementwise_names() -> frozenset[str]: + """Every name that is one call per period.""" + return frozenset(n for n, s in _SPECS.items() if s.shape == 'elementwise') + + +def file_prefixes() -> dict[str, str]: + """The names the file stores with a prefix, and the prefix of each.""" + return {n: s.file_prefix for n, s in _SPECS.items() if s.file_prefix} diff --git a/src/modeleon/functions/statistics.py b/src/modeleon/functions/statistics.py new file mode 100644 index 0000000..9b2e2d6 --- /dev/null +++ b/src/modeleon/functions/statistics.py @@ -0,0 +1,1052 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Statistics — ``MEDIAN``, ``STDEV_S``, ``STDEV_P``, ``VAR_S``, ``VAR_P``, +``PERCENTILE_INC``, ``PERCENTILE_EXC``, ``QUARTILE_INC``, ``QUARTILE_EXC``, +``LARGE``, ``SMALL``, ``RANK_EQ``, ``RANK_AVG`` — and the names an older +workbook carries for seven of them: ``STDEV``, ``STDEVP``, ``VAR``, +``VARP``, ``PERCENTILE``, ``QUARTILE``, ``RANK``. + +Each is written as the spreadsheet writes it: ``=MEDIAN(Comps!C3:C7)``, +``=_xlfn.PERCENTILE.INC(Comps!C3:C7, 0.25)``, ``=_xlfn.RANK.EQ(C3, +C3:C7)``. A name Excel spells with a dot is written in Python with an +underscore — ``PERCENTILE_INC`` is ``PERCENTILE.INC`` — and, Excel 2010's, +stored with the ``_xlfn.`` prefix; the older names are the same functions +under their own names (``STDEV`` is ``STDEV_S``, ``RANK`` is ``RANK_EQ``). + +Several series are combined period by period, as ``a + b`` combines +them; one series alone is reduced over its periods; scalars only stay one +number. ``MEDIAN`` and the spreads (``STDEV_*``, ``VAR_*``) over ONE +argument reduce its cells to one number: a row, written as the range of +its cells (``=MEDIAN(B2:M2)``), or a list of scalar lines whose cells +stand one under another — one field down the records of a table, written +as a vertical range, ``Comps!C3:C7``. Over two or more values with a +series among them — rows, a list typed in (one value per period, as the +operators read it), the lines of a list of lines — the call is one per +period over that period's cells: ``MEDIAN(a, b)`` is ``=MEDIAN(C5, C9)`` +in every column; a scalar, a scalar line and a one-value series stand in +every period, series of other lengths are refused, and a line with tracks +lifts per track. Over scalars only it is one number, ``=MEDIAN(B2, B3)``. +The others read ONE range (``PERCENTILE``'s array, ``RANK``'s ref): a +row, or a list of lines. A list of scalar lines is one sample; a list of +two or more lines with a series among them — a field of yearly lines down +a records table, or sibling lines — is one sample per period, and the +call is one per period over that period's cells, written as the run they +form in its column (``=_xlfn.PERCENTILE.INC(Peers!C3:C7, 0.25)``), with +``RANK``'s number this period's value of the line ranked. A list holding +one series alone is refused: the line itself is read over its periods. +Cells that form no run cannot be one range, so the cell carries the +computed value and ``check_compat`` lists it — never a comma list, which +these functions read as further arguments. Inside a loop container a call +over two or more values runs one period at a time. ``k``, ``quart``, +``RANK``'s number and its order are a line — one value, or a series for +one call per period — or a value typed in (a list typed in is such a +series, one value per period, never the spreadsheet's array of ``k``), +read as the spreadsheet reads a number: a blank is 0, +TRUE 1, a date its serial, numeric text its number, other text +``#VALUE!``. ``RANK`` of each record among its peers is one call per +record against the same vertical range: +``mo.RANK_EQ(c.multiple, [c.multiple for c in peers])``. + +The edges, as the spreadsheet reads them. In a range or a line, numbers +and dates count (a date as its serial) and a blank, text or a boolean is +skipped; typed into the call, a number, TRUE / FALSE, numeric text and +None (the 0 the formula holds) count, other text is ``#VALUE!``. An error +is the result: in the reductions the first one in reading order; in the +others, as LibreOffice reads them (Excel documents no order), a ``k`` or +a quart that is an error or out of its range before the range's error — +``LARGE`` / ``SMALL`` hold ``k`` against the count after the range — and +``RANK`` its order, its range, then its number. No number at all +is ``#NUM!`` — ``#N/A`` for ``RANK``, which finds nothing to rank among — +and ``STDEV_S`` / ``VAR_S`` need two numbers, ``STDEV_P`` / ``VAR_P`` one +(``#DIV/0!``). ``PERCENTILE_INC``'s ``k`` lies in [0, 1], interpolating +between the cells at ``(n - 1) * k``; ``PERCENTILE_EXC``'s inside (0, 1) +and at ``(n + 1) * k`` between 1 and n; ``QUARTILE_INC``'s quart is 0 to +4 and ``QUARTILE_EXC``'s 1 to 3, a fraction truncated, each the percentile +at quart / 4 (2 is the median); out of its range a ``k`` or a quart is +``#NUM!``. ``LARGE`` takes a fractional ``k`` up and ``SMALL`` down +(``LARGE(x, 2.5)`` is the third largest, ``SMALL(x, 2.5)`` the second +smallest) — LibreOffice's reading; Excel documents none — and ``k`` past +the count is ``#NUM!``. ``RANK`` finds its number by exact equality, +``order`` 0 (the default) ranking the largest first and any other the +smallest; ties share the best rank (``RANK_EQ``) or their mean +(``RANK_AVG``); a number the range lacks is ``#N/A``. A position within +15 significant digits of a whole one is that position, as the sheet +reads it: ``100 * 0.29``, 28.999999999999996, is the 30th cell. +The middle, a percentile, a quartile and the k-th of dates is a date; a +spread of dates is a number of days. The values a call reads are in one +unit, as ``+`` needs them — two are refused, ``RANK``'s number against +its range too (``k``, a quart and an order are no values) — and the +result is in it: a variance in its square, a rank in none. + +LibreOffice departs from Excel in five places, and the engine keeps +Excel's rule: there a boolean in a range is a number; numeric text typed +into ``MEDIAN`` or a spread is ``#VALUE!``, and so is numeric text as +``LARGE`` / ``SMALL``'s ``k``, typed or in a cell, which LibreOffice +reads from a number only — the engine reads that ``k`` as Excel reads a +value argument, though Excel documents nothing for it; a ``k`` or a +quart out of its range and a range without numbers are written as +``#VALUE!``, not ``#NUM!`` (``RANK``'s: not ``#N/A``); a negative +fractional quart is rounded down, not truncated; the spreads read their +arguments from the last, so of two errors the later one shows. + +Not covered: one line with tracks alone, and a parameter with tracks +(slice a coordinate first), the spreadsheet's array of ``k`` +(``LARGE(x, {1, 2, 3})``, one result per ``k``), and ``MODE``, +``PERCENTRANK``, ``AVEDEV``. +""" + +from __future__ import annotations + +import bisect +import math +import numbers +from datetime import date, timedelta +from typing import Any, Callable, Sequence + +from ..core.expr import Expr, ListExpr, Literal, VarRef +from ..core.multi_variable import MultiVariableBase +from ..core.shape import keys_along, reject_axised +from ..core.tracks import TrackValues +from ..core.unit import Unit +from ..core.variable import Variable +from ..core.variable_ops import _to_serial +from ._helpers import ( + UnitLaw, _per_period, align_keyed, align_located, make_func_var, placed, + refuse_mixed_units, same_unit, shared_unit, src, unit_law, +) +from .criteria import ( + _ERRORS, _cells, _checked, _from_serial, _label, _Lines, _Range, _range, _series_value, + _shared, _unit_items, +) +from .registry import per_period_arity, register +from .text import _number_of + +#: An optional argument the call did not give — left out of the formula too. +_OMITTED: Any = object() + +#: The most arguments a spreadsheet function takes. +_MAX_ARGS = 255 + +# ─── What a statistic reads ───────────────────────────────────────── + + +# A position computed off a number is read to 15 significant digits, as +# the spreadsheet reads it: 2.9999999999999996 is 3. +def _approx(x: Any) -> Any: + if isinstance(x, float) and math.isfinite(x): + return float(f"{x:.15g}") + return x + + +# What a statistic reads of a range, in order — numbers, dates and +# durations — or the first error among them; a blank, text and a boolean +# are skipped. +def _counted(cells: Sequence[Any]) -> list[Any] | str: + out: list[Any] = [] + for cell in cells: + if isinstance(cell, str): + if cell in _ERRORS: + return cell + elif isinstance(cell, (date, timedelta)): + out.append(cell) + elif isinstance(cell, numbers.Real) and not isinstance(cell, bool): + if not math.isfinite(cell): + return '#NUM!' + out.append(cell) + return out + + +# The counted cells sorted, with their serials worked out once for every +# call that ranks a number among them. +class _Ranked(list[Any]): + __slots__ = ('_serials',) + _serials: list[Any] + + def serials(self) -> list[Any]: + try: + return self._serials + except AttributeError: + self._serials = [_to_serial(c) for c in self] + return self._serials + + +# What a range function picks from: its cells counted and sorted, or the +# first error among them. +def _ranked_of(cells: Sequence[Any]) -> Any: + counted = _counted(cells) + return counted if isinstance(counted, str) else _Ranked(sorted(counted, key=_to_serial)) + + +# A k, a quart, a number or an order as the spreadsheet reads a number. An +# error a cell holds is that error; typed into the call, such text is text. +def _number(value: Any, typed: bool) -> Any: + if not typed and isinstance(value, str) and value in _ERRORS: + return value + try: + return _number_of(value) + except TypeError: + return '#VALUE!' + except OverflowError: + return '#NUM!' + + +# A value typed into a reduction's call: a date stays a date (its serial +# counts); any other is read as a number, None as the 0 the formula holds. +def _typed(value: Any) -> Any: + if isinstance(value, (date, timedelta)): + return value + return _number(value, True) + + +def _lines(arg: Any) -> list[Variable]: + if isinstance(arg, Variable): + return [arg] + if isinstance(arg, (list, tuple)): + return [item for item in arg if isinstance(item, Variable)] + return [] + + +# A number computed between cells is a date when they are dates. +def _as_cell(number: float, among: Sequence[Any]) -> Any: + if not math.isfinite(number): + return '#NUM!' + if all(isinstance(c, date) for c in among): + return _from_serial(number) + return number + + +def _type_of(value: Any, otherwise: str) -> str: + fibers = list(value.values()) if isinstance(value, TrackValues) else [value] + for fiber in fibers: + for v in (fiber if isinstance(fiber, list) else [fiber]): + if not isinstance(v, str): + return 'datetime' if isinstance(v, date) else otherwise + return otherwise + + +#: Where a result runs: the combined window its operands were read on by +#: date and the keys they ran along (``_aligned``); either may be None. +_Place = tuple[Any, list[Any] | None] +_NOWHERE: _Place = (None, None) + + +def _result(func: str, nodes: list[Expr], texts: list[str], value: Any, + value_type: str, unit: Any, place: _Place = _NOWHERE) -> Variable: + if isinstance(value, TrackValues): + var_type = 'list' if value.time_length else 'scalar' + else: + var_type = 'list' if isinstance(value, list) else 'scalar' + with _checked(nodes): + out = make_func_var( + func, nodes, value, value_type, var_type, + source_code=f"{func}({', '.join(texts)})", + ) + out._unit = unit + return placed(out, *place) + + +# The values a call reads per period, read as ``a + b`` reads its operands: +# series on other time windows by date (each continued by its own extend= +# rule), lines along keys key by key; ``owners[i]`` is the line ``raws[i]`` +# came from (None for a value typed in). With where the result runs. +def _aligned(where: str, owners: Sequence[Any], raws: list[Any]) -> tuple[list[Any], _Place]: + raws, keys = align_keyed(where, owners, raws) + raws, window = align_located(owners, raws) + return raws, (window, keys) + + +# Where every line of a range read once runs, worked out once: the window +# and the length of its series when all are plain series of one window — +# None where one runs along keys, holds tracks or another window, which +# each call aligns (``_aligned``). +def _range_window(shared: _Lines) -> tuple[Any, int] | None: + first: tuple[Any, int] | None = None + for line, raw in zip(shared.lines, shared.raws): + if not isinstance(raw, list) or keys_along(line) is not None: + return None + loc = line.time + if loc is None or loc.start is None: + return None + if first is None: + first = (loc, len(raw)) + elif (loc, len(raw)) != first: + return None + return first + + +# The values of a call over a range of lines, read with the range's as +# ``_aligned`` reads them. Beside a range read once whose lines are plain +# series of one window, values that add no other window and no keys leave +# nothing to align — each call of a rank per record over a table reads its +# own line, not every peer's window again. +def _with_range(where: str, shared: _Lines | None, ranged: Sequence[Any], + owners: Sequence[Any], cells: list[Any], + raws: list[Any]) -> tuple[list[Any], _Place]: + if shared is not None: + window = shared.keep('window', lambda: _range_window(shared)) + if window is not None and all(_on_window(window, owner, raw) + for owner, raw in zip(owners, raws)): + return [*cells, *raws], _NOWHERE + return _aligned(where, [*ranged, *owners], [*cells, *raws]) + + +def _on_window(window: tuple[Any, int], owner: Any, raw: Any) -> bool: + if not isinstance(owner, Variable): + return True # a value typed in is read as it is + if keys_along(owner) is not None or isinstance(raw, TrackValues): + return False + if not isinstance(raw, list): + return True + loc = owner.time + return loc is not None and (loc, len(raw)) == window + + +# A variance is in the square of its cells' unit; a label squared is none. +# What is no unit — none, or one a loop has not derived yet — stays as is. +def _squared(unit: Any) -> Any: + if not isinstance(unit, Unit): + return unit + unit = unit ** 2 + return None if unit.is_dimensionless else unit + + +# ─── Several series: one call per period ──────────────────────────── + + +def _is_series(x: Any) -> bool: + return isinstance(x, Variable) and _series_value(x._value) + + +# A list or a tuple holding a line is a list of lines (with values typed +# in among them): a range, each line one value of the sample. +def _of_lines(arg: Any) -> bool: + return isinstance(arg, (list, tuple)) and any(isinstance(i, Variable) for i in arg) + + +# Is a call over these values one call per period: as many as the +# function's per_period_arity or more, a series among them? A reduction's +# values are its arguments — a list typed in a series, as the operators +# read it — and the lines of a list of lines; a range function's are the +# lines its range lists. +def _by_period(where: str, values: Sequence[Any], typed_series: bool) -> bool: + count, shaped = 0, False + for v in values: + if _of_lines(v): + count += len(v) + shaped = shaped or any(_is_series(i) for i in v) + else: + count += 1 + shaped = shaped or _is_series(v) or (typed_series and isinstance(v, list) and bool(v)) + return shaped and count >= (per_period_arity(where) or 2) + + +# A list of lines that holds one series alone is no sample per period: the +# line itself is read over its periods. +def _refuse_one_series(where: str, arg: Any, example: str) -> None: + if _of_lines(arg) and len(arg) == 1 and _is_series(arg[0]): + line = _label(arg[0]) + raise ValueError( + f"{where}(): [{line}] lists one line, and a list of lines with a series in " + f"it is one sample per period, which needs two or more values; " + f"{example.format(line=line)}" + ) + + +# The values a call reads per period, in reading order — a line's cells, a +# list of lines' lines and values, a value typed in (a list typed in a +# series of them) — with whether each was typed in, the AST nodes, the +# names in the source, the lines and the line each value came from. +def _per_period_values(args: Sequence[Any], where: str) -> tuple[ + list[Any], list[bool], list[Expr], list[str], list[Variable], list[Any]]: + raws: list[Any] = [] + typed: list[bool] = [] + nodes: list[Expr] = [] + texts: list[str] = [] + lines: list[Variable] = [] + owners: list[Any] = [] + for a in args: + if isinstance(a, Variable): + reject_axised(a, where) + raws.append(a._value) + typed.append(False) + nodes.append(VarRef(a)) + texts.append(src(a)) + lines.append(a) + owners.append(a) + elif isinstance(a, (list, tuple)) and not a: + raise ValueError(f"{where}(): a range needs at least one cell; got {a!r}.") + elif isinstance(a, tuple) or _of_lines(a): + items: list[Expr] = [] + labels: list[str] = [] + for item in a: + if isinstance(item, Variable): + reject_axised(item, where) + raws.append(item._value) + items.append(VarRef(item)) + labels.append(_label(item)) + lines.append(item) + owners.append(item) + elif isinstance(item, (list, tuple, dict, set, MultiVariableBase)): + raise TypeError( + f"{where}(): an item of a list of lines is a line or one value; " + f"got {type(item).__name__}." + ) + else: + # A value among lines is the array's own, read as a cell. + raws.append(item) + items.append(Literal(item)) + labels.append(repr(item)) + owners.append(None) + typed.append(False) + nodes.append(ListExpr(items)) + texts.append(f"[{', '.join(labels)}]") + elif isinstance(a, (dict, set, MultiVariableBase)): + raise TypeError( + f"{where}(): an argument is a row, a list of lines, a line or a value " + f"typed in; got {type(a).__name__}." + ) + else: + raws.append(list(a) if isinstance(a, list) else a) + typed.append(True) + nodes.append(Literal(list(a) if isinstance(a, list) else a)) + texts.append(repr(a)) + owners.append(None) + return raws, typed, nodes, texts, lines, owners + + +# ``call`` over each period's values; a scalar or a one-value series stands +# in every period, other lengths are refused, a line with tracks lifts per +# track. +def _each_period(raws: list[Any], call: Callable[[list[Any]], Any]) -> Any: + if any(isinstance(r, TrackValues) for r in raws): + return TrackValues.lift(lambda *fibers: _per_period(list(fibers), call), *raws) + return _per_period(raws, call) + + +# ─── The reductions ───────────────────────────────────────────────── + + +# Every cell a reduction reads, in reading order, with the AST nodes, the +# names in the source and the one unit its lines may be in. +def _operands(args: tuple[Any, ...], where: str) -> tuple[list[Any], list[Expr], + list[str], Any]: + cells: list[Any] = [] + nodes: list[Expr] = [] + texts: list[str] = [] + for a in args: + if isinstance(a, (Variable, list, tuple)): + rng = _range(a, where) + cells.extend(rng.values) + nodes.append(rng.node) + texts.append(rng.text) + elif isinstance(a, (dict, set, MultiVariableBase)): + raise TypeError( + f"{where}(): an argument is a row, a list of lines, a line or a value " + f"typed in; got {type(a).__name__}." + ) + else: + cells.append(_typed(a)) + nodes.append(Literal(a)) + texts.append(repr(a)) + lines = [line for a in args for line in _lines(a)] + refuse_mixed_units(where, lines) + return cells, nodes, texts, shared_unit(lines) + + +# reduce(the counted cells, or the first error among them) — over every +# cell of every argument, or, over several series, once per period over +# that period's cells; and the AST nodes, the names in the source, the +# unit the lines share and where the result runs. +def _reduction(where: str, args: tuple[Any, ...], reduce: Callable[[list[Any]], Any] + ) -> tuple[Any, list[Expr], list[str], Any, _Place]: + if not args: + raise ValueError( + f"{where}() needs at least one argument. " + f"Syntax: mo.{where}(row) or mo.{where}(a, b, c). Got 0 arguments." + ) + if len(args) > _MAX_ARGS: + raise ValueError( + f"{where}() takes at most {_MAX_ARGS} arguments, as the spreadsheet does; " + f"got {len(args)}. Pass the lines as one list — one range." + ) + shared = _shared(args[0]) if len(args) == 1 else None + if shared is not None: + return _one_reduction(where, args[0], shared, reduce) + if _by_period(where, args, typed_series=True): + raws, typed, nodes, texts, lines, owners = _per_period_values(args, where) + refuse_mixed_units(where, lines) + raws, place = _aligned(where, owners, raws) + + def one(values: list[Any]) -> Any: + counted = _counted([_typed(v) if t else v for v, t in zip(values, typed)]) + return counted if isinstance(counted, str) else reduce(counted) + + return _each_period(raws, one), nodes, texts, shared_unit(lines), place + for a in args: + _refuse_one_series(where, a, f"mo.{where}({{line}}) reduces the line over its periods.") + cells, nodes, texts, unit = _operands(args, where) + counted = _counted(cells) + return ((counted if isinstance(counted, str) else reduce(counted)), nodes, texts, unit, + _NOWHERE) + + +# A reduction over one range read once (``_shared``): what it reads and +# what it gives, as ``_reduction`` reads and refuses them, the value worked +# out once per function — each call gets a list of its own. +def _one_reduction(where: str, arg: Any, shared: _Lines, reduce: Callable[[list[Any]], Any] + ) -> tuple[Any, list[Expr], list[str], Any, _Place]: + def reduced(cells: list[Any]) -> Any: + counted = _counted(cells) + return counted if isinstance(counted, str) else reduce(counted) + + lines = shared.unit_lines()[0] + if (not shared.row and shared.series + and len(shared.lines) >= (per_period_arity(where) or 2)): + rng = shared.keep('per period', lambda: _per_period_range(shared, where)) + refuse_mixed_units(where, lines) + if any(isinstance(raw, TrackValues) for raw in rng.values): + value = _each_period(list(rng.values), reduced) + return value, [rng.node], [rng.text], rng.unit, _NOWHERE + # The lines read on one window and along one set of keys, worked + # out once: the same for every call over them. + cells, place = shared.keep( + 'aligned', lambda: _aligned(where, shared.lines, list(rng.values))) + value = shared.keep(('reduced', where), lambda: _each_period(list(cells), reduced)) + value = list(value) if isinstance(value, list) else value + return value, [rng.node], [rng.text], rng.unit, place + _refuse_one_series(where, arg, f"mo.{where}({{line}}) reduces the line over its periods.") + rng = shared.cells(where) + refuse_mixed_units(where, lines) + value = shared.keep(('reduced', where), lambda: reduced(rng.values)) + return value, [rng.node], [rng.text], rng.unit, _NOWHERE + + +# A list of lines read once as one sample per period: each line's value as +# it holds it, as ``_per_period_values`` reads a list of lines. +def _per_period_range(shared: _Lines, where: str) -> _Range: + for var in shared.lines: + reject_axised(var, where) + return _Range(list(shared.raws), shared.node, shared.text, shared_unit(shared.lines), + shared.first_type, shared) + + +# Each period's cells of a list of lines, counted and sorted, worked out +# once for every call over them — None when a line holds tracks or the lines' +# lengths differ, which each call reads (and refuses) as before. +def _ranked_per_period(raws: list[Any]) -> list[Any] | None: + if any(isinstance(raw, TrackValues) for raw in raws): + return None + lengths = {len(raw) for raw in raws if isinstance(raw, list)} + if not lengths or lengths - {1, max(lengths)}: + return None + ranked: list[Any] = _per_period(list(raws), _ranked_of) + return ranked + + +def _median(ranked: list[Any]) -> Any: + n = len(ranked) + if n % 2: + return ranked[n // 2] + lo, hi = ranked[n // 2 - 1], ranked[n // 2] + return _as_cell((_to_serial(lo) + _to_serial(hi)) / 2, (lo, hi)) + + +# Two passes over exact sums, so 1e15 + 1, + 2, + 3 spread by exactly 1. +def _variance(values: list[Any], ddof: int) -> Any: + n = len(values) + if n <= ddof: + return '#DIV/0!' + try: + mean = math.fsum(values) / n + out = math.fsum((v - mean) ** 2 for v in values) / (n - ddof) + except OverflowError: + return '#NUM!' + return out if math.isfinite(out) else '#NUM!' + + +def _spread(where: str, args: tuple[Any, ...], ddof: int, root: bool) -> Variable: + def spread(counted: list[Any]) -> Any: + value = _variance([_to_serial(c) for c in counted], ddof) + return math.sqrt(value) if root and not isinstance(value, str) else value + + value, nodes, texts, unit, place = _reduction(where, args, spread) + return _result(where, nodes, texts, value, 'float', unit if root else _squared(unit), place) + + +def _middle(counted: list[Any]) -> Any: + return _median(sorted(counted, key=_to_serial)) if counted else '#NUM!' + + +def MEDIAN(*args: Any) -> Variable: + """The middle number of the cells (renders ``=MEDIAN(...)``): of an + even count, the mean of the two middle ones; ``#NUM!`` with no number. + Over ONE row or list of scalar lines, its cells; over two or more values + with a series among them, each period's cells — ``MEDIAN(a, b)`` is one + call per period, as ``a + b`` is; over scalars only, one number. + + Example:: + + mid = mo.MEDIAN([c.a.multiple, c.b.multiple, c.c.multiple]) + # =MEDIAN(Comps!C3:C5) + mo.MEDIAN(revenue) # =MEDIAN(B2:M2) + mo.MEDIAN([p.revenue for p in peers]) # each year: =MEDIAN(Peers!C3:C7) + mo.MEDIAN(plan, actual) # each period: =MEDIAN(C5, C9) + """ + where = "MEDIAN" + value, nodes, texts, unit, place = _reduction(where, args, _middle) + return _result(where, nodes, texts, value, _type_of(value, 'float'), unit, place) + + +def STDEV_S(*args: Any) -> Variable: + """The standard deviation of a sample (renders ``=STDEV.S(...)``, + stored as ``_xlfn.STDEV.S``): over ``n - 1``; ``#DIV/0!`` with fewer + than two numbers. ``mo.STDEV_S(returns)`` → ``=_xlfn.STDEV.S(B2:M2)``.""" + return _spread("STDEV_S", args, 1, True) + + +def STDEV_P(*args: Any) -> Variable: + """The standard deviation of a whole population (renders + ``=STDEV.P(...)``, stored as ``_xlfn.STDEV.P``): over ``n``; + ``#DIV/0!`` with no number.""" + return _spread("STDEV_P", args, 0, True) + + +def VAR_S(*args: Any) -> Variable: + """The variance of a sample (renders ``=VAR.S(...)``, stored as + ``_xlfn.VAR.S``): over ``n - 1``; ``#DIV/0!`` with fewer than two + numbers. In the square of the cells' unit.""" + return _spread("VAR_S", args, 1, False) + + +def VAR_P(*args: Any) -> Variable: + """The variance of a whole population (renders ``=VAR.P(...)``, + stored as ``_xlfn.VAR.P``): over ``n``; ``#DIV/0!`` with no number.""" + return _spread("VAR_P", args, 0, False) + + +def STDEV(*args: Any) -> Variable: + """:func:`STDEV_S` under the name an older workbook carries (renders + ``=STDEV(...)``).""" + return _spread("STDEV", args, 1, True) + + +def STDEVP(*args: Any) -> Variable: + """:func:`STDEV_P` under the name an older workbook carries (renders + ``=STDEVP(...)``).""" + return _spread("STDEVP", args, 0, True) + + +def VAR(*args: Any) -> Variable: + """:func:`VAR_S` under the name an older workbook carries (renders + ``=VAR(...)``).""" + return _spread("VAR", args, 1, False) + + +def VARP(*args: Any) -> Variable: + """:func:`VAR_P` under the name an older workbook carries (renders + ``=VARP(...)``).""" + return _spread("VARP", args, 0, False) + + +# ─── One range ────────────────────────────────────────────────────── + + +# A k, a quart, a number or an order: a line — one value, or a series for +# one call per period — or a value typed in, and whether it was. Empty text +# typed into a line is the blank cell the workbook holds. +def _param(value: Any, where: str, name: str) -> tuple[Any, Expr, str, bool]: + if isinstance(value, Variable): + reject_axised(value, where) + raw = value._value + if hasattr(raw, 'roles'): + raise ValueError( + f"{where}(): {src(value)} has tracks — slice a coordinate first " + f"(.at(track='...'))." + ) + cells = _cells(value, raw) + return (cells if isinstance(raw, list) else cells[0]), VarRef(value), src(value), False + if isinstance(value, (dict, set, tuple, MultiVariableBase)) or ( + isinstance(value, list) and any(isinstance(v, Variable) for v in value)): + raise TypeError( + f"{where}(): {name} is one value, or a line (one call per period); " + f"got {type(value).__name__}." + ) + return value, Literal(value), repr(value), True + + +# pick(the range's cells sorted ascending — or its error —, the parameters +# read as numbers) per period; the range is argument ``at`` of the call: +# read whole by every period's call, or, a list of lines with series among +# them, this period's cells of each. The values compared or picked among +# are in one unit: the range's lines and those of ``compared`` (RANK's +# number); a k or an order is not one. Series on other time windows are +# read by date, lines along keys key by key (``_aligned``). +def _one_range(where: str, array: Any, at: int, params: list[tuple[str, Any]], + pick: Callable[..., Any], compared: Sequence[Any] = () + ) -> tuple[Any, _Range, list[Expr], list[str], _Place]: + shared = _shared(array) + if shared is not None: + by_period = (not shared.row and shared.series + and len(shared.lines) >= (per_period_arity(where) or 2)) + else: + by_period = _by_period(where, [array], typed_series=False) + ranged_owners: list[Any] = [] + if by_period: + if shared is not None: + rng = shared.keep('per period', lambda: _per_period_range(shared, where)) + ranged_owners = list(shared.lines) + else: + cells, _, items, labels, lines, ranged_owners = _per_period_values([array], where) + rng = _Range(cells, items[0], labels[0], shared_unit(lines), lines[0].value_type) + else: + shown = [name for name, _ in params] + shown.insert(at, "{line}") + _refuse_one_series( + where, array, f"mo.{where}({', '.join(shown)}) reads the line over its periods.") + rng = shared.cells(where) if shared is not None else _range(array, where) + ranged = list(shared.unit_lines()[0]) if shared is not None else _lines(array) + refuse_mixed_units(where, [line for a in compared for line in _lines(a)] + ranged) + raws: list[Any] = [] + typed: list[bool] = [] + nodes: list[Expr] = [] + texts: list[str] = [] + owners: list[Any] = [] + for name, value in params: + raw, node, text, is_typed = _param(value, where, name) + raws.append(raw) + typed.append(is_typed) + nodes.append(node) + texts.append(text) + owners.append(value if isinstance(value, Variable) else None) + nodes.insert(at, rng.node) + texts.insert(at, rng.text) + + def numbers(values: list[Any]) -> list[Any]: + return [_number(v, t) for v, t in zip(values, typed)] + + if by_period: + n = len(rng.values) + values, place = _with_range(where, shared, ranged_owners, owners, rng.values, raws) + sorted_by = (shared.keep('ranked per period', lambda: _ranked_per_period(rng.values)) + if shared is not None and place == _NOWHERE else None) + if sorted_by is not None: + def picked(values: list[Any]) -> Any: + return pick(values[0], *numbers(values[1:])) + + return _per_period([sorted_by, *raws], picked), rng, nodes, texts, place + + def each(values: list[Any]) -> Any: + return pick(_ranked_of(values[:n]), *numbers(values[n:])) + + return _each_period(values, each), rng, nodes, texts, place + + ranked = (shared.keep('ranked', lambda: _ranked_of(rng.values)) if shared is not None + else _ranked_of(rng.values)) + + def one(values: list[Any]) -> Any: + return pick(ranked, *numbers(values)) + + raws, place = _aligned(where, owners, raws) + return _per_period(raws, one), rng, nodes, texts, place + + +# The cell at 0-based position ``at`` of the sorted cells, or the point +# between two neighbours on the line joining them. +def _interpolated(ranked: list[Any], at: float) -> Any: + lo = math.floor(_approx(at)) + part = at - lo + if part <= 0 or lo + 1 >= len(ranked): + return ranked[lo] + a, b = ranked[lo], ranked[lo + 1] + x, y = _to_serial(a), _to_serial(b) + return _as_cell(x + part * (y - x), (a, b)) + + +def _inclusive(ranked: list[Any] | str, k: Any) -> Any: + if isinstance(k, str): + return k + if not 0 <= k <= 1: + return '#NUM!' + if isinstance(ranked, str): + return ranked + if not ranked: + return '#NUM!' + return _interpolated(ranked, k * (len(ranked) - 1)) + + +# The exclusive percentile lies at (n + 1) * k, between 1 and n. +def _between(ranked: list[Any], k: Any) -> Any: + n = len(ranked) + rank = k * (n + 1) + if rank < 1 or rank > n: + return '#NUM!' + return _interpolated(ranked, rank - 1) + + +def _exclusive(ranked: list[Any] | str, k: Any) -> Any: + if isinstance(k, str): + return k + if not 0 < k < 1: + return '#NUM!' + if isinstance(ranked, str): + return ranked + if not ranked: + return '#NUM!' + return _between(ranked, k) + + +def _quartile(ranked: list[Any] | str, quart: Any, *, inclusive: bool) -> Any: + if isinstance(quart, str): + return quart + q = math.trunc(_approx(quart)) + if not (0 <= q <= 4 if inclusive else 1 <= q <= 3): + return '#NUM!' + if isinstance(ranked, str): + return ranked + if not ranked: + return '#NUM!' + if q == 2: + return _median(ranked) + if inclusive: + return _interpolated(ranked, 0.25 * q * (len(ranked) - 1)) + return _between(ranked, 0.25 * q) + + +def _kth(ranked: list[Any] | str, k: Any, *, largest: bool) -> Any: + if isinstance(k, str): + return k + if isinstance(ranked, str): + return ranked + n = len(ranked) + if not n: + return '#NUM!' + at = math.ceil(_approx(k)) if largest else math.floor(_approx(k)) + if not 1 <= at <= n: + return '#NUM!' + return ranked[n - at] if largest else ranked[at - 1] + + +def _ranked_type(rng: _Range) -> str: + return rng.value_type if rng.value_type in ('int', 'float') else 'float' + + +def PERCENTILE_INC(array: Any, k: Any) -> Variable: + """The ``k``-th percentile of the cells, ``k`` in [0, 1] (renders + ``=PERCENTILE.INC(...)``, stored as ``_xlfn.PERCENTILE.INC``): between + the cells at ``(n - 1) * k`` of the sorted numbers; ``#NUM!`` for a + ``k`` outside [0, 1] or no number. + + Example:: + + q1 = mo.PERCENTILE_INC([c.a.multiple, c.b.multiple, c.c.multiple], 0.25) + # =_xlfn.PERCENTILE.INC(Comps!C3:C5, 0.25) + mo.PERCENTILE_INC([p.revenue for p in peers], 0.25) + # each year: =_xlfn.PERCENTILE.INC(Peers!C3:C5, 0.25) + """ + where = "PERCENTILE_INC" + value, rng, nodes, texts, place = _one_range(where, array, 0, [("k", k)], _inclusive) + return _result(where, nodes, texts, value, _type_of(value, 'float'), rng.unit, place) + + +def PERCENTILE_EXC(array: Any, k: Any) -> Variable: + """The ``k``-th percentile of the cells, ``k`` inside (0, 1), the ends + left out (renders ``=PERCENTILE.EXC(...)``, stored as + ``_xlfn.PERCENTILE.EXC``): between the cells at ``(n + 1) * k``, which + must lie between 1 and n; else ``#NUM!``.""" + where = "PERCENTILE_EXC" + value, rng, nodes, texts, place = _one_range(where, array, 0, [("k", k)], _exclusive) + return _result(where, nodes, texts, value, _type_of(value, 'float'), rng.unit, place) + + +def QUARTILE_INC(array: Any, quart: Any) -> Variable: + """A quartile of the cells (renders ``=QUARTILE.INC(...)``, stored as + ``_xlfn.QUARTILE.INC``): ``quart`` 0 the smallest, 1 the first + quartile, 2 the median, 3 the third, 4 the largest — the inclusive + percentile at quart / 4; a fraction is truncated; ``#NUM!`` outside + 0 to 4.""" + where = "QUARTILE_INC" + value, rng, nodes, texts, place = _one_range( + where, array, 0, [("quart", quart)], + lambda ranked, q: _quartile(ranked, q, inclusive=True)) + return _result(where, nodes, texts, value, _type_of(value, 'float'), rng.unit, place) + + +def QUARTILE_EXC(array: Any, quart: Any) -> Variable: + """A quartile of the cells, the ends left out (renders + ``=QUARTILE.EXC(...)``, stored as ``_xlfn.QUARTILE.EXC``): ``quart`` 1 + to 3, the exclusive percentile at quart / 4; a fraction is truncated; + ``#NUM!`` outside 1 to 3 or where too few cells hold that + percentile.""" + where = "QUARTILE_EXC" + value, rng, nodes, texts, place = _one_range( + where, array, 0, [("quart", quart)], + lambda ranked, q: _quartile(ranked, q, inclusive=False)) + return _result(where, nodes, texts, value, _type_of(value, 'float'), rng.unit, place) + + +def PERCENTILE(array: Any, k: Any) -> Variable: + """:func:`PERCENTILE_INC` under the name an older workbook carries + (renders ``=PERCENTILE(...)``).""" + where = "PERCENTILE" + value, rng, nodes, texts, place = _one_range(where, array, 0, [("k", k)], _inclusive) + return _result(where, nodes, texts, value, _type_of(value, 'float'), rng.unit, place) + + +def QUARTILE(array: Any, quart: Any) -> Variable: + """:func:`QUARTILE_INC` under the name an older workbook carries + (renders ``=QUARTILE(...)``).""" + where = "QUARTILE" + value, rng, nodes, texts, place = _one_range( + where, array, 0, [("quart", quart)], + lambda ranked, q: _quartile(ranked, q, inclusive=True)) + return _result(where, nodes, texts, value, _type_of(value, 'float'), rng.unit, place) + + +def LARGE(array: Any, k: Any) -> Variable: + """The ``k``-th largest cell (renders ``=LARGE(...)``): ``k`` 1 is the + largest; a fractional ``k`` is taken up; ``#NUM!`` for ``k`` below 1 + or past the count. The cell comes back as it holds it — a date stays + a date. + + Example:: + + runner_up = mo.LARGE(revenue, 2) # =LARGE(B2:M2, 2) + """ + where = "LARGE" + value, rng, nodes, texts, place = _one_range( + where, array, 0, [("k", k)], lambda ranked, n: _kth(ranked, n, largest=True)) + return _result(where, nodes, texts, value, _type_of(value, _ranked_type(rng)), rng.unit, + place) + + +def SMALL(array: Any, k: Any) -> Variable: + """The ``k``-th smallest cell (renders ``=SMALL(...)``): ``k`` 1 is the + smallest; a fractional ``k`` is taken down; ``#NUM!`` for ``k`` below + 1 or past the count.""" + where = "SMALL" + value, rng, nodes, texts, place = _one_range( + where, array, 0, [("k", k)], lambda ranked, n: _kth(ranked, n, largest=False)) + return _result(where, nodes, texts, value, _type_of(value, _ranked_type(rng)), rng.unit, + place) + + +# The arguments are read as LibreOffice reads them: the order, the range, +# then the number. +def _ranking(average: bool) -> Callable[..., Any]: + def rank(ranked: list[Any] | str, number: Any, order: Any = 0) -> Any: + if isinstance(order, str): + return order + if isinstance(ranked, str): + return ranked + if not ranked: + return '#N/A' + if isinstance(number, str): + return number + keys = (ranked.serials() if isinstance(ranked, _Ranked) + else [_to_serial(c) for c in ranked]) + below = bisect.bisect_left(keys, number) + equal = bisect.bisect_right(keys, number) - below + if not equal: + return '#N/A' + first = (below if order != 0 else len(keys) - below - equal) + 1 + return first + (equal - 1) / 2 if average else first + return rank + + +def _rank(where: str, number: Any, ref: Any, order: Any, average: bool) -> Variable: + params: list[tuple[str, Any]] = [("number", number)] + if order is not _OMITTED: + params.append(("order", order)) + value, _rng, nodes, texts, place = _one_range( + where, ref, 1, params, _ranking(average), (number,)) + return _result(where, nodes, texts, value, 'float' if average else 'int', None, place) + + +def RANK_EQ(number: Any, ref: Any, order: Any = _OMITTED) -> Variable: + """The rank of ``number`` among the cells of ``ref`` (renders + ``=RANK.EQ(...)``, stored as ``_xlfn.RANK.EQ``): the largest is 1 + unless ``order`` is non-zero, when the smallest is; ties share the + best rank; ``#N/A`` when ``ref`` lacks the number. + + Example:: + + for c in peers: # one call per record, one range + c.rank = mo.RANK_EQ(c.multiple, [p.multiple for p in peers]) + # =_xlfn.RANK.EQ(C3, C3:C7), =_xlfn.RANK.EQ(C4, C3:C7), … + # A yearly line among its peers' — each year's rank: + c.rank = mo.RANK_EQ(c.revenue, [p.revenue for p in peers]) + # =_xlfn.RANK.EQ(C3, C3:C7), =_xlfn.RANK.EQ(D3, D3:D7), … + """ + return _rank("RANK_EQ", number, ref, order, False) + + +def RANK_AVG(number: Any, ref: Any, order: Any = _OMITTED) -> Variable: + """The rank of ``number`` among the cells of ``ref`` (renders + ``=RANK.AVG(...)``, stored as ``_xlfn.RANK.AVG``), ties sharing the + mean of their ranks: 5, 5, 3 rank 1.5, 1.5, 3.""" + return _rank("RANK_AVG", number, ref, order, True) + + +def RANK(number: Any, ref: Any, order: Any = _OMITTED) -> Variable: + """:func:`RANK_EQ` under the name an older workbook carries (renders + ``=RANK(...)``).""" + return _rank("RANK", number, ref, order, False) + + +# The unit a call gives, the law a loop container derives its rows' units +# by again: the values a reduction reads (a list's lines each) share one +# unit, the call's — a variance's its square; the values a range function +# compares or picks among (RANK's number and its range) share one, the +# call's — a rank's none. +def _values_law(where: str, square: bool = False) -> UnitLaw: + def law(args: Sequence[Any], unit_of: Callable[[Any], Any]) -> Any: + unit = same_unit(where, [n for a in args for n in _unit_items(a, unit_of)], unit_of) + return _squared(unit) if square else unit + return law + + +def _range_law(where: str, at: int) -> UnitLaw: + def law(args: Sequence[Any], unit_of: Callable[[Any], Any]) -> Any: + nodes = [*args[:at], *_unit_items(args[at], unit_of)] if at < len(args) else list(args) + unit = same_unit(where, nodes, unit_of) + return None if at else unit + return law + + +for _name in ("MEDIAN", "STDEV_S", "STDEV_P", "STDEV", "STDEVP"): + unit_law(_name, _values_law(_name)) +for _name in ("VAR_S", "VAR_P", "VAR", "VARP"): + unit_law(_name, _values_law(_name, square=True)) +for _name in ("PERCENTILE_INC", "PERCENTILE_EXC", "QUARTILE_INC", "QUARTILE_EXC", "PERCENTILE", + "QUARTILE", "LARGE", "SMALL"): + unit_law(_name, _range_law(_name, 0)) +for _name in ("RANK_EQ", "RANK_AVG", "RANK"): + unit_law(_name, _range_law(_name, 1)) + +# Two values or more, a series among them — a reduction's operands, the +# lines a range lists — make a call one per period. +register("MEDIAN", shape="reduction", per_period_arity=2) +register("STDEV_S", shape="reduction", excel="STDEV.S", file_prefix="_xlfn.", per_period_arity=2) +register("STDEV_P", shape="reduction", excel="STDEV.P", file_prefix="_xlfn.", per_period_arity=2) +register("VAR_S", shape="reduction", excel="VAR.S", file_prefix="_xlfn.", per_period_arity=2) +register("VAR_P", shape="reduction", excel="VAR.P", file_prefix="_xlfn.", per_period_arity=2) +register("STDEV", shape="reduction", per_period_arity=2) +register("STDEVP", shape="reduction", per_period_arity=2) +register("VAR", shape="reduction", per_period_arity=2) +register("VARP", shape="reduction", per_period_arity=2) +register("PERCENTILE_INC", shape="range", range_args=(0,), excel="PERCENTILE.INC", + file_prefix="_xlfn.", per_period_arity=2) +register("PERCENTILE_EXC", shape="range", range_args=(0,), excel="PERCENTILE.EXC", + file_prefix="_xlfn.", per_period_arity=2) +register("QUARTILE_INC", shape="range", range_args=(0,), excel="QUARTILE.INC", + file_prefix="_xlfn.", per_period_arity=2) +register("QUARTILE_EXC", shape="range", range_args=(0,), excel="QUARTILE.EXC", + file_prefix="_xlfn.", per_period_arity=2) +register("PERCENTILE", shape="range", range_args=(0,), per_period_arity=2) +register("QUARTILE", shape="range", range_args=(0,), per_period_arity=2) +register("LARGE", shape="range", range_args=(0,), per_period_arity=2) +register("SMALL", shape="range", range_args=(0,), per_period_arity=2) +register("RANK_EQ", shape="range", range_args=(1,), excel="RANK.EQ", file_prefix="_xlfn.", + per_period_arity=2) +register("RANK_AVG", shape="range", range_args=(1,), excel="RANK.AVG", file_prefix="_xlfn.", + per_period_arity=2) +register("RANK", shape="range", range_args=(1,), per_period_arity=2) diff --git a/src/modeleon/functions/text.py b/src/modeleon/functions/text.py index 2671155..d49c6ca 100644 --- a/src/modeleon/functions/text.py +++ b/src/modeleon/functions/text.py @@ -1,17 +1,55 @@ # SPDX-License-Identifier: Apache-2.0 -"""Text helpers — ``LEN``, ``UPPER``, ``LOWER``, ``CONCAT``. +"""Text helpers — ``LEN``, ``UPPER``, ``LOWER``, ``CONCAT``, ``LEFT``, +``RIGHT``, ``MID``, ``TRIM``, ``FIND``, ``SEARCH``, ``SUBSTITUTE``, +``VALUE``, ``TEXT``, ``TEXTJOIN``, ``REPT``. **Invariant:** every call returns a :class:`Variable` with a tracked formula — even when all inputs are plain strings. ``mo.UPPER("hello")`` produces ``=UPPER("hello")`` in Excel, not a bare ``"HELLO"``. + +The helpers from ``LEFT`` on read an operand as the spreadsheet reads +the cell: a blank is empty text (0 where a number is asked for), TRUE +and FALSE are the words, a number is its General text as Excel writes +it — 15 significant digits, written out in full up to 1E20 and down to +a fraction whose spelling fits 20 characters (``0.00001``, +``1234567890123460``), E notation past them (``1E+20``, +``1.23456789012346E-05``) — and a date is its serial number: ``LEFT`` +of a date cell gives the digits of 46023, as the workbook will; +``TEXT(d, 'yyyy-mm-dd')`` spells it. A blank given as a literal where +text is read (``None``, or a gap in a list) is written to the workbook +as empty text, which is what it reads as. A count or a position +(``LEFT``'s ``num_chars``, ``MID``'s ``start_num``) is truncated to a +whole number; numeric text is read as its number, other text is +``#VALUE!``. The first spreadsheet error among a period's operands +(``#VALUE!``, ``#DIV/0!``, …) is that period's result; text that only +starts with ``#`` — a ``'#12'`` label, a ``'#,##0'`` format, the +``'#'`` of a ``REPT`` bar — is text, while text that spells an error +exactly (``'#N/A'``) is read as that error: the engine carries errors as +their text. A result carries no unit: a label, a position and a number +read from text are not amounts. """ from __future__ import annotations -from typing import Any, Callable, Union +import decimal +import functools +import logging +import math +import numbers +import re +import sys +from datetime import date, datetime, timedelta +from typing import Any, Callable, Iterable, Union +from ..core.expr import Literal, VarRef from ..core.variable import Variable -from ._helpers import make_func_var, src, text_operand +from ._helpers import ( + align_keyed, align_located, elementwise, is_series, make_func_var, placed, + refuse_unpaired, src, text_operand, +) +from .registry import register + +logger = logging.getLogger(__name__) TextOperand = Union[Variable, str] @@ -27,7 +65,7 @@ def _unary_text_func( a plain string. """ def helper(value: TextOperand) -> Variable: - raw, expr = text_operand(value) + raw, expr = text_operand(_text_arg(value)) # A blank cell reads as empty text, as in Excel (LEN of a blank # is 0, UPPER of a blank is ""). if isinstance(raw, list): @@ -36,10 +74,11 @@ def helper(value: TextOperand) -> Variable: else: calc = py_fn('' if raw is None else raw) var_type = 'scalar' - return make_func_var( + out = make_func_var( func_name, [expr], calc, result_value_type, var_type, source_code=f"{func_name}({src(value)})", ) + return placed(out, None, align_keyed(func_name, [value], [raw])[1]) return helper @@ -72,8 +111,10 @@ def CONCAT(*args: TextOperand) -> Variable: Element-wise over list (period-indexed) operands, so ``mo.CONCAT(quarter, "Q ", year, " ", phase)`` over per-period - Variables yields per-period labels (``"1Q 2022 A"``, …). Always - returns a Variable carrying the ``=CONCAT(...)`` formula. + Variables yields per-period labels (``"1Q 2022 A"``, …); lines on + other time windows are read by date, as ``a + b`` reads them, and + lines along keys key by key. Always returns a Variable carrying the + ``=CONCAT(...)`` formula. """ if not args: raise ValueError( @@ -84,9 +125,12 @@ def CONCAT(*args: TextOperand) -> Variable: raws, expr_args = [], [] for a in args: - raw, expr = text_operand(a) + raw, expr = text_operand(_text_arg(a)) raws.append(raw) expr_args.append(expr) + raws, keys = align_keyed("CONCAT", args, raws) + raws, window = align_located(args, raws) + refuse_unpaired(args, raws, window) if any(isinstance(r, list) for r in raws): n = max(len(r) for r in raws if isinstance(r, list)) @@ -103,7 +147,651 @@ def at(r: Any, i: int) -> Any: var_type = 'scalar' source_code = "CONCAT(" + ", ".join(src(a) for a in args) + ")" - return make_func_var( + out = make_func_var( "CONCAT", expr_args, value, 'string', var_type, source_code=source_code, ) + return placed(out, window, keys) + + +# ─── Reading an operand as the spreadsheet does ───────────────────── + +#: The longest text a cell holds. +_MAX_TEXT = 32767 +#: The spreadsheet's error values, as arithmetic carries them. Text that +#: only starts with ``#`` is text to these helpers (module docstring). +_ERRORS = frozenset({'#DIV/0!', '#VALUE!', '#NUM!', '#N/A', '#NAME?', '#REF!', '#NULL!'}) +#: Day 0 of the spreadsheet's calendar: 1 January 1900 is serial 1. +_SERIAL_EPOCH = date(1899, 12, 30) +#: The 15 significant digits a cell shows, a half in the 16th going away +#: from zero; and room for every digit of a number written out in full. +_SHOWN = decimal.Context(prec=15, rounding=decimal.ROUND_HALF_UP) +_WIDE = decimal.Context(prec=decimal.MAX_PREC, rounding=decimal.ROUND_HALF_UP, + Emax=decimal.MAX_EMAX, Emin=decimal.MIN_EMIN) + + +def _serial(d: date) -> int | float: + """A date as the spreadsheet holds it — its day count from the epoch, + a time of day as a fraction. Dates from 1 March 1900 on: the + 29 February 1900 Excel counts is not imitated.""" + if isinstance(d, datetime): + days = (d.replace(tzinfo=None) - datetime(1899, 12, 30)) / timedelta(days=1) + return int(days) if days.is_integer() else days + return (d - _SERIAL_EPOCH).days + + +def _error_among(values: Iterable[Any]) -> Any: + """The first spreadsheet error among the values, else None.""" + return next((v for v in values if isinstance(v, str) and v in _ERRORS), None) + + +def _shown(x: float) -> decimal.Decimal: + """The number as a cell shows it, to 15 significant digits; a number + no cell holds (infinite, not a number) is ``OverflowError`` (``#NUM!``).""" + if not math.isfinite(x): + raise OverflowError(f"{x!r} is not a number a cell holds") + return _SHOWN.plus(decimal.Decimal(x)) + + +def _general(x: float) -> str: + """A number as General text, as Excel writes it (module docstring).""" + if x == 0: + return '0' + shown = _shown(x) + digits = ''.join(map(str, shown.as_tuple().digits)).rstrip('0') + point = shown.adjusted() # the power of ten of the first digit + if point > 19 or (point < 0 and 1 - point + len(digits) > 20): + mantissa = digits[0] + ('.' + digits[1:] if len(digits) > 1 else '') + body = f"{mantissa}E{'+' if point >= 0 else '-'}{abs(point):02d}" + elif point < 0: + body = '0.' + '0' * (-point - 1) + digits + elif len(digits) > point + 1: + body = digits[:point + 1] + '.' + digits[point + 1:] + else: + body = digits + '0' * (point + 1 - len(digits)) + return ('-' if x < 0 else '') + body + + +def _text_arg(a: Any) -> Any: + """A literal blank where text is read, as the empty text it reads as: + the workbook spells a literal None as 0, whose text is "0". A + Variable stays as it is — its blank is a blank cell.""" + if a is None: + return '' + if isinstance(a, list): + return ['' if v is None else v for v in a] + return a + + +def _text_of(v: Any) -> str: + """The text the spreadsheet reads a value as (module docstring).""" + if v is None: + return '' + if isinstance(v, str): + return v + if isinstance(v, bool): + return 'TRUE' if v else 'FALSE' + if isinstance(v, date): + v = _serial(v) + if isinstance(v, timedelta): # a duration is its count of days + v = v / timedelta(days=1) + if isinstance(v, numbers.Real): + return _general(float(v)) # a cell holds a double, whole or not + return str(v) + + +#: Numeric text: an optional sign, spaces allowed after it, digits with +#: or without thousands separators (the first group of any length, the +#: others of three), a decimal part, an exponent. ASCII digits only. +_NUMBER_TEXT = re.compile( + r'[+-]?\s*(?:[0-9]+(?:,[0-9]{3})*(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?') +#: A date written the one way every locale reads it — ``2026-01-31`` — +#: and a time of day after it. +_ISO_DATE = re.compile(r'([0-9]{4}-[0-9]{2}-[0-9]{2})(?:\s+(.+))?') +#: A time of day: hours, minutes and seconds (``12:30``, ``12:30:45.5``), +#: AM or PM after them (``1:30 PM``, ``12 AM``). +_TIME = re.compile( + r'([0-9]+)(?::([0-9]{1,2})(?::([0-9]{1,2}(?:\.[0-9]*)?))?)?(?:\s*([AP]M))?', re.IGNORECASE) + + +def _day_fraction(text: str) -> float | None: + """The fraction of a day time text spells (``'12:00'`` is 0.5), or + None. Minutes and seconds stay below 60; with AM or PM the hours + are 0 to 12, without them any count (``'25:00'`` is a day and an hour).""" + match = _TIME.fullmatch(text) + if match is None or (match[2] is None and match[4] is None): # a bare number + return None + hours, minutes, seconds = int(match[1]), int(match[2] or 0), float(match[3] or 0) + if minutes > 59 or seconds >= 60: + return None + if match[4] is not None: + if hours > 12: + return None + hours = hours % 12 + (12 if match[4].upper() == 'PM' else 0) + return (hours * 3600 + minutes * 60 + seconds) / 86400 + + +def _parse_number(text: str) -> int | float: + """The number numeric text spells — ``'1,234'``, ``'12%'``, + ``'(1,234)'``, ``'1e3'``, a time of day (the fraction of a day), an + ISO date with or without a time (its serial) — or ``TypeError`` (the + spreadsheet's ``#VALUE!``); a number too large for a cell is + ``OverflowError`` (``#NUM!``). Other date text, a currency sign and + the words TRUE and FALSE are not read: the spreadsheet would, by its + locale.""" + s = text.strip() + iso = _ISO_DATE.fullmatch(s) + if iso is not None: + try: + day = _serial(date.fromisoformat(iso[1])) + except ValueError: + raise TypeError(f"{text!r} is not a date") from None + fraction = 0.0 if iso[2] is None else _day_fraction(iso[2]) + if fraction is None: + raise TypeError(f"{text!r} is not a date and time") + return day if iso[2] is None else day + fraction + # A plus sign may lead a time, as it leads a number; a time of day + # is never negative. + fraction = _day_fraction(s[1:] if s[:1] == '+' else s) + if fraction is not None: + return fraction + percent = s.endswith('%') # read before the parentheses: '(5%)' is no number + if percent: + s = s[:-1].strip() + negative = len(s) > 1 and s[0] == '(' and s[-1] == ')' + if negative: + s = s[1:-1].strip() + if _NUMBER_TEXT.fullmatch(s) is None or (negative and s[:1] in ('+', '-')): + raise TypeError(f"{text!r} is not a number") + plain = ''.join(s.split()).replace(',', '') + exponent = 'e' in plain.lower() + if percent and exponent: + raise TypeError(f"{text!r} is not a number") + # An integral text is an int while a cell holds it exactly — 15 digits. + whole = not (percent or exponent or '.' in plain) + x: int | float = (int(plain) if whole and len(plain.lstrip('+-').lstrip('0')) <= 15 + else float(plain)) + if abs(x) > sys.float_info.max: + raise OverflowError(f"{text!r} is too large for a cell") + if negative: + x = -x + return x / 100 if percent else x + + +def _number_of(v: Any) -> int | float: + """The number the spreadsheet reads an operand as: a blank is 0, + TRUE 1, a date its serial, a duration its days, numeric text its + number.""" + if v is None: + return 0 + if isinstance(v, bool): + return int(v) + if isinstance(v, str): + return _parse_number(v) + if isinstance(v, date): + return _serial(v) + if isinstance(v, timedelta): + return v / timedelta(days=1) + if isinstance(v, float) and not math.isfinite(v): + raise OverflowError(f"{v!r} is not a number a cell holds") + if isinstance(v, (int, float)): + return v + if isinstance(v, numbers.Real): + return float(v) + raise TypeError(f"{v!r} is not a number") + + +def _count_of(v: Any) -> int: + """A count or a position, truncated to a whole number as the + spreadsheet truncates it (``LEFT(text, 2.9)`` takes 2).""" + return math.trunc(_number_of(v)) + + +def _flag_of(v: Any) -> bool: + """TRUE or FALSE as the spreadsheet reads a logical operand: a number + is TRUE unless it is zero, a blank is FALSE, the words TRUE and + FALSE are themselves; other text is ``TypeError`` (``#VALUE!``).""" + if isinstance(v, str) and v.strip().upper() in ('TRUE', 'FALSE'): + return v.strip().upper() == 'TRUE' + return _number_of(v) != 0 + + +# ─── One cell at a time ───────────────────────────────────────────── + + +def _left(text: Any, num_chars: Any = 1) -> Any: + t, k = _text_of(text), _count_of(num_chars) + return '#VALUE!' if k < 0 else t[:k] + + +def _right(text: Any, num_chars: Any = 1) -> Any: + t, k = _text_of(text), _count_of(num_chars) + return '#VALUE!' if k < 0 else t[max(len(t) - k, 0):] + + +def _mid(text: Any, start_num: Any, num_chars: Any) -> Any: + t, s, k = _text_of(text), _count_of(start_num), _count_of(num_chars) + return '#VALUE!' if s < 1 or k < 0 else t[s - 1:s - 1 + k] + + +def _trim(text: Any) -> str: + # Spaces only — not tabs, not the non-breaking space — and a run of + # them inside the text collapses to one, as Excel's TRIM collapses it. + return ' '.join(part for part in _text_of(text).split(' ') if part) + + +def _wildcard_pattern(find: str) -> str: + """``SEARCH``'s pattern as a regular expression: ``?`` one character, + ``*`` any run, ``~`` makes the character after it literal (whatever + it is); a last ``~`` is itself.""" + out: list[str] = [] + i = 0 + while i < len(find): + c = find[i] + if c == '~' and i + 1 < len(find): + out.append(re.escape(find[i + 1])) + i += 2 + continue + out.append('.' if c == '?' else '.*' if c == '*' else re.escape(c)) + i += 1 + return ''.join(out) + + +def _position(find: Any, within: Any, start: Any, wildcards: bool) -> Any: + f, w, s = _text_of(find), _text_of(within), _count_of(start) + if s < 1 or s > len(w): + return '#VALUE!' + if not f: + # Excel's rule: empty find_text is found at start_num. LibreOffice + # says #VALUE!. + return s + if wildcards: + match = re.compile(_wildcard_pattern(f), re.IGNORECASE | re.DOTALL).search(w, s - 1) + at = match.start() if match else -1 + else: + at = w.find(f, s - 1) + return at + 1 if at >= 0 else '#VALUE!' + + +def _find(find_text: Any, within_text: Any, start_num: Any = 1) -> Any: + return _position(find_text, within_text, start_num, wildcards=False) + + +def _search(find_text: Any, within_text: Any, start_num: Any = 1) -> Any: + return _position(find_text, within_text, start_num, wildcards=True) + + +def _substitute(text: Any, old_text: Any, new_text: Any, instance_num: Any = None) -> Any: + t, old, new = _text_of(text), _text_of(old_text), _text_of(new_text) + if instance_num is None: + out = t.replace(old, new) if old else t + else: + k = _count_of(instance_num) + if k < 1: + return '#VALUE!' + at = -1 + for _ in range(k): # the k-th occurrence, counted without overlap + at = t.find(old, at + len(old) if at >= 0 else 0) if old else -1 + if at < 0: + return t + out = t[:at] + new + t[at + len(old):] + return out if len(out) <= _MAX_TEXT else '#VALUE!' + + +def _value(text: Any) -> Any: + if isinstance(text, bool): + return '#VALUE!' + return _number_of(text) + + +def _rept(text: Any, number_times: Any) -> Any: + t, k = _text_of(text), _count_of(number_times) + return '#VALUE!' if k < 0 or len(t) * k > _MAX_TEXT else t * k + + +def _join(delimiter: Any, ignore_empty: Any, pieces: list[Any]) -> Any: + words = [_text_of(v) for v in pieces] + if _flag_of(ignore_empty): + words = [w for w in words if w] + out = _text_of(delimiter).join(words) + return out if len(out) <= _MAX_TEXT else '#VALUE!' + + +#: The number formats ``TEXT`` writes: ``0``, ``00``, ``0.00``, ``#,##0``, +#: ``#,##0.00``, any of them with a ``%``. +_NUMBER_FORMAT = re.compile( + r'(?P#,##)?(?P0+)(?:\.(?P0+))?(?P%)?') +_MONTHS = ('January', 'February', 'March', 'April', 'May', 'June', 'July', + 'August', 'September', 'October', 'November', 'December') +_WEEKDAYS = ('Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday') +#: The date codes ``TEXT`` writes, longest first so ``mmm`` is read +#: before ``mm``; English names, as Excel's English locale writes them. +_DATE_FIELDS: dict[str, Callable[[date], str]] = { + 'yyyy': lambda d: f"{d.year:04d}", + 'yy': lambda d: f"{d.year % 100:02d}", + 'mmmmm': lambda d: _MONTHS[d.month - 1][0], + 'mmmm': lambda d: _MONTHS[d.month - 1], + 'mmm': lambda d: _MONTHS[d.month - 1][:3], + 'mm': lambda d: f"{d.month:02d}", + 'm': lambda d: str(d.month), + 'dddd': lambda d: _WEEKDAYS[d.weekday()], + 'ddd': lambda d: _WEEKDAYS[d.weekday()][:3], + 'dd': lambda d: f"{d.day:02d}", + 'd': lambda d: str(d.day), +} +_DATE_SEPARATORS = frozenset(' -/.,:') + + +def _date_pieces(fmt: str) -> list[str] | None: + """A date format as its codes and separators (``'dd.mm.yyyy'`` → + ``['dd', '.', 'mm', '.', 'yyyy']``), or None when it is not one.""" + pieces: list[str] = [] + i = 0 + while i < len(fmt): + code = next((c for c in _DATE_FIELDS if fmt.startswith(c, i)), None) + if code is not None: + pieces.append(code) + i += len(code) + elif fmt[i] in _DATE_SEPARATORS: + pieces.append(fmt[i]) + i += 1 + else: + return None + return pieces if any(p in _DATE_FIELDS for p in pieces) else None + + +def _format_number(x: float, thousands: bool, whole: int, decimals: int, percent: bool) -> str: + # The number as a cell shows it, 15 significant digits and zeros past + # them, rounded half away from zero to the format's decimals. + shown = _shown(x) + if percent: + shown = shown.scaleb(2, context=_WIDE) + rounded = shown.quantize(decimal.Decimal(1).scaleb(-decimals), context=_WIDE) + integer, dot, fraction = format(rounded.copy_abs(), 'f').partition('.') + integer = integer.zfill(whole) # the padding counts digits: '#,##000000' is 001,234 + if thousands: + head = len(integer) % 3 or 3 + integer = ','.join([integer[:head]] + + [integer[i:i + 3] for i in range(head, len(integer), 3)]) + # The sign is the number's, as the spreadsheet shows it: -0.4 to "0" is "-0". + return ('-' if x < 0 else '') + integer + dot + fraction + ('%' if percent else '') + + +def _text(value: Any, format_text: Any, warned: set[str]) -> Any: + fmt = _text_of(format_text) + v = value + if isinstance(v, str): + try: + v = _parse_number(v) + except TypeError: + return v # text that is not a number is given back as it is + if isinstance(v, bool): + return 'TRUE' if v else 'FALSE' + if not fmt: + return '' # an empty format shows nothing + x = float(_number_of(v)) # a blank is 0, a date its serial number + low = fmt.lower() + if low in ('general', '@'): + return _general(x) + number = _NUMBER_FORMAT.fullmatch(fmt) + if number is not None: + return _format_number(x, bool(number['thousands']), len(number['whole']), + len(number['decimals'] or ''), bool(number['percent'])) + pieces = _date_pieces(low) + if pieces is not None: + d = v if isinstance(v, date) else _SERIAL_EPOCH + timedelta(days=math.floor(x)) + return ''.join(_DATE_FIELDS[p](d) if p in _DATE_FIELDS else p for p in pieces) + if fmt not in warned: + warned.add(fmt) + logger.warning( + "TEXT: the format %r is not one the engine writes (0, 0.00, #,##0, 0%%, " + "yyyy-mm-dd, mmm yyyy, …). The workbook's cell shows the spreadsheet's " + "result; the Python value is the number as General text.", fmt, + ) + return _general(x) + + +def _per_cell(func_name: str, pyfunc: Callable[..., Any], args: list[Any], + value_type: str = 'string') -> Variable: + """One call per period through the shared helper. The first error + among a period's operands is its result and text that only starts + with ``#`` is text (:data:`_ERRORS`); the result carries no unit + (module docstring).""" + @functools.wraps(pyfunc) # keeps the helper's arity check on ``pyfunc`` + def guarded(*values: Any) -> Any: + error = _error_among(values) + return pyfunc(*values) if error is None else error + + out = elementwise(func_name, guarded, args, value_type=value_type, propagate_errors=False) + out._unit = None + return out + + +# ─── The helpers ──────────────────────────────────────────────────── + + +def LEFT(text: Any, num_chars: Any = None) -> Variable: + """The first ``num_chars`` characters, one by default (renders ``=LEFT(...)``). + + Element-wise over a series. A count past the end gives the whole + text; a negative count is ``#VALUE!``. + """ + text = _text_arg(text) + return _per_cell("LEFT", _left, [text] if num_chars is None else [text, num_chars]) + + +def RIGHT(text: Any, num_chars: Any = None) -> Variable: + """The last ``num_chars`` characters, one by default (renders ``=RIGHT(...)``). + + Element-wise over a series. A count past the start gives the whole + text; a negative count is ``#VALUE!``. + """ + text = _text_arg(text) + return _per_cell("RIGHT", _right, [text] if num_chars is None else [text, num_chars]) + + +def MID(text: Any, start_num: Any, num_chars: Any) -> Variable: + """``num_chars`` characters from position ``start_num``, 1-based (renders ``=MID(...)``). + + Element-wise over a series. A start past the end gives empty text; + a start below 1 or a negative count is ``#VALUE!``. + """ + return _per_cell("MID", _mid, [_text_arg(text), start_num, num_chars]) + + +def TRIM(text: Any) -> Variable: + """Text without its leading, trailing and repeated spaces (renders ``=TRIM(...)``). + + Spaces only, as in Excel: a tab or a non-breaking space stays, and a + run of spaces inside the text collapses to one. Element-wise over a + series. + """ + return _per_cell("TRIM", _trim, [_text_arg(text)]) + + +def FIND(find_text: Any, within_text: Any, start_num: Any = None) -> Variable: + """1-based position of ``find_text`` in ``within_text`` (renders ``=FIND(...)``). + + Case-sensitive, no wildcards; the search starts at ``start_num`` + (1 by default). Absent text, a start below 1 or past the end is + ``#VALUE!``. Empty ``find_text`` is found at ``start_num`` + (``FIND('', 'abc')`` is 1), as Excel documents it; LibreOffice gives + ``#VALUE!``. Element-wise over a series. + """ + args = [_text_arg(find_text), _text_arg(within_text)] + return _per_cell("FIND", _find, args if start_num is None else [*args, start_num], + value_type='int') + + +def SEARCH(find_text: Any, within_text: Any, start_num: Any = None) -> Variable: + """1-based position of ``find_text`` in ``within_text``, case-insensitive + (renders ``=SEARCH(...)``). + + Excel's wildcards: ``?`` is one character, ``*`` any run, ``~`` + makes the character after it literal (``~?``, ``~*``, ``~~``; a + last ``~`` is itself). The search starts at ``start_num`` (1 by + default). No match, a start below 1 or past the end is ``#VALUE!``; + empty ``find_text`` is found at ``start_num`` as in :func:`FIND`. + Element-wise over a series. + """ + args = [_text_arg(find_text), _text_arg(within_text)] + return _per_cell("SEARCH", _search, args if start_num is None else [*args, start_num], + value_type='int') + + +def SUBSTITUTE(text: Any, old_text: Any, new_text: Any, instance_num: Any = None) -> Variable: + """``text`` with ``old_text`` replaced by ``new_text`` (renders ``=SUBSTITUTE(...)``). + + Every occurrence, or only the ``instance_num``-th (1-based, counted + without overlap: the second ``aa`` in ``aaa`` is not there). Case- + sensitive; an instance below 1 is ``#VALUE!``, one past the last + occurrence leaves the text as it is. Element-wise over a series. + """ + args = [_text_arg(text), _text_arg(old_text), _text_arg(new_text)] + return _per_cell("SUBSTITUTE", _substitute, + args if instance_num is None else [*args, instance_num]) + + +def VALUE(text: Any) -> Variable: + """The number numeric text spells (renders ``=VALUE(...)``). + + ``'1,234'`` is 1234, ``'12%'`` 0.12, ``'(1,234)'`` -1234, ``'1e3'`` + 1000, spaces around the number (and after its sign) ignored; a comma + is a thousands separator — the first group of any length, the others + of three. A time of day is the fraction of a day it spells — + ``'12:00'`` is 0.5, ``'1:30 PM'`` 0.5625; hours, minutes and seconds + (``'12:30:45.5'``), AM or PM after them — and ``'2026-01-31'`` is + the date's serial number, a time after it (``'2026-01-31 18:00'``) + adding its fraction. A number stays itself, a blank is 0, a date its + serial number. Other text — words, empty text, TRUE, a currency + sign, date text in any other form (which the spreadsheet would read + by its locale: ``'1/2'``, a ``T`` between the date and the time), a + negative time, minutes or seconds of 60 or more — is ``#VALUE!``; a + number too large for a cell (``'1e999'``) is ``#NUM!``. Element-wise + over a series. + """ + return _per_cell("VALUE", _value, [text], value_type='float') + + +def TEXT(value: Any, format_text: Any) -> Variable: + """A number as text in a format (renders ``=TEXT(value, format)``). + + The formats the engine writes, as the spreadsheet writes them: + + - ``0``, ``0.0``, ``0.00``, …, ``00``, ``000`` (zero-padded), + ``#,##0``, ``#,##0.00``, …, and any of them with ``%`` (``0%``, + ``0.0%``): halves round away from zero on the number as a cell + shows it (``TEXT(2.5, '0')`` is ``3``, ``TEXT(1.005, '0.00')`` is + ``1.01``); a negative number keeps its sign, ``TEXT(-0.4, '0')`` + being ``-0``; + - ``General`` and ``@``: the number as General text (15 significant + digits, E notation from 1E20 and for a fraction whose spelling + passes 20 characters); + - a date format of ``yyyy``, ``yy``, ``mmmmm`` (the month's first + letter), ``mmmm``, ``mmm``, ``mm``, ``m``, ``dddd``, ``ddd``, + ``dd``, ``d`` with ``-``, ``/``, ``.``, ``,``, ``:`` or spaces + between (``yyyy-mm-dd``, ``dd.mm.yyyy``, ``mmm yyyy``, ``d mmmm + yyyy``), English month and day names, over a date or a serial + number. Codes are case-insensitive; hours, minutes and seconds are + not written. + + Every format shows the number to 15 significant digits, zeros past + them (``TEXT(123456789012345678, '0')`` is ``123456789012346000``); + an empty format shows nothing. Any other format is written to the + workbook as given, with a warning; its Python value is the number as + General text. Numeric text is formatted as its number, other text is + given back as it is, a blank reads as 0, TRUE and FALSE are the + words. Element-wise over a series. + """ + warned: set[str] = set() + return _per_cell("TEXT", lambda v, f: _text(v, f, warned), [value, _text_arg(format_text)]) + + +def REPT(text: Any, number_times: Any) -> Variable: + """``text`` repeated ``number_times`` times (renders ``=REPT(...)``). + + A count is truncated to a whole number; a negative one, or a result + longer than a cell holds (32,767 characters — Excel's limit, which + LibreOffice does not apply), is ``#VALUE!``. Element-wise over a + series. + """ + return _per_cell("REPT", _rept, [_text_arg(text), number_times]) + + +def TEXTJOIN(delimiter: Any, ignore_empty: Any, *texts: Any) -> Variable: + """Texts joined by a delimiter (renders ``=TEXTJOIN(...)``, stored + as ``_xlfn.TEXTJOIN`` — an Excel 2019 name). + + One text for the whole call: a series operand is joined along its + periods, as the workbook's ``TEXTJOIN(", ", TRUE, B2:F2)`` joins the + row, and a scalar operand is one piece. ``ignore_empty`` TRUE skips + blanks and empty text; FALSE keeps them as empty pieces between + delimiters (a number is TRUE unless it is zero, a blank FALSE, the + words themselves are read, other text is ``#VALUE!``). Numbers, + TRUE/FALSE and dates read as the spreadsheet reads them (module + docstring); the first error among the operands is the result. + """ + from ..core.shape import reject_axised + from ..core.tracks import TrackValues + if not texts: + raise ValueError( + "TEXTJOIN() needs a delimiter, ignore_empty and at least one text. " + "Syntax: mo.TEXTJOIN(', ', True, names) or mo.TEXTJOIN('-', True, a, b). " + "Got the delimiter and ignore_empty only." + ) + delimiter = _text_arg(delimiter) + texts = tuple(_text_arg(t) for t in texts) + for role, a in (("delimiter", delimiter), ("ignore_empty", ignore_empty)): + if is_series(a): + raise ValueError( + f"TEXTJOIN: the {role} is one value for the whole call, not a series. " + f"Give one text ({role}=', ') or one cell." + ) + raws: list[Any] = [] + ast_args: list[Any] = [] + for a in (delimiter, ignore_empty, *texts): + if isinstance(a, Variable): + reject_axised(a, "TEXTJOIN") + if isinstance(a._value, TrackValues): + raise ValueError( + "TEXTJOIN() over a Variable with tracks joins one track at a " + "time — slice a track first (.at(track='...'))." + ) + ast_args.append(VarRef(a)) + raws.append(a._value) + else: + ast_args.append(Literal(a)) + raws.append(a) + delim, ignore, *rows = raws + pieces = [v for row in rows for v in (row if isinstance(row, list) else [row])] + value = _error_among((delim, ignore, *pieces)) + if value is None: + try: + value = _join(delim, ignore, pieces) + except TypeError: + value = '#VALUE!' + except OverflowError: + value = '#NUM!' + source_code = "TEXTJOIN(" + ", ".join(src(a) for a in (delimiter, ignore_empty, *texts)) + ")" + return make_func_var("TEXTJOIN", ast_args, value, 'string', 'scalar', + source_code=source_code) + + +register("LEN", shape="elementwise") +register("UPPER", shape="elementwise") +register("LOWER", shape="elementwise") +# The Excel renderer writes CONCAT as the ``&`` operator, not as a call. +register("CONCAT", shape="elementwise", passthrough=False) +register("LEFT", shape="elementwise") +register("RIGHT", shape="elementwise") +register("MID", shape="elementwise") +register("TRIM", shape="elementwise") +register("FIND", shape="elementwise") +register("SEARCH", shape="elementwise") +register("SUBSTITUTE", shape="elementwise") +register("VALUE", shape="elementwise") +register("TEXT", shape="elementwise") +register("REPT", shape="elementwise") +# TEXTJOIN joins whole rows: a series operand is a range, a scalar one a +# cell. An Excel 2019 name — the file stores it as ``_xlfn.TEXTJOIN``. +register("TEXTJOIN", shape="range", file_prefix="_xlfn.") diff --git a/tests/golden/forecast.xlsx b/tests/golden/forecast.xlsx index de7c7eae858341bd454eee0cdae72f18d637a3aa..497c74a021dcc58b5b57521eaeb11f82cbb28ebd 100644 GIT binary patch delta 2896 zcmZ9Oc{mi@9>>Qr$nr9n8QHTZ#$>4}mBiSmtQj(vWQtJ8PGe-3FoV}fgp_^X#*7-I zWM+iORxc&8WwK{USKa&G_jb=8=l7iN^E==3JLma5&-wHn-PIAWH0R(H1poj%0LKKo zbOG9d{ebcC5f5HUe=Ch^_v(c6$GLcX%j*tu5L9A%xIck#9r0eodN=AMi@0A`h}=2h zd|$JPd!&DWbs_(3%7Gq!Q2vw>k)9aPLvcbZ1FI@1XvT4C)R5- zOcQ{2@)TW|051oYe5SjbSbuTlwFn}gN zIe!~-`qWZB`Uu&{KHA=TG1k_pEPU$>W$!EHz)pMXot>92?Ax30F2Vh`*?xNp49O{1 zzJ8VM1efs77kO=@x0J;W03>q*0Ac{5EL0qqlIe3{94_?oSrXVsn0Sm3r-W2VG&DCJ zoh@-}YCFbMtTkC=_{mJZVJ$XT;VlQCm8{)oI}b2BlHM^96`BBK6SB9wuO~xQ!<*uW zwd|X>&7k=yf$AReqDW(Qhw(3zO1gc9{URt2?_RQUjnFy>yj<-6!cEDvQI%IUKMt3c zjJmd7(dOPeVBr}r1yn@2vCRjcAYL0qWo3hDN0bohX)$j-SeH-sg~@;GbhVMXo9O)N zj!^1YOQHe2_7?V!D-@LgEZQ(}1tP2yeM+$M(8*crS{;TZ#1Y|H)*%kIn>Be9!sVNg z#-UnESjlWSZ>dZjr0+DB4|!!3^5ITOvdZ^-H$*pQ@fYF3TP~1twT51GVS{t_MdGdNXQxGlE5x3Vkd4`izlg9^pg0T<8HS(R#Q8c=CT+Q$yfawf5o9t#E@n((#L%H9HYP-^IBm%Mc)-M3;QDrf)AZn@ z@s!UQPik&-RS%3XcO<@Z|K7>N4y5W$AON5U{!j0Mahk9YAMeO)hkzm@(bykm#+mUX z1@BKLGqn*xhzY8=M!|YRAs8`m{gate$K8+?Be5?BR`E zY}#3QylQP1ui=J1^Rydtk!wqT=>y}WMvw!L(iMP~<|Uoh>GPE!s0e?6jW6nccMe#r zn%!I(#(7aj=@j!CQm%#3<=Y^80m$tqHLVs+QJO)I>0&hUX_s7C+GPB)5YRczUA zO~!(kZ1ed2UUkWC>2l_M>K(V^QixFo?+n@ljFUt|w{ZRwPFB8(cVkGB6$QrlqRHXy zjlC1h#vHs~Uj!(r~N4Hna8-T#g0$yw8?`eC2I{AU-YJs*TrZf4)0 z&m0fayVuolN7{w1I`P0YyWO{5Z;yHA{DOoud8V+lqG!8J4rh#vgbdE7U}s)}Tj#tt zaO?HqCAJW>_cs@Qh(J*#c~AoSE+ z>w9Qz2Wz*(f)EIfVD zf2eYE1-0a&xV^S|HUH}Cnkk2{9QJ!TGW5oiLFRoLvm!>%|+s{Dbmczd@H zD;nrSQ-_JYSQywShdmYeA*(AeHr?JW!b&!K)r7OD+Yov%9I2GU!;oud3&zc84tXd! zlje|dbt-0|vgl7d%v%Me_Sb?ym#z2(5w0;j!RG@oUNf0F*RA~+2tlzTKY~$55)r1e zX{HXd&UH5<04|k0=8kLCG}+d@{BvIT*xQ z`aoJkoS0#xT{Cx}(O~KcFh=iywb!bR*VQh%GAKDjvM=TYua=kb^JEdxMW{>IE!-tzWBa~VnR=$~+Yf%a?;0*w zr7NF2LyjIsp(l@y+CsCN__V)3wf&xMnE#pGXs(@NFVv{*+}gdHWb)o0NG+ZoKtjIA zq3Xd3>dY)yoj25TtRkwh!J)Og4D-md=Ang)Pfd(jW=l@lgvU_UVK*%Ad^=gi%H>`R zNxp>J3y0XaVw^pS4|^DAupGD|$Dp1)ZQ1EyK^S?J0M}v%>$3$OeWCxc!L)+`9OR5u z_&O%zSEiiOeNp+&39y>9_kL@LlPOf%`Hm)NY!B zT}5Iq)Rl14wgU6)KR*zv_(6Tmx23I`L{(wFUu|v~pP?Ko({p?3TW_@75Sjlae|k%v z1TGdW(<9Fn#-}$4IonxPNh)+N>KVyBGa3I?+fk$-im#`*!{@(%;qWcdi5)&cE zWPVc8Pi7M00swNA0RWMGsQi0Gu}u>Hg2?^K`JJTx&HwvabYuN*p(0#Fgu#3NkcYp@$=WJ=t#xBb7U&6%*B>o67 zX9EBl_hC-(A7?-mTeT=rR!r)b&si}Ec4Y_=Bc}4J9N#AjRO$Ua_MHEJEfFZOSWJ=Y Jve-|je*r|OIZXfn delta 2351 zcmZWr2UHWt77Zm30-;C|5}F{@Na&C#-Bl2T&=!?m0t5l2Hzg2CBq+rQ3Me8i7LcaE zN|CA{qC}RK7y%6dQHV&}rT)O4|99O#XU@Dc@65gL%sVspy)C^VEeP70i(3o;0Du7B zGLgBE4nDSV=ez_^X625`;_R)vHHBZjATrw8^hc8K4`lkH0}N)?dp#<@skTd8inV-)7JY}36sv_t1lN9O!V79uO$-cX<-)zd z?M}lI2Z{%>ml2<2Z@+L(06>#u2vWdZr0!=06XKu}0x9Qyq@Lk4$3=flETMU{FwCoz zqy38rYs0K>Lsy@vK@c6ryR5|7Zes%Mp==b7S6^daT2`RHO&l{WKG&>sr4DX3bz0rI zQ^0oQ{cr_mz9Cs*ean{262Ha|CA`&l4pVhep5?40&N|Cs|6!DqR^ zoQ2jmzxj$N4!Bwlf_@w?wa~3io4#atB_N>RXhwJr1jWi*O1-sK3{R;7?tvJldE1d(6z19DGx2=oD^o)4ew^dP=)SUCdtN8_vK0?1J1V zhy@b|P7sPay+#>FqqDlNoJ!yx4I2|`i&9$3aqQlLGUZYSA&$6oui~Me$CtN_JP>od zW|MD{dtpUK>dz0Ui*05x zS;x!8(~tG0Jgf0{C~G5#`d7VlXD=T%A&eu0MhLmtv- zYw+f~6&D8BTzneqq1=cY&Gy?8`sdrkA>HZxc5RDX06>oj03gZ!zOliGn2X^TA_6g( z$Oy!~(bk9!3BG>YJ)%rsEPa=l|CkY7ymJYkkKyt5(O zv1BP97ez{nl{+*bG%MgyY_(iw$e%;vqfUld0Fqx|oyG zPvf+x&pRjD39q|uPtqq&7R=-|daU^h4aQs9KJ7cQHB_?Zlz(oS#dDC+`re^>=?~^_0vap%}PFT6~EY_OH`mW*Ad|4DeDW|BB~l5=SqS>D#h#U6*?)+;sKCP)W$qXT5bydf7a=#U8NVGu zTvckyKVd3C2iu>*KZnnQ>|OBj@OiMk9ex8o&u8z6=Yh}jQ%v#ca43j^#-D^k!4ymU zOE{E|au%-)hw@WS;49&UAW9@LJUc1w3Xc0tQFh0g-L4hm1#+z*0VMH>&>>w^1wb;# zy@7ptuh!H}i;-);a{L&qs{^CGvfQ1M%p-yya*7rBnuwQ-OBX!^C^5&afU21LE3?An z+9>Do=MQTqa~j2w!ApW)Eq`r2`q5P;q}S}#C-;$DMZxD1EOM4x*ib{PN)#g_1=rr2 z@IcX1S^)mhNCx=HD{!g(i>er}VrGoZsa!)z*H#x4vA475llyyn1;I)QR(95VtDFe` zuqgkq;ev}3FbPzH*WJ38T~r_3*}%sjB%GmYuEGBsG3S+eu>zwZrv?Osi>aC);!h)@ zd(3kc#2V05ASL)sRdY4|3KN1MjLZme&H5K*os6t*J&U82%v+%z55joZ&@&+))%T!5 z%gSbK=$WD>UN+Pb`i2(0zIn1x4-q#~$|+}B(1Li8Oc?wgYI5f&+F{+?NEBSNyge-v zknkarqd8GNwf$ZsYh_F>baa>NpJIV*P!V~^$;B>Wq+wC!o^5Cqi{jXK=t_LqcgP>K z{mT(1dF<~7fb>jKfB)E`B#dL41eQ{e|EU!6{v|zZ_1Pau$qFEn5eRw&ZOy?c#`V9Z z;{UYjZ@_C(u9V7th!!c>xkbT!9gB#J3&uq3iPyi0ZV_9gJJ@JZb|L2fFALiyaTKy; z&@&tp9D&f-TmJ&M`EaW&2LRy0&8Gf|c2 4.0, "big", "") + assert shown.value == ["", "big", "big"] + assert mo.COUNTA(shown, _b()).value == [2, 2, 2] + assert mo.COUNTA(_b(), "").value == [2, 2, 2] + + +class TestTracks: + def _tracked(self): + m = mo.Model("m", tracks=mo.Tracks("plan", "actual"), + default_grain="month", default_start="2026-01", default_periods=3) + with m: + m.x = mo.Variable(plan=[1.0, 5.0, 3.0], actual=[2.0, None, 2.0]) + m.y = mo.Variable(plan=[1.0, 1.0, 1.0], actual=[3.0, 3.0, 3.0]) + return m + + def test_tracks_are_taken_track_by_track(self): + m = self._tracked() + assert dict(mo.SUM(m.x, m.y)._value.items()) == { + "plan": [2.0, 6.0, 4.0], "actual": [5.0, 3.0, 5.0]} + assert dict(mo.SUM(m.x, 1.0)._value.items()) == { + "plan": [2.0, 6.0, 4.0], "actual": [3.0, 1.0, 3.0]} + assert dict(mo.AVERAGE(m.x, m.y)._value.items()) == { + "plan": [1.0, 3.0, 2.0], "actual": [2.5, 3.0, 2.5]} + assert dict(mo.PRODUCT(m.x, m.y)._value.items()) == { + "plan": [1.0, 5.0, 3.0], "actual": [6.0, 3.0, 6.0]} + assert dict(mo.COUNT(m.x, m.y)._value.items()) == { + "plan": [2, 2, 2], "actual": [2, 1, 2]} + assert dict(mo.COUNTA(m.x, 0)._value.items()) == { + "plan": [2, 2, 2], "actual": [2, 1, 2]} + assert mo.SUM(m.x, m.y).var_type == "list" + + def test_one_line_with_tracks_alone_is_still_its_totals(self): + m = self._tracked() + assert dict(mo.SUM(m.x)._value.items()) == {"plan": 9.0, "actual": 4.0} + + +class TestUnits: + def test_sum_and_average_keep_the_one_unit(self): + a = mo.Variable([3.0, 8.0, 5.0], unit="$") + b = mo.Variable([4.0, 4.0, 4.0], unit="$") + c = mo.Variable([4.0, 4.0, 4.0], unit="hr") + assert str(mo.SUM(a, b).unit) == "$" + assert str(mo.AVERAGE(a, 0.0).unit) == "$" + with pytest.raises(ValueError, match="mo.SUM takes its values in one unit"): + mo.SUM(a, c) + with pytest.raises(ValueError, match="mo.AVERAGE takes its values in one unit"): + mo.AVERAGE(c, a) + + def test_a_product_per_period_multiplies_its_operands_units(self): + price = mo.Variable([10.0, 11.0, 12.0], unit="USD/t") + volume = mo.Variable([2.0, 3.0, 4.0], unit="t") + revenue = mo.PRODUCT(price, volume) + assert revenue.value == [20.0, 33.0, 48.0] + assert str(revenue.unit) == "USD" + assert str(mo.PRODUCT(volume, 2.0).unit) == "t" + # One row of amounts alone is still refused: its cells' product is + # in tonnes to the power of their count. + with pytest.raises(ValueError, match="to the power of their count"): + mo.PRODUCT(volume) + + def test_a_count_has_no_unit(self): + a = mo.Variable([3.0, 8.0, 5.0], unit="$") + assert mo.COUNT(a, a).unit is None + assert mo.COUNTA(a, 1.0).unit is None + + +class TestLinesAlongKeys: + """A line along keys is no period series: against one it is refused, + as ``a + b`` refuses it; lines along the same keys combine key by key.""" + + @staticmethod + def _model(): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with m: + m.s = mo.Variable([1.0, 2.0, 3.0], display_name="s") + m.k = mo.Variable({"Bear": 0.9, "Base": 1.0, "Bull": 1.1}, display_name="k") + m.k2 = mo.Variable({"Bear": 1.0, "Base": 2.0, "Bull": 3.0}, display_name="k2") + m.k3 = mo.Variable({"Bull": 1.0, "Base": 2.0, "Bear": 3.0}, display_name="k3") + return m + + @pytest.mark.parametrize("name", FIVE + ("MIN", "MAX")) + def test_against_a_period_series_it_is_refused_as_a_plus_b_refuses_it(self, name): + m = self._model() + with pytest.raises(ValueError, match="runs along its keys"): + m.s + m.k + with pytest.raises(ValueError, match="runs along its keys"): + getattr(mo, name)(m.s, m.k) + + def test_lines_along_the_same_keys_combine_key_by_key(self, tmp_path): + m = self._model() + with m: + m.t = mo.SUM(m.k, m.k2).set_display_name("t") + m.n = mo.COUNT(m.k, m.k2, None).set_display_name("n") + assert m.t.value == pytest.approx([1.9, 3.0, 4.1]) + assert m.t._keys == ["Bear", "Base", "Bull"] == (m.k + m.k2)._keys + assert m.n.value == [3, 3, 3] and m.n._keys == ["Bear", "Base", "Bull"] + assert mo.MAX(m.k, 1.0).value == [1.0, 1.0, 1.1] + # One line along keys alone is still its total. + assert mo.SUM(m.k).value == pytest.approx(3.0) + + def test_lines_along_keys_in_another_order_combine_key_by_key(self): + m = self._model() + # k3 is Bull 1, Base 2, Bear 3: read in k's order, as ``a + b`` reads it. + assert mo.SUM(m.k, m.k3).value == pytest.approx((m.k + m.k3).value) + assert mo.SUM(m.k, m.k3).value == pytest.approx([3.9, 3.0, 2.1]) + assert mo.COUNT(m.k, m.k3)._keys == ["Bear", "Base", "Bull"] + m4 = mo.Variable({"Bear": 1.0, "Base": 2.0, "Mid": 3.0}, display_name="k4") + with pytest.raises(ValueError, match="other keys"): + mo.SUM(m.k, m4) + + +# ══════════════════════════════════════════════════════════════════════ +# Inside a loop container +# ══════════════════════════════════════════════════════════════════════ + + +class TestLinesOnOtherWindows: + """Located series on other time windows are aligned by date, as ``a + + b`` aligns them: each continued by its own ``extend=`` rule, one that + does not cover the combined window refused, the result on that + window. ``MIN`` / ``MAX`` paired them by position before.""" + + @staticmethod + def _windows(): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.x = mo.MultiVariable("X") + m.x.rev = mo.Variable([1.0, 2.0, 3.0, 4.0], display_name="rev", extend=mo.zero()) + m.t = mo.MultiVariable("T", default_grain="month", default_start="2026-03", + default_periods=4) + m.t.fee = mo.Variable([10.0, 20.0, 30.0, 40.0], display_name="fee", + extend=mo.zero()) + return m + + def test_each_is_the_call_over_the_date_s_cells_as_its_operator_is(self): + m = self._windows() + rev, fee = m.x.rev, m.t.fee + plus = rev + fee + assert plus.value == [1.0, 2.0, 13.0, 24.0, 30.0, 40.0] + assert mo.SUM(rev, fee).value == plus.value + assert mo.AVERAGE(rev, fee).value == (plus / 2).value + assert mo.PRODUCT(rev, fee).value == (rev * fee).value + assert mo.MAX(rev, fee).value == [1.0, 2.0, 10.0, 20.0, 30.0, 40.0] + assert mo.MIN(rev, fee).value == [0.0, 0.0, 3.0, 4.0, 0.0, 0.0] + # A continued cell holds the 0 the sheet writes for it: counted. + assert mo.COUNT(rev, fee).value == [2] * 6 + assert mo.COUNTA(rev, fee).value == [2] * 6 + for name in FIVE + ("MIN", "MAX"): + assert getattr(mo, name)(rev, fee).time == plus.time, name + + @pytest.mark.parametrize("name", FIVE + ("MIN", "MAX")) + def test_a_window_not_covered_is_refused_as_a_plus_b_refuses_it(self, name): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.x = mo.MultiVariable("X") + m.x.rev = mo.Variable([1.0, 2.0, 3.0, 4.0], display_name="rev") + m.t = mo.MultiVariable("T", default_grain="month", default_start="2026-03", + default_periods=4) + m.t.fee = mo.Variable([10.0, 20.0, 30.0, 40.0], display_name="fee") + with pytest.raises(ValueError, match="'rev' does not cover the combined window"): + m.x.rev + m.t.fee + with pytest.raises(ValueError, match="'rev' does not cover the combined window"): + getattr(mo, name)(m.x.rev, m.t.fee) + + def test_each_of_three_windows_continues_by_its_own_rule(self): + m = self._windows() + with m: + m.u = mo.MultiVariable("U", default_grain="month", default_start="2026-05", + default_periods=2) + m.u.rate = mo.Variable([100.0, 200.0], display_name="rate", extend=mo.hold()) + s = mo.SUM(m.x.rev, m.t.fee, m.u.rate) + assert s.value == [101.0, 102.0, 113.0, 124.0, 130.0, 240.0] + assert s.time.start == "2026-01" + + def test_a_one_value_line_is_a_constant_as_the_operators_read_it(self): + # One value on a window of three months is read as a + # constant: every period reads it, as its one cell is written. + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=3) + with m: + m.x = mo.MultiVariable("X") + m.x.a = mo.Variable([1.0, 2.0, 3.0], display_name="a") + m.x.one = mo.Variable([5.0], display_name="one") + m.x.held = mo.Variable([5.0], display_name="held", extend=mo.hold()) + m.o = mo.MultiVariable("O", default_grain="month", default_start="2026-02", + default_periods=1) + m.o.feb = mo.Variable([5.0], display_name="feb") + assert mo.SUM(m.x.a, m.x.one).value == (m.x.a + m.x.one).value == [6.0, 7.0, 8.0] + assert mo.SUM(m.x.a, m.x.held).value == (m.x.a + m.x.held).value == [6.0, 7.0, 8.0] + # On a window of one period it is that period's value. + with pytest.raises(ValueError, match="'feb' does not cover"): + m.x.a + m.o.feb + with pytest.raises(ValueError, match="'feb' does not cover"): + mo.SUM(m.x.a, m.o.feb) + + def test_two_grains_are_refused(self): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=3) + with m: + m.x = mo.MultiVariable("X") + m.x.a = mo.Variable([1.0, 2.0, 3.0], display_name="a") + m.q = mo.MultiVariable("Q", default_grain="quarter", default_start="2026-Q1", + default_periods=3) + m.q.b = mo.Variable([1.0, 2.0, 3.0], display_name="b") + with pytest.raises(ValueError, match="different grains"): + mo.SUM(m.x.a, m.q.b) + with pytest.raises(ValueError, match="different grains"): + mo.COUNT(m.x.a, m.q.b) + + def test_the_workbook_reads_the_cells_the_sheet_holds(self, tmp_path): + # A line that starts with the window and ends early is written + # over the window, continued as it declares: the call reads those + # cells, live, and computes what Python computes. + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.x = mo.MultiVariable("X") + m.x.rev = mo.Variable([1.0, 2.0, 3.0, 4.0], display_name="rev") + m.x.short = mo.Variable([5.0, 6.0], display_name="short", extend=mo.zero()) + m.x.s = mo.SUM(m.x.rev, m.x.short).set_display_name("s") + m.x.n = mo.COUNT(m.x.rev, m.x.short).set_display_name("n") + assert m.x.s.value == (m.x.rev + m.x.short).value == [6.0, 8.0, 3.0, 4.0] + assert m.x.n.value == [2, 2, 2, 2] + rows = _book(m, tmp_path, "X") + assert rows["short"] == [5, 6, 0, 0] + assert rows["s"] == ["=SUM(B2, B3)", "=SUM(C2, C3)", "=SUM(D2, D3)", "=SUM(E2, E3)"] + assert rows["n"][3] == "=COUNT(E2, E3)" + + def test_a_line_on_a_window_inside_the_sheet_s_is_read_by_date(self, tmp_path): + # A sub-container's sheet has the columns of its own window, so + # a formula on the model's window reads its cell of the same + # DATE — never the cell in the same position — and, for a + # period it has no column for, the line's declared rule: 0 for + # a flow, the first or last cell for a rate held. The operators' + # cells, which paired by position before, follow the same rule. + m = _windows_inside() + assert m.x.s.value == m.x.plus.value == [1.0, 2.0, 13.0, 24.0] + assert m.x.hp.value == m.x.pr.value == [0.5, 1.0, 1.5, 1.0] + assert m.x.se.value == [101.0, 202.0, 3.0, 4.0] + assert m.x.so.value == [1.0, 2.0, 53.0, 4.0] + assert m.x.sl.value == [6.0, 7.0, 8.0, 9.0] + path = tmp_path / "book.xlsx" + m.to_excel(path) + book = load_workbook(path) + x, t, u, o = (_rows(book[name]) for name in ("X", "T", "U", "O")) + assert t["fee"] == [None, 10, 20] and t["k"] == [7, None, None] + assert u["early"] == [100, 200] and o["once"] == [50] + assert x["s"] == ["=SUM(B2, 0)", "=SUM(C2, 0)", "=SUM(D2, T!C2)", "=SUM(E2, T!D2)"] + assert x["plus"] == ["=B2 + 0", "=C2 + 0", "=D2 + T!C2", "=E2 + T!D2"] + assert x["av"] == ["=AVERAGE(B2, 0, T!C3)", "=AVERAGE(C2, 0, T!C3)", + "=AVERAGE(D2, T!C2, T!C3)", "=AVERAGE(E2, T!D2, T!D3)"] + assert x["pr"] == ["=PRODUCT(B2, T!C3)", "=PRODUCT(C2, T!C3)", + "=PRODUCT(D2, T!C3)", "=PRODUCT(E2, T!D3)"] + assert x["hp"] == ["=B2 * T!C3", "=C2 * T!C3", "=D2 * T!C3", "=E2 * T!D3"] + assert x["mx"][0] == "=MAX(B2, 0)" and x["mx"][3] == "=MAX(E2, T!D2)" + assert x["pw"] == ["=POWER(B2, T!C3)", "=POWER(C2, T!C3)", + "=POWER(D2, T!C3)", "=POWER(E2, T!D3)"] + # A scalar on that sheet is its one cell in every period, as before. + assert x["sk"] == ["=SUM(B2, T!B4)"] + [f"=SUM({c}2, T!B4)" for c in "CDE"] + # The same start, ending early: its last cell held, or 0. + assert x["se"] == ["=SUM(B2, U!B2)", "=SUM(C2, U!C2)", "=SUM(D2, 0)", "=SUM(E2, 0)"] + assert x["mn"] == ["=MIN(B2, U!B3)", "=MIN(C2, U!C3)", "=MIN(D2, U!C3)", "=MIN(E2, U!C3)"] + assert x["n"] == ["=COUNT(B2, 0, U!B2)", "=COUNT(C2, 0, U!C2)", + "=COUNT(D2, T!C2, 0)", "=COUNT(E2, T!D2, 0)"] + # One period, one cell: a located series still, read by date. + assert x["so"] == ["=SUM(B2, 0)", "=SUM(C2, 0)", "=SUM(D2, O!B2)", "=SUM(E2, 0)"] + assert x["sl"] == ["=SUM(B2, O!B3)"] + [f"=SUM({c}2, O!B3)" for c in "CDE"] + assert x["na"] == ["=COUNTA(B2, 0)", "=COUNTA(C2, 0)", "=COUNTA(D2, O!B2)", + "=COUNTA(E2, 0)"] + assert mo.check_compat(m, "excel") == [] + + def test_a_combined_window_beyond_the_sheet_s_is_refused_by_the_export(self, tmp_path): + # As the export refuses the operator's result on that window. + m = self._windows() + with m: + m.x.s = mo.SUM(m.x.rev, m.t.fee).set_display_name("s") + m.x.plus = (m.x.rev + m.t.fee).set_display_name("plus") + with pytest.raises(ValueError, match="window of its own") as refused: + m.to_excel(tmp_path / "w.xlsx") + assert "'s'" in str(refused.value) and "'plus'" in str(refused.value) + + +def _loop(m): + with m: + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + prev = mo.lag(d.bal, fill=m.i.q) + d.inflow = mo.SUM(m.x.a, prev, -10.0).set_display_name("inflow") + d.mean = mo.AVERAGE(m.x.b, prev).set_display_name("mean") + d.grown = mo.PRODUCT(prev, 0.5, m.x.b).set_display_name("grown") + d.n = mo.COUNT(prev, m.x.a, "x").set_display_name("n") + d.na = mo.COUNTA(prev, m.x.a, "x").set_display_name("na") + d.bal = (prev + d.inflow - d.mean).set_display_name("bal") + return m + + +class TestInsideALoop: + def test_each_period_reads_the_previous_period(self): + m = _loop(_model()) + # bal: 9 → inflow 3+9-10=2, mean (4+9)/2=6.5, bal 9+2-6.5=4.5; + # 4.5 → inflow 8+4.5-10=2.5, mean 4.25, bal 2.75; 2.75 → inflow + # -2.25, mean 3.375, bal -2.875. + assert m.d.inflow.value == pytest.approx([2.0, 2.5, -2.25]) + assert m.d.mean.value == pytest.approx([6.5, 4.25, 3.375]) + assert m.d.grown.value == pytest.approx([18.0, 9.0, 5.5]) + assert m.d.n.value == [2, 2, 2] + assert m.d.na.value == [3, 3, 3] + assert m.d.bal.value == pytest.approx([4.5, 2.75, -2.875]) + + def test_the_same_numbers_as_outside_the_loop(self): + m = _loop(_model()) + prev = mo.Variable([9.0, 4.5, 2.75]) + assert mo.SUM(m.x.a, prev, -10.0).value == pytest.approx(m.d.inflow.value) + assert mo.AVERAGE(m.x.b, prev).value == pytest.approx(m.d.mean.value) + + @pytest.mark.parametrize("name", FIVE) + def test_one_series_alone_stays_refused_and_points_at_the_per_period_form(self, name): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with pytest.raises(ValueError, match=f"{name} reads a whole series") as err: + with m: + m.x = mo.MultiVariable("X", loop=True) + with m.x as x: + x.a = mo.lag(x.b, fill=1.0) + 1.0 + x.peak = getattr(mo, name)(x.a) + x.b = x.a - x.peak + assert f"mo.{name}(a, b)" in str(err.value) + + +# ══════════════════════════════════════════════════════════════════════ +# The workbook +# ══════════════════════════════════════════════════════════════════════ + + +class TestTheWorkbook: + def test_one_call_per_period(self, tmp_path): + m = _model() + with m: + m.x.s = mo.SUM(m.x.a, m.x.b).set_display_name("s") + m.x.av = mo.AVERAGE(m.x.a, m.i.floor).set_display_name("av") + m.x.pr = mo.PRODUCT(m.x.a, m.x.b, 0.5).set_display_name("pr") + m.x.n = mo.COUNT(m.x.a, [1.0, None, "x"]).set_display_name("n") + m.x.na = mo.COUNTA(m.x.a, "x").set_display_name("na") + x = _book(m, tmp_path, "X") + assert x["s"] == ["=SUM(B2, B3)", "=SUM(C2, C3)", "=SUM(D2, D3)"], x["s"] + assert x["av"] == ["=AVERAGE(B2, I!B1)", "=AVERAGE(C2, I!B1)", + "=AVERAGE(D2, I!B1)"], x["av"] + assert x["pr"] == ["=PRODUCT(B2, B3, 0.5)", "=PRODUCT(C2, C3, 0.5)", + "=PRODUCT(D2, D3, 0.5)"], x["pr"] + assert x["n"] == ["=COUNT(B2, 1.0)", "=COUNT(C2, 0)", '=COUNT(D2, "x")'], x["n"] + assert x["na"] == ['=COUNTA(B2, "x")', '=COUNTA(C2, "x")', '=COUNTA(D2, "x")'], x["na"] + + def test_inside_a_larger_formula(self, tmp_path): + m = _model() + with m: + m.x.twice = (mo.SUM(m.x.a, m.x.b) * 2).set_display_name("twice") + m.x.inner = mo.SUM(m.x.a * 2, m.x.b).set_display_name("inner") + x = _book(m, tmp_path, "X") + assert x["twice"] == ["=SUM(B2, B3) * 2", "=SUM(C2, C3) * 2", "=SUM(D2, D3) * 2"] + assert x["inner"] == ["=SUM(B2 * 2, B3)", "=SUM(C2 * 2, C3)", "=SUM(D2 * 2, D3)"] + + def test_one_series_and_scalars_only_are_written_as_before(self, tmp_path): + m = _model() + with m: + m.x.red = mo.SUM(m.x.a).set_display_name("red") + m.x.both = (mo.SUM(m.x.a) + mo.SUM(m.x.b)).set_display_name("both") + m.x.pair = mo.SUM(m.i.p, m.i.q).set_display_name("pair") + m.x.cnt = mo.COUNT(m.x.a).set_display_name("cnt") + # A scalar line gives the sheet a constants column (B); the + # periods move to C..E. + x = _book(m, tmp_path, "X") + assert x["red"][0] == "=SUM(C2:E2)", x["red"] + assert x["both"][0] == "=SUM(C2:E2) + SUM(C3:E3)", x["both"] + assert x["pair"][0] == "=SUM(I!B2, I!B3)", x["pair"] + assert x["cnt"][0] == "=COUNT(C2:E2)", x["cnt"] + + def test_a_loop_reads_the_previous_column(self, tmp_path): + m = _loop(_model()) + d = _book(m, tmp_path, "D") + # bal is the sheet's last row, 7: each period reads its previous cell. + assert d["inflow"] == ["=SUM(X!B2, I!B3, -10.0)", "=SUM(X!C2, B7, -10.0)", + "=SUM(X!D2, C7, -10.0)"], d["inflow"] + assert d["n"][1] == '=COUNT(B7, X!C2, "x")', d["n"] + + def test_check_compat_lists_nothing(self): + m = _loop(_model()) + with m: + m.x.s = mo.SUM(m.x.a, m.x.b).set_display_name("s") + m.x.n = mo.COUNTA(m.x.a, m.x.b).set_display_name("n") + assert mo.check_compat(m, "excel") == [] + + def test_json_and_the_notebook(self): + m = _model() + with m: + m.x.s = mo.SUM(m.x.a, m.x.b).set_display_name("s") + assert mo.to_json(m.x.s._expr) == { + "kind": "funccall", "func": "SUM", + "args": [{"kind": "varref", "path": "m.x.a"}, {"kind": "varref", "path": "m.x.b"}], + } + html = m._repr_html_() + assert "SUM(B2, B3)" in html + assert "B2:D2" not in html + + +# ══════════════════════════════════════════════════════════════════════ +# Another grain: totals columns and the projection +# ══════════════════════════════════════════════════════════════════════ + + +def _monthly(): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=6) + with m: + m.p = mo.MultiVariable("P") + with m.p as p: + p.a = mo.Variable([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], display_name="a", + regrain=mo.up("sum")) + p.b = mo.Variable([10.0, 10.0, 10.0, 20.0, 20.0, 20.0], display_name="b", + regrain=mo.up("sum")) + p.cash = mo.Variable([5.0, 6.0, 7.0, 8.0, 9.0, 10.0], + display_name="cash").set_regrain(mo.up("last")) + p.price = mo.Variable([2.0, 2.0, 2.0, 3.0, 3.0, 3.0], display_name="price", + regrain=mo.up("mean")) + p.s = mo.SUM(p.a, p.b).set_display_name("s") + p.av = mo.AVERAGE(p.a, p.b).set_display_name("av") + p.pos = mo.SUM(p.cash, p.a).set_display_name("pos") + p.rev = mo.PRODUCT(p.price, p.b).set_display_name("rev") + p.twice = mo.PRODUCT(p.b, 2.0).set_display_name("twice") + return m + + +class TestAnotherGrain: + def test_a_quarter_is_the_call_over_the_operands_quarters(self): + q = _monthly().at("quarter") + assert q.p.s.value == [36.0, 75.0] + assert q.p.av.value == [18.0, 37.5] + # A balance takes its quarter's last month, a flow its sum, as + # in cash + a. + assert q.p.pos.value == [13.0, 25.0] + + def test_each_re_grains_as_its_operator_does(self): + m = _monthly() + p = m.p + with m: + p.s_op = p.a + p.b + p.av_op = (p.a + p.b) / 2 + p.pos_op = p.cash + p.a + p.rev_op = p.price * p.b + p.twice_op = p.b * 2.0 + q = m.at("quarter").p + for name in ("s", "av", "pos", "rev", "twice"): + assert getattr(q, name).value == getattr(q, f"{name}_op").value, name + # A product of two flows is a flow: the quarter sums its months. + assert q.rev.value == [60.0, 180.0] + + def test_a_count_per_period_asks_for_its_rule(self): + from modeleon.core.projection import project_model + + m = _monthly() + with m: + m.p.n = mo.COUNT(m.p.a, m.p.b).set_display_name("n") + with pytest.raises(ValueError, match="explicit rule"): + m.at("quarter") + absorbed = project_model(m, "quarter", on_error="absorb") + assert absorbed.p.n.value == "#VALUE!" + assert absorbed.p.s.value == [36.0, 75.0] + + def test_a_totals_column_folds_it_live(self, tmp_path): + from modeleon.compile.excel.view import ExcelView + + m = _monthly() + m.default_excel_view = ExcelView(timeline={"totals": ["quarter"]}) + path = tmp_path / "t.xlsx" + m.to_excel(path) + rows = _rows(load_workbook(path)["P"]) + s = [v for v in rows["s"] if isinstance(v, str) and v.startswith("=SUM(")] + assert len(s) == 8, rows["s"] + assert all("," in f for f in s), s + assert "#VALUE!" not in rows["s"] and "#VALUE!" not in rows["av"] + + +# ══════════════════════════════════════════════════════════════════════ +# The registry +# ══════════════════════════════════════════════════════════════════════ + + +class TestTheRegistry: + @pytest.mark.parametrize("name", FIVE) + def test_one_call_per_period_from_two_values(self, name): + assert per_period_arity(name) == 2 + assert not admits_in_loop(name, 1) + assert admits_in_loop(name, 2) and admits_in_loop(name, 3) + + +# ══════════════════════════════════════════════════════════════════════ +# The spreadsheet computes the same numbers +# ══════════════════════════════════════════════════════════════════════ + + +def _soffice(): + for candidate in ("soffice", "libreoffice"): + found = shutil.which(candidate) + if found: + return found + mac = Path("/Applications/LibreOffice.app/Contents/MacOS/soffice") + return str(mac) if mac.exists() else None + + +@pytest.mark.slow +def test_the_workbook_recalculates_to_the_python_values(tmp_path): + soffice = _soffice() + if soffice is None: + pytest.skip("LibreOffice not installed") + m = _loop(_model()) + with m: + m.x.s = mo.SUM(m.x.a, m.x.b).set_display_name("s") + m.x.av = mo.AVERAGE(m.x.a, m.i.floor).set_display_name("av") + m.x.pr = mo.PRODUCT(m.x.a, m.x.b, 0.5).set_display_name("pr") + m.x.n = mo.COUNT(m.x.a, [1.0, None, "x"]).set_display_name("n") + m.x.na = mo.COUNTA(m.x.a, "x").set_display_name("na") + m.x.lit = mo.SUM(m.x.a, [1.0, 9.0, 1.0]).set_display_name("lit") + m.x.twice = (mo.SUM(m.x.a, m.x.b) * 2).set_display_name("twice") + m.x.inner = mo.AVERAGE(m.x.a * 2, m.x.b).set_display_name("inner") + m.x.short = mo.Variable([5.0, 6.0], display_name="short", extend=mo.zero()) + m.x.s2 = mo.SUM(m.x.a, m.x.short).set_display_name("s2") + m.x.n2 = mo.COUNT(m.x.a, m.x.short).set_display_name("n2") + m.x.mn = mo.MIN(m.x.a, None).set_display_name("mn") + m.x.hole = mo.AVERAGE(m.x.a, [1.0, None, 3.0]).set_display_name("hole") + # On the scalars' sheet, so X keeps its period columns at B..D. + m.i.red = mo.SUM(m.x.a).set_display_name("red") + m.i.both = (mo.SUM(m.x.a) + mo.SUM(m.x.b)).set_display_name("both") + path = tmp_path / "aggregates.xlsx" + m.to_excel(path) + subprocess.run( + [soffice, f"-env:UserInstallation={(tmp_path / 'prof').as_uri()}", + "--headless", "--convert-to", "xlsx", "--outdir", + str(tmp_path / "calc"), str(path)], + check=True, capture_output=True, + ) + book = load_workbook(tmp_path / "calc" / "aggregates.xlsx", data_only=True) + x = _rows(book["X"]) + d = _rows(book["D"]) + i = _rows(book["I"]) + for name in ("s", "av", "pr", "n", "na", "lit", "twice", "inner", "s2", "n2", "mn", "hole"): + got = [round(v, 9) for v in x[name][:3]] + assert got == pytest.approx(getattr(m.x, name).value), name + assert i["red"][0] == m.i.red.value == 16.0 + assert i["both"][0] == m.i.both.value == 28.0 + for name in ("inflow", "mean", "grown", "n", "na", "bal"): + got = [round(v, 9) for v in d[name][:3]] + assert got == pytest.approx(getattr(m.d, name).value), name + + +@pytest.mark.slow +def test_the_workbook_recalculates_lines_on_windows_inside_the_sheet_s(tmp_path): + soffice = _soffice() + if soffice is None: + pytest.skip("LibreOffice not installed") + m = _windows_inside() + path = tmp_path / "inside.xlsx" + m.to_excel(path) + subprocess.run( + [soffice, f"-env:UserInstallation={(tmp_path / 'prof').as_uri()}", + "--headless", "--convert-to", "xlsx", "--outdir", + str(tmp_path / "calc"), str(path)], + check=True, capture_output=True, + ) + x = _rows(load_workbook(tmp_path / "calc" / "inside.xlsx", data_only=True)["X"]) + for name in _INSIDE: + got = [round(v, 9) for v in x[name][:4]] + assert got == pytest.approx(getattr(m.x, name).value), name + + +@pytest.mark.slow +def test_the_totals_recalculate_to_the_projection(tmp_path): + soffice = _soffice() + if soffice is None: + pytest.skip("LibreOffice not installed") + from modeleon.compile.excel.view import ExcelView + + m = _monthly() + m.default_excel_view = ExcelView(timeline={"totals": ["quarter"]}) + path = tmp_path / "totals.xlsx" + m.to_excel(path) + subprocess.run( + [soffice, f"-env:UserInstallation={(tmp_path / 'prof').as_uri()}", + "--headless", "--convert-to", "xlsx", "--outdir", + str(tmp_path / "calc"), str(path)], + check=True, capture_output=True, + ) + raw = _rows(load_workbook(path)["P"]) + calc = _rows(load_workbook(tmp_path / "calc" / "totals.xlsx", data_only=True)["P"]) + q = m.at("quarter") + for name in ("s", "av", "pos", "rev", "twice"): + native = getattr(m.p, name).value + quarters = getattr(q.p, name).value + cells = [v for v in calc[name] if v is not None] + formulas = [f for f in raw[name] if f is not None] + assert len(cells) == len(formulas) == 8, (name, raw[name]) + # Months then their quarter: Jan Feb Mar Q1 Apr May Jun Q2. + assert cells == pytest.approx(native[:3] + quarters[:1] + native[3:] + quarters[1:]), name diff --git a/tests/test_calendar.py b/tests/test_calendar.py new file mode 100644 index 0000000..5a26ec7 --- /dev/null +++ b/tests/test_calendar.py @@ -0,0 +1,181 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The calendar: rulers answer four questions; a family says how they nest.""" +from datetime import date + +import pytest + +from modeleon.core.calendar import ( + DAY, GREGORIAN, HALF, ISO_YEAR, MONTH, QUARTER, WEEK, YEAR, Calendar, Ruler, _closure, +) + + +class TestTheFourQuestions: + @pytest.mark.parametrize("ruler, label, first", [ + (DAY, "2026-01-05", date(2026, 1, 5)), + (MONTH, "2026-03", date(2026, 3, 1)), + (QUARTER, "2026-Q3", date(2026, 7, 1)), + (HALF, "2026-H2", date(2026, 7, 1)), + (YEAR, "2026", date(2026, 1, 1)), + (WEEK, "2026-W01", date(2025, 12, 29)), + (WEEK, "2026-W53", date(2026, 12, 28)), + (ISO_YEAR, "2026", date(2025, 12, 29)), + ]) + def test_parse_names_a_first_day_and_label_reads_it_back(self, ruler, label, first): + assert ruler.parse(label) == first + assert ruler.label(first) == label + + @pytest.mark.parametrize("ruler, day, first", [ + (DAY, date(2026, 2, 14), date(2026, 2, 14)), + (MONTH, date(2026, 2, 14), date(2026, 2, 1)), + (QUARTER, date(2026, 2, 14), date(2026, 1, 1)), + (QUARTER, date(2026, 11, 30), date(2026, 10, 1)), + (HALF, date(2026, 6, 30), date(2026, 1, 1)), + (HALF, date(2026, 7, 1), date(2026, 7, 1)), + (YEAR, date(2026, 11, 30), date(2026, 1, 1)), + (WEEK, date(2026, 1, 1), date(2025, 12, 29)), # a Thursday + (ISO_YEAR, date(2027, 1, 2), date(2025, 12, 29)), # a Saturday of ISO week 2026-W53 + ]) + def test_start_of_is_the_period_holding_the_day(self, ruler, day, first): + assert ruler.start_of(day) == first + + @pytest.mark.parametrize("ruler, first, nxt", [ + (DAY, date(2024, 2, 28), date(2024, 2, 29)), + (MONTH, date(2026, 12, 1), date(2027, 1, 1)), + (QUARTER, date(2026, 10, 1), date(2027, 1, 1)), + (YEAR, date(2026, 1, 1), date(2027, 1, 1)), + ]) + def test_next_start_crosses_the_year_end(self, ruler, first, nxt): + assert ruler.next_start(first) == nxt + + def test_measure_is_the_calendars_day_count(self): + assert DAY.measure(date(2026, 1, 1)) == 1 + assert MONTH.measure(date(2024, 2, 1)) == 29 + assert MONTH.measure(date(2026, 2, 1)) == 28 + assert QUARTER.measure(date(2026, 1, 1)) == 90 + assert HALF.measure(date(2026, 1, 1)) == 181 + assert HALF.measure(date(2026, 7, 1)) == 184 + assert YEAR.measure(date(2024, 1, 1)) == 366 + assert WEEK.measure(date(2026, 1, 5)) == 7 + assert ISO_YEAR.measure(date(2025, 12, 29)) == 7 * 53 # ISO 2026 has 53 weeks + assert ISO_YEAR.measure(date(2027, 1, 4)) == 7 * 52 + + def test_count_is_inclusive(self): + assert MONTH.count(date(2025, 11, 1), date(2026, 2, 1)) == 4 + assert QUARTER.count(date(2026, 1, 1), date(2026, 1, 1)) == 1 + assert DAY.count(date(2026, 1, 30), date(2026, 2, 2)) == 4 + assert YEAR.count(date(2024, 1, 1), date(2026, 1, 1)) == 3 + + def test_styles(self): + assert MONTH.label(date(2026, 3, 1), "finance") == "Mar 2026" + assert MONTH.label(date(2026, 3, 1), "compact") == "Mar-26" + assert QUARTER.label(date(2026, 7, 1), "finance") == "Q3 2026" + assert HALF.label(date(2026, 7, 1), "finance") == "H2 2026" + assert HALF.label(date(2026, 7, 1), "compact") == "H2-26" + assert HALF.label(date(2026, 1, 1)) == "2026-H1" + assert YEAR.label(date(2026, 1, 1), "compact") == "FY26" + assert WEEK.label(date(2025, 12, 29), "finance") == "W01 2026" + assert WEEK.label(date(2025, 12, 29), "compact") == "W01-26" + assert ISO_YEAR.label(date(2025, 12, 29)) == "2026" + assert DAY.label(date(2026, 1, 5), "finance") == "05 Jan 2026" + + @pytest.mark.parametrize("ruler, label, message", [ + (DAY, "2026-W01", "Time day start must be 'YYYY-MM-DD'"), + (MONTH, "2026-13", "Month must be 01-12"), + (MONTH, "2026", "Time month start must be 'YYYY-MM'"), + (QUARTER, "2026-Q5", "Quarter must be Q1-Q4"), + (HALF, "2026-H3", "Half must be H1-H2"), + (HALF, "2026-Q1", "Time half start must be 'YYYY-Hn'"), + (YEAR, "FY26", "Time year start must be 'YYYY'"), + (WEEK, "2026-01", "Time week start must be 'YYYY-Www'"), + (WEEK, "2027-W53", "names no ISO week: the year 2027 has 52 weeks"), + (ISO_YEAR, "W2026", "Time iso_year start must be 'YYYY'"), + ]) + def test_a_bad_label_is_refused_with_its_grammar(self, ruler, label, message): + with pytest.raises(ValueError, match=message): + ruler.parse(label) + + +class TestTheFamily: + def test_the_gregorian_rulers_are_the_grains_finest_first(self): + assert GREGORIAN.grains == ("hour", "day", "week", "month", "quarter", "half", "year", + "iso_year") + assert [r.rank for r in GREGORIAN.rulers] == [0, 1, 2, 3, 4, 5, 6, 7] + assert [(r.word, r.letter) for r in GREGORIAN.rulers] == [ + ("Hourly", "Hr"), ("Daily", "D"), ("Weekly", "W"), ("Monthly", "M"), + ("Quarterly", "Q"), ("Half-yearly", "H"), ("Yearly", "Y"), ("Yearly (ISO)", "IY"), + ] + + def test_nesting_is_a_relation_not_an_order(self): + assert GREGORIAN.nests("day", "month") + assert GREGORIAN.nests("month", "year") + assert GREGORIAN.nests("quarter", "half") and GREGORIAN.nests("half", "year") + assert not GREGORIAN.nests("year", "month") + assert not GREGORIAN.nests("month", "month") + # A week lies inside one ISO year and in no month, quarter or year. + assert GREGORIAN.nests("day", "week") and GREGORIAN.nests("week", "iso_year") + assert GREGORIAN.nests("day", "iso_year") + for coarse in ("month", "quarter", "half", "year"): + assert not GREGORIAN.nests("week", coarse) + assert not GREGORIAN.nests("month", "iso_year") + assert not GREGORIAN.comparable("year", "iso_year") + assert GREGORIAN.rolls_up_to("week") == ("iso_year",) + assert GREGORIAN.rolls_up_to("day") == ("week", "month", "quarter", "half", "year", "iso_year") + # An hour lies inside its day, and so inside everything a day is. + assert GREGORIAN.rolls_up_to("hour") == ("day",) + GREGORIAN.rolls_up_to("day") + assert not GREGORIAN.nests("day", "hour") and not GREGORIAN.nests("hour", "hour") + + def test_every_claim_is_checked_on_the_line(self): + GREGORIAN.verify() + # A false claim names its first bad period. + bad = Calendar("bad", (DAY, WEEK, MONTH), _closure([("day", "week"), ("week", "month")])) + with pytest.raises(AssertionError, match="week does not nest in month"): + bad.verify() + + def test_coarser_never_says_yes_to_a_word_it_does_not_know(self): + assert GREGORIAN.coarser("quarter", "month") + assert GREGORIAN.coarser("half", "quarter") + assert not GREGORIAN.coarser("month", "quarter") + assert not GREGORIAN.coarser("fortnight", "month") + assert not GREGORIAN.coarser("year", "week") + assert not GREGORIAN.coarser("month", "week") + assert GREGORIAN.coarser("iso_year", "week") + assert not GREGORIAN.coarser(None, "month") + + def test_an_unknown_ruler_is_refused_by_name(self): + with pytest.raises(ValueError, match="Unknown time grain 'minute'; supported: hour, day, week, month, quarter, half, year, iso_year"): + GREGORIAN.ruler("minute") + assert not GREGORIAN.knows("minute") + assert GREGORIAN.knows("week") and GREGORIAN.knows("hour") + + def test_whole_months_and_stubs(self): + assert [r.months for r in GREGORIAN.rulers] == [None, None, None, 1, 3, 6, 12, None] + assert [r.stub_unit for r in GREGORIAN.rulers] == [None, None, None, None, "month", "month", "month", "week"] + assert [r.stride_days for r in GREGORIAN.rulers] == [None, 1, 7, None, None, None, None, None] + assert [r.stride_hours for r in GREGORIAN.rulers] == [1, None, None, None, None, None, None, None] + assert [r.name for r in GREGORIAN.rulers if r.sub_day] == ["hour"] + + def test_a_ruler_is_a_rule_not_a_table(self): + # The base class answers measure and count from next_start alone, + # so a new ruler needs only the four questions. + class Fortnight(Ruler): + name, rank = "fortnight", 0 + + def parse(self, label): + return date.fromisoformat(label) + + def label(self, first, style="iso"): + return first.isoformat() + + def start_of(self, day): + return day + + def next_start(self, first): + from datetime import timedelta + return first + timedelta(days=14) + + f = Fortnight() + assert f.measure(date(2026, 1, 1)) == 14 + assert f.count(date(2026, 1, 1), date(2026, 1, 29)) == 3 + cal = Calendar("test", (f,)) + assert cal.grains == ("fortnight",) + assert repr(cal) == "Calendar('test', ('fortnight',))" diff --git a/tests/test_choose_keyed_line.py b/tests/test_choose_keyed_line.py new file mode 100644 index 0000000..e24ed3c --- /dev/null +++ b/tests/test_choose_keyed_line.py @@ -0,0 +1,288 @@ +# SPDX-License-Identifier: Apache-2.0 +"""``CHOOSE`` over a line that runs along keys is one choice per key. + +A keyed line (``mo.Variable({"Bear": 0.9, "Base": 1.0, "Bull": 1.1})``) +or a line on one finite axis that is not time runs along KEYS, not +periods. ``mo.CHOOSE(pick, growth)`` used to read it as ONE series +choice: its keys were lost, a 3-key line in a 4-period window failed +the length check (``'g' has 3 value(s) in a 4-period window``), in a +3-period window it silently became a period series, and the workbook +held ``=CHOOSE(B1, B3)`` in every period — one option, ``#VALUE!`` in +Excel. Such a line is now spread into its elements, one choice per key +in key order, so ``mo.CHOOSE(pick, growth)`` is ``mo.CHOOSE(pick, +growth["Bear"], growth["Base"], growth["Bull"])``: value ``1.0`` for +``pick = 2``, written ``=CHOOSE(B1, B3, C3, D3)`` over the line's cells +(one such formula per period when the index is a series). A tracked +line, an index that runs along keys and a line on two axes are refused +with an error that names the fix. Bare series choices keep their branch: +``mo.CHOOSE(scenario, base, bull)`` still writes one CHOOSE per period +over that period's cells. +""" + +import pytest +from openpyxl import load_workbook + +import modeleon as mo + + +def _rows(model, sheet, tmp_path, n, cols="ABCDEFGH"): + path = tmp_path / "t.xlsx" + model.to_excel(str(path)) + ws = load_workbook(str(path))[sheet] + return [[ws[f"{c}{r}"].value for c in cols] for r in range(1, n + 1)] + + +def _keyed(): + return mo.Variable({"Bear": 0.9, "Base": 1.0, "Bull": 1.1}, display_name="Growth") + + +def _scenario_axis(): + return mo.Variable(["Bear", "Base", "Bull"]) + + +def _model(pick): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.inputs = mo.MultiVariable("Inputs", excel_props={"tab": True}) + with m.inputs as i: + i.pick = mo.Variable(pick, display_name="Pick") + i.growth = _keyed() + m.out = mo.MultiVariable("Out", excel_props={"tab": True}) + m.out.g = mo.CHOOSE(i.pick, i.growth).set_display_name("G") + return m + + +def _tracked_model(): + m = mo.Model("t", default_grain="month", default_start="2026-01", default_periods=3, + tracks=mo.Tracks("plan", "actual")) + with m: + m.x = mo.Variable(plan=[1, 2, 3], actual=[1, 2, 3], display_name="X") + m.pick = mo.Variable(1) + return m + + +def _grid(): + a = mo.Variable(["x", "y"]) + b = mo.Variable(["p", "q"]) + return mo.Variable([1, 2, 3, 4], indexed_by=[a, b], display_name="Grid") + + +class TestTheValue: + def test_a_keyed_choice_is_one_choice_per_key(self): + g = _keyed() + r = mo.CHOOSE(mo.Variable(2), g) + assert r._value == 1.0 + assert r.var_type == "scalar" + assert mo.CHOOSE(3, g)._value == 1.1 + assert mo.CHOOSE(4, g)._value == "#VALUE!" + + def test_the_short_form_is_the_long_form(self): + g = _keyed() + pick = mo.Variable(2) + short = mo.CHOOSE(pick, g) + long = mo.CHOOSE(pick, g["Bear"], g["Base"], g["Bull"]) + assert short.formula == long.formula + assert short._value == long._value + # The spread is an AST of element references, never a value. + assert short._expr is not None + assert len(short._expr.args) == 4 + + def test_a_line_on_a_finite_axis_spreads_by_position(self): + scen = _scenario_axis() + ga = mo.Variable([0.9, 1.0, 1.1], indexed_by=[scen], display_name="Growth") + assert mo.CHOOSE(3, ga)._value == 1.1 + assert mo.CHOOSE(3, ga).formula == mo.CHOOSE(3, ga[0], ga[1], ga[2]).formula + + def test_a_per_period_index_picks_a_key_each_period(self): + r = mo.CHOOSE(mo.Variable([1, 2, 3, 2]), _keyed()) + assert r._value == [0.9, 1.0, 1.1, 1.0] + assert r.var_type == "list" + + def test_a_keyed_choice_beside_other_choices_is_refused(self): + # Beside other choices the numbering would depend on how many keys + # the line has, and two keyed lines would flatten into one row. + with pytest.raises(ValueError, match="beside 2 other choice") as e: + mo.CHOOSE(2, 0.5, _keyed(), 2.0) + assert 'mo.CHOOSE(pick, Growth)' in str(e.value) + assert "Growth['Bear'], Growth['Base'], Growth['Bull']" in str(e.value) + base = mo.Variable({"A": 10.0, "B": 20.0}, display_name="Base") + bull = mo.Variable({"A": 12.0, "B": 25.0}, display_name="Bull") + with pytest.raises(ValueError, match="beside 1 other choice"): + mo.CHOOSE(2, base, bull) + + def test_a_keyed_choice_beside_a_series_choice_is_refused_too(self): + with pytest.raises(ValueError, match="beside 1 other choice"): + mo.CHOOSE(1, _keyed(), mo.Variable([10, 20, 30])) + + def test_integer_keys_spread_by_position(self): + # An int key inside the positional range is read as a POSITION by + # ``[]`` and by the workbook alike, so the spread goes by position + # and both agree — key order, whatever the ints say. + g = mo.Variable({1: 0.9, 2: 1.0, 3: 1.1}) + assert [mo.CHOOSE(p, g)._value for p in (1, 2, 3)] == [0.9, 1.0, 1.1] + g2 = mo.Variable({3: 0.9, 2: 1.0, 1: 1.1}) + assert [mo.CHOOSE(p, g2)._value for p in (1, 2, 3)] == [0.9, 1.0, 1.1] + + def test_a_computed_keyed_choice_is_refused_until_named(self, tmp_path): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=3) + with m: + m.pick = mo.Variable(2, display_name="Pick") + m.growth = _keyed() + with pytest.raises(ValueError, match="computed line with no cells") as e: + mo.CHOOSE(m.pick, m.growth * 2) + assert "growth * 2" in str(e.value) + m.growth_adj = mo.Variable(m.growth * 2, display_name="Growth adj") + m.g = mo.CHOOSE(m.pick, m.growth_adj).set_display_name("G") + assert m.g._value == 2.0 + rows = _rows(m, "M", tmp_path, 6, "ABCDE") + g_row = next(r for r in rows if r[0] == "G") + # live: one CHOOSE over the three cells of the NAMED computed line + assert g_row[1] == "=CHOOSE(B1, B4, C4, D4)" + + def test_the_lines_unit_carries(self): + from modeleon.core.unit import unit_key + + g = mo.Variable({"Bear": 0.9, "Base": 1.0, "Bull": 1.1}, unit="$") + r = mo.CHOOSE(1, g) + assert unit_key(r._unit) == unit_key(g._unit) + + +class TestTheRenderers: + def test_the_workbook_writes_one_choose_over_the_lines_cells(self, tmp_path): + m = _model(2) + # No per-period line on Inputs, so no period header: Pick is row + # 1, the caption row of keys is row 2, Growth's cells are row 3. + assert _rows(m, "Inputs", tmp_path, 3, "ABCD") == [ + ["Pick", 2, None, None], + [None, "Bear", "Base", "Bull"], + ["Growth", 0.9, 1, 1.1], + ] + assert _rows(m, "Out", tmp_path, 1, "AB") == [ + ["G", "=CHOOSE(Inputs!B1, Inputs!B3, Inputs!C3, Inputs!D3)"], + ] + + def test_a_series_index_writes_one_choose_per_period_over_the_same_cells( + self, tmp_path, + ): + m = _model([1, 2, 3, 2]) + assert m.out.g._value == [0.9, 1.0, 1.1, 1.0] + # Inputs now carries a period header (row 1): Pick is row 2, the + # caption row 3, Growth row 4. Each period's CHOOSE reads its own + # Pick cell and the same three Growth cells. + rows = _rows(m, "Out", tmp_path, 2, "ABCDE") + assert rows[1] == ["G"] + [ + f"=CHOOSE(Inputs!{c}2, Inputs!B4, Inputs!C4, Inputs!D4)" for c in "BCDE" + ] + + def test_the_notebook_repr_shows_the_same_formula(self): + assert "=CHOOSE(Inputs!B1, Inputs!B3, Inputs!C3, Inputs!D3)" in _model(2)._repr_html_() + + def test_the_json_carries_one_argument_per_choice(self): + out = mo.to_json(_model(2).out.g._expr) + assert out["kind"] == "funccall" + assert out["func"] == "CHOOSE" + assert [a["kind"] for a in out["args"]] == ["varref"] * 4 + assert out["args"][0]["path"] == "m.inputs.pick" + + def test_the_choice_depends_on_the_line(self): + m = _model(2) + deps = m.out.g._dependency_refs + assert deps[0] is m.inputs.pick + assert len(deps) == 4 + assert all(e._dependency_refs == [m.inputs.growth] for e in deps[1:]) + + +class TestTheRefusals: + def test_tracked_choices_still_switch_track_by_track(self, tmp_path): + # The usual scenario switch over plan/actual rows: one index, tracked + # choices — computed per track, one CHOOSE row per track in the book. + m = mo.Model("t", default_grain="month", default_start="2026-01", default_periods=3, + tracks=mo.Tracks("plan", "actual")) + with m: + m.scenario = mo.Variable(2, display_name="Scenario") + m.base = mo.Variable(plan=[100, 110, 120], actual=[101, 111, 121], display_name="Base") + m.bull = mo.Variable(plan=[150, 170, 190], actual=[151, 171, 191], display_name="Bull") + m.pick = mo.CHOOSE(m.scenario, m.base, m.bull).set_display_name("Pick") + assert list(m.pick._value['plan']) == [150, 170, 190] + assert list(m.pick._value['actual']) == [151, 171, 191] + rows = _rows(m, "T", tmp_path, 8, "ABCD") + formulas = [r for r in rows if r[0] and str(r[0]).startswith("Pick")] + assert len(formulas) == 2 + assert all(str(r[2]).startswith("=CHOOSE(") for r in formulas) + + def test_a_series_index_over_tracked_choices_is_refused(self): + m = mo.Model("t", default_grain="month", default_start="2026-01", default_periods=3, + tracks=mo.Tracks("plan", "actual")) + with m: + m.base = mo.Variable(plan=[100, 110, 120], actual=[101, 111, 121], display_name="Base") + m.flag = mo.Variable([1, 2, 1], display_name="Flag") + with pytest.raises(ValueError, match="each period would pick a whole tracked") as e: + mo.CHOOSE(m.flag, m.base, 5) + assert "base.plan" in str(e.value) + + def test_a_tracked_index_is_refused(self): + m = _tracked_model() + with pytest.raises(ValueError, match="tracked line"): + mo.CHOOSE(m.x, 1, 2) + + def test_a_choice_on_two_axes_is_refused(self): + grid = _grid() + with pytest.raises(ValueError, match="2 axes"): + mo.CHOOSE(1, grid) + with pytest.raises(ValueError, match="2 axes"): + mo.CHOOSE(grid, 1, 2) + + def test_an_index_along_keys_is_refused(self): + with pytest.raises(ValueError, match="runs along keys") as e: + mo.CHOOSE(mo.Variable({"a": 1, "b": 2}, display_name="Pick"), 10, 20) + assert "Pick['a']" in str(e.value) + scen = _scenario_axis() + axis_pick = mo.Variable([1, 2, 3], indexed_by=[scen], display_name="Pick") + with pytest.raises(ValueError, match="runs along keys") as e: + mo.CHOOSE(axis_pick, 10, 20, 30) + assert "Pick[0]" in str(e.value) + + +class TestWhatStays: + def test_a_line_on_the_time_axis_is_still_a_series(self): + t = mo.Time("2026-01", periods=3) + tl = mo.Variable([1, 2, 3], indexed_by=[t], display_name="TL") + first = mo.CHOOSE(1, tl, 5) + assert first._value == [1, 2, 3] + assert first.var_type == "list" + second = mo.CHOOSE(2, tl, 5) + assert second._value == [5, 5, 5] + assert second.var_type == "list" + + def test_bare_series_choices_are_unchanged(self): + r = mo.CHOOSE(mo.Variable(2), mo.Variable([100, 110, 120]), mo.Variable([150, 170, 190])) + assert r._value == [150, 170, 190] + # A series is one choice, never spread. + assert len(r._expr.args) == 3 + assert mo.CHOOSE(1, 125, mo.Variable([150, 170, 190]))._value == [125, 125, 125] + assert mo.CHOOSE(mo.Variable([0, 0, 1, 1]) + 1, "A", "F")._value == ["A", "A", "F", "F"] + + def test_line_keys_lives_in_core_and_the_layout_shares_it(self): + from modeleon.compile.excel import layout + from modeleon.core.shape import line_keys + + assert layout.line_keys is line_keys + scen = _scenario_axis() + assert line_keys(_keyed()) == ["Bear", "Base", "Bull"] + axised = mo.Variable([0.9, 1.0, 1.1], indexed_by=[scen]) + assert line_keys(axised) == ["Bear", "Base", "Bull"] + on_time = mo.Variable([1, 2, 3], indexed_by=[mo.Time("2026-01", periods=3)]) + assert line_keys(on_time) is None + assert line_keys(mo.Variable([1, 2, 3])) is None + assert line_keys(mo.Variable(5)) is None + assert line_keys(_grid()) is None + assert line_keys(_tracked_model().x) is None + + +def test_a_wrapper_keeps_the_keys_and_axes_of_the_line_it_wraps(): + g = mo.Variable({"Bear": 0.9, "Base": 1.0, "Bull": 1.1}) + assert mo.Variable(g * 2, display_name="Twice")._keys == ["Bear", "Base", "Bull"] + scen = mo.Variable(["Bear", "Base", "Bull"]) + a = mo.Variable([1, 2, 3], indexed_by=[scen]) + assert mo.Variable(a * 2, display_name="Twice")._indexed_by == (scen,) + assert mo.Variable(g, keys=["x", "y", "z"])._keys == ["x", "y", "z"], "a declared key set wins" diff --git a/tests/test_component.py b/tests/test_component.py index f755466..2455895 100644 --- a/tests/test_component.py +++ b/tests/test_component.py @@ -28,6 +28,25 @@ def test_multivariable_is_a_component(self): assert isinstance(m, Component) +class TestBareVariableFields: + """``mo.Variable()`` with no label, props or description writes the + fields ``Component.__init__`` would, without walking the chain (every + operator result is built that way). A subclass always walks it, so + comparing the two pins the shortcut to what it stands for.""" + + def test_same_fields_in_the_same_order(self): + class Walked(mo.Variable): + pass + + bare, walked = mo.Variable(), Walked() + assert list(vars(bare)) == list(vars(walked)) + assert vars(bare) == vars(walked) + + def test_each_variable_gets_its_own_props_dict(self): + a, b = mo.Variable(), mo.Variable() + assert a.excel_props is not b.excel_props + + class TestComponentTypingTarget: """Functions that should accept any model-tree node take ``Component`` as their parameter type. This pins that Variable and MultiVariable diff --git a/tests/test_criteria.py b/tests/test_criteria.py new file mode 100644 index 0000000..d2d9bef --- /dev/null +++ b/tests/test_criteria.py @@ -0,0 +1,757 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Conditional aggregates and the counts: the criteria read as the +spreadsheet reads it, a range a row or a list of scalar lines, one +number each with its formula.""" + +import datetime as dt + +import pytest + +import modeleon as mo +from modeleon.core.expr import FuncCall, ListExpr, Literal, VarRef +from modeleon.functions.criteria import ( + AVERAGEIF, + AVERAGEIFS, + COUNT, + COUNTA, + COUNTIF, + COUNTIFS, + MAXIFS, + MINIFS, + SUMIF, + SUMIFS, + matches, + parse_criteria, +) +from modeleon.functions.registry import ( + admits_in_loop, + file_prefix, + is_passthrough, + is_range_taking, + range_positions, + spec, +) + +# ══════════════════════════════════════════════════════════════════════ +# The criteria grammar +# ══════════════════════════════════════════════════════════════════════ + + +class TestParseCriteria: + @pytest.mark.parametrize("criteria, parsed", [ + (">=10", (">=", 10)), + ("<>0", ("<>", 0)), + ("<5", ("<", 5)), + ("<=-2.5", ("<=", -2.5)), + (">1e3", (">", 1000.0)), + (">= 10", (">=", 10)), + ("=text", ("=", "text")), + ("text", ("=", "text")), + ("N*", ("=", "N*")), + ("", ("=", "")), + ("=", ("=", None)), + ("<>", ("<>", None)), + (">=", (">=", None)), + ("<>#DIV/0!", ("<>", "#DIV/0!")), + ("5%", ("=", 0.05)), + ("TRUE", ("=", True)), + ("=false", ("=", False)), + (10, ("=", 10)), + (-3.5, ("=", -3.5)), + (True, ("=", True)), + (None, ("=", 0)), + (dt.date(2026, 1, 1), ("=", dt.date(2026, 1, 1))), + (dt.timedelta(hours=12), ("=", dt.timedelta(hours=12))), + # A date written 2026-01-31 is that date, a time of day after it or not. + (">=2026-01-01", (">=", dt.date(2026, 1, 1))), + ("2026-1-31", ("=", dt.date(2026, 1, 31))), + ("<2026-01-31 12:00", ("<", dt.datetime(2026, 1, 31, 12))), + ("2026-01-31 9:05:30.5", ("=", dt.datetime(2026, 1, 31, 9, 5, 30, 500000))), + ]) + def test_an_operator_and_a_comparand_read_as_a_cell_entry(self, criteria, parsed): + assert parse_criteria(criteria) == parsed + + @pytest.mark.parametrize("text", [ + "1_000", "nan", "inf", "x10", "1e3%", "Q1 2026", "10-K", "A-100", "T1", "#5", + "Jan", "100 USD", "Jan?2026", "10?20", "2026-02-30", "a.b", "(x)", + ]) + def test_what_does_not_read_as_a_number_is_text(self, text): + op, comparand = parse_criteria(text) + assert isinstance(comparand, str) and not isinstance(comparand, bool) + + @pytest.mark.parametrize("text", [ + "1,000", ">$100", "100€", "(5)", "- 5", "5 %", "1,5", "1 000", "12:00", "<1 PM", + "1/31/2026", "31.01.2026", "1/2", "10-20", "Jan 31, 2026", "31-Jan-2026", "Jan 2026", + "2026-01-31T12:00", "1899-12-31", "2026-01-31 25:00", + ]) + def test_an_entry_read_as_a_number_by_locale_is_refused(self, text): + # The spreadsheet reads these as a number or a date in some locales + # and as text in others: no answer is the workbook's in every one. + with pytest.raises(ValueError, match="in some spreadsheets and as text in others"): + parse_criteria(text) + with pytest.raises(ValueError, match=r"^COUNTIF\(\): the criteria entry"): + COUNTIF(mo.Variable([1, 2]), text) + + +class TestMatches: + @pytest.mark.parametrize("value, criteria, hit", [ + # numbers + (5, ">=5", True), (4.9, ">=5", False), (-6, "<-5", True), (-4, "<-5", False), + (5, "<>5", False), (6, "<>5", True), (5, 5.0, True), (5, "=5", True), + (0.05, "5%", True), + # what is not a number never meets a numeric comparison but '<>' + (None, ">0", False), (None, "<>5", True), ("x", ">0", False), ("x", "<>5", True), + (True, 1, False), (True, "<>1", True), ("#DIV/0!", ">0", False), ("#DIV/0!", "<>0", True), + # text that reads as a number meets an equality, never an ordering + ("10", 10, True), ("10", "=10", True), ("10", "<>10", False), ("ten", 10, False), + ("10", ">=10", False), ("9.5", "<10", False), ("-1", "<0", False), ("10", ">5", False), + # two numbers are equal as the spreadsheet compares them + (0.1 + 0.2, 0.3, True), (0.1 + 0.2, "<>0.3", False), (0.1 + 0.2, ">0.3", False), + (0.1 + 0.2, ">=0.3", True), (0.7 - 0.4, "<=0.3", True), (0.7 - 0.4, "<0.3", False), + (1 + 15 * 2.0 ** -52, 1, True), (1 + 16 * 2.0 ** -52, 1, False), + (-(1 + 16 * 2.0 ** -52), -1, False), (2.0 ** 50 + 2, 2 ** 50, False), (1e-300, 0, False), + # text, whole cell, case-insensitively + ("North", "north", True), ("North", "NORTH", True), ("Northern", "North", False), + ("North", "=North", True), ("North", "<>north", False), ("South", "<>north", True), + (5, "North", False), (5, "<>North", True), (None, "<>North", True), + # wildcards and the escape + ("Northern", "North*", True), ("North", "N?rth", True), ("Nrth", "N?rth", False), + ("abc", "*", True), ("", "*", True), (5, "*", False), (None, "*", False), + ("a*b", "a~*b", True), ("axb", "a~*b", False), ("a?b", "a~?b", True), + ("a~b", "a~~b", True), ("ab", "a~b", True), ("a~b", "a~b", False), + ("a.b", "a.b", True), ("axb", "a.b", False), + ("North", "~north", True), ("north~", "north~", True), ("north", "north~", False), + # text ordered, case-insensitively + ("b", ">a", True), ("B", ">a", True), ("a", ">a", False), ("a", ">=a", True), + (5, ">a", False), (None, ">a", False), + # booleans; the text TRUE meets the criteria written "TRUE" + (True, True, True), (False, "FALSE", True), (1, True, False), (True, "<>FALSE", True), + (None, "TRUE", False), ("TRUE", "TRUE", True), ("true", "TRUE", True), + ("TRUE", "<>TRUE", False), ("TRUE", True, False), (1, "TRUE", False), + # dates are their serials: a date cell meets a numeric criteria, + # a number cell a date one + (dt.date(2026, 1, 1), dt.date(2026, 1, 1), True), + (dt.datetime(2026, 1, 1, 12), dt.date(2026, 1, 1), False), + (dt.datetime(2026, 1, 1, 12), ">46023", True), + (46023, dt.date(2026, 1, 1), True), ("46023", dt.date(2026, 1, 1), True), + (dt.date(2026, 1, 1), 46023, True), (dt.date(2026, 1, 1), ">45000", True), + (dt.date(2025, 12, 31), ">=46023", False), (dt.date(2026, 1, 1), "<>", True), + ("x", dt.date(2026, 1, 1), False), (None, "<>46023", True), + # a date written 2026-01-31 is a date; the text equal to it meets it too + (dt.date(2026, 1, 31), "2026-01-31", True), (dt.date(2026, 1, 1), ">=2026-01-01", True), + (dt.date(2025, 12, 31), ">=2026-01-01", False), (46053, "2026-01-31", True), + (dt.datetime(2026, 1, 31, 12), "2026-01-31", False), + (dt.datetime(2026, 1, 31, 12), "2026-01-31 12:00", True), + ("2026-01-31", "2026-01-31", True), ("2026-01-31", dt.date(2026, 1, 31), False), + ("2026-01-31", ">=2026-01-01", False), + # errors + ("#N/A", "#N/A", True), ("#n/a", "#N/A", True), (5, "#N/A", False), + ("#N/A", "x", False), ("#N/A", "<>x", True), ("#N/A", "<>#N/A", False), + # blanks and empty text ('' here is a formula's empty text, a cell; + # typed into a line it is read as a blank — TestCOUNTIF, TestCOUNTA) + (None, "", True), ("", "", True), (0, "", False), (None, "=", True), ("", "=", False), + (None, "<>", False), ("", "<>", True), (0, "<>", True), ("x", "<>", True), + # an ordering with nothing after it compares text with the empty text + ("", ">=", True), ("x", ">=", True), (None, ">=", False), (5, ">=", False), + ("x", ">", True), ("", ">", False), ("", "<=", True), ("x", "<=", False), + ("", "<", False), (True, ">=", False), ("#N/A", ">=", False), + # None as the criteria is 0, as a reference to an empty cell reads + (0, None, True), (0.0, None, True), (None, None, False), ("", None, False), + ("0", None, True), (False, None, False), + ]) + def test_one_cell_against_one_criteria(self, value, criteria, hit): + assert matches(value, criteria) is hit + + +# ══════════════════════════════════════════════════════════════════════ +# The functions, over rows +# ══════════════════════════════════════════════════════════════════════ + + +def _rows(): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.d = mo.MultiVariable("D") + m.d.region = mo.Variable(["North", "South", "North", "West"], display_name="region") + m.d.amount = mo.Variable([100.0, 200.0, 300.0, 400.0], display_name="amount", unit="USD") + m.d.units = mo.Variable([1, 2, 3, 4], display_name="units", unit="pcs") + m.d.year = mo.Variable([2026, 2026, 2027, 2027], display_name="year") + m.i = mo.MultiVariable("I") + m.i.floor = mo.Variable(250.0, display_name="floor") + m.i.pick = mo.Variable("north", display_name="pick") + m.i.rule = mo.Variable(">=200", display_name="rule") + return m + + +class TestSUMIF: + def test_over_a_sum_range_and_over_the_range_itself(self): + m = _rows() + assert SUMIF(m.d.region, "North", m.d.amount).value == 400.0 + assert SUMIF(m.d.amount, ">=200").value == 900.0 + assert SUMIF(m.d.amount, "<>100").value == 900.0 + + def test_every_criteria_kind(self): + m = _rows() + assert SUMIF(m.d.region, m.i.pick, m.d.amount).value == 400.0 + assert SUMIF(m.d.amount, m.i.rule).value == 900.0 + assert SUMIF(m.d.amount, (">=", m.i.floor)).value == 700.0 + assert SUMIF(m.d.amount, mo.CONCAT(">=", m.i.floor)).value == 700.0 + assert SUMIF(m.d.amount, (">", 250)).value == 700.0 + assert SUMIF(m.d.year, 2027, m.d.amount).value == 700.0 + assert SUMIF(m.d.region, "n*", m.d.amount).value == 400.0 + + def test_a_one_cell_range(self): + m = _rows() + assert SUMIF(m.i.floor, ">=100").value == 250.0 + assert SUMIF(m.i.floor, "<100").value == 0 + + def test_nothing_matched_is_zero(self): + m = _rows() + assert SUMIF(m.d.region, "East", m.d.amount).value == 0 + + def test_the_sum_range_skips_blanks_text_and_booleans(self): + flags = mo.Variable(["y", "y", "y", "y", "y"]) + mixed = mo.Variable([10.0, None, "text", True, 5.0]) + assert SUMIF(flags, "y", mixed).value == 15.0 + + def test_an_error_among_the_matched_cells_is_the_result(self): + flags = mo.Variable(["y", "n", "y"]) + assert SUMIF(flags, "y", mo.Variable([1.0, "#DIV/0!", 2.0])).value == 3.0 + assert SUMIF(flags, "y", mo.Variable([1.0, 2.0, "#DIV/0!"])).value == "#DIV/0!" + assert SUMIF(flags, "y", mo.Variable(["#N/A", 2.0, "#DIV/0!"])).value == "#N/A" + + def test_an_error_in_the_criteria_range_matches_only_its_text(self): + keys = mo.Variable(["a", "#DIV/0!", "a"]) + amounts = mo.Variable([1.0, 2.0, 3.0]) + assert SUMIF(keys, "a", amounts).value == 4.0 + assert SUMIF(keys, "#DIV/0!", amounts).value == 2.0 + assert SUMIF(keys, "<>a", amounts).value == 2.0 + + def test_the_result_is_a_scalar_whose_type_follows_the_summed_range(self): + m = _rows() + out = SUMIF(m.d.region, "North", m.d.amount) + assert out.var_type == "scalar" and out.value_type == "float" + assert SUMIF(m.d.region, "North", m.d.units).value_type == "int" + assert SUMIF(m.d.region, "North", m.d.units).value == 4 + # A sum is a number, over a range of text too. + assert SUMIF(m.d.region, "North").value_type == "float" + + def test_the_unit_is_the_summed_range_s_never_the_criteria_range_s(self): + from modeleon.core.unit import unit_key + m = _rows() + assert unit_key(SUMIF(m.d.region, "North", m.d.amount)._unit) == unit_key(m.d.amount._unit) + assert unit_key(SUMIF(m.d.units, ">1", m.d.amount)._unit) == unit_key(m.d.amount._unit) + assert SUMIF(m.d.units, ">1", m.d.year)._unit is None + assert SUMIF(m.d.amount, ">1", m.d.year)._unit is None + + def test_ranges_of_different_lengths_are_refused(self): + m = _rows() + with pytest.raises(ValueError, match="same number of cells"): + SUMIF(m.d.region, "North", mo.Variable([1.0, 2.0])) + + def test_a_series_criteria_is_refused(self): + m = _rows() + with pytest.raises(ValueError, match="a criteria is one value"): + SUMIF(m.d.amount, m.d.units) + with pytest.raises(ValueError, match="a criteria is one value"): + SUMIF(m.d.amount, [1, 2]) + + def test_a_bad_pair_is_refused(self): + m = _rows() + with pytest.raises(ValueError, match=r"\(operator, value\)"): + SUMIF(m.d.amount, ("~", 5)) + with pytest.raises(ValueError, match=r"\(operator, value\)"): + SUMIF(m.d.amount, (">=", 5, 6)) + + def test_a_line_with_tracks_is_refused(self): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=2, + tracks=mo.Tracks("plan", "actual")) + with m: + m.d = mo.MultiVariable("D") + m.d.amount = mo.Variable(plan=[1.0, 2.0], actual=[1.5, 2.5]) + with pytest.raises(ValueError, match="tracks"): + SUMIF(m.d.amount, ">1") + + def test_a_range_that_is_neither_a_line_nor_a_list_is_refused(self): + with pytest.raises(TypeError, match="a range is a list-valued line"): + SUMIF(5, ">1") + + +class TestSUMIFS: + def test_every_pair_must_hold(self): + m = _rows() + assert SUMIFS(m.d.amount, m.d.region, "North", m.d.year, 2027).value == 300.0 + assert SUMIFS(m.d.amount, m.d.region, "North").value == 400.0 + assert SUMIFS(m.d.amount, m.d.region, "<>North", m.d.amount, (">", m.i.floor)).value == 400.0 + assert SUMIFS(m.d.amount, m.d.region, "East").value == 0 + + def test_the_pairs_are_checked(self): + m = _rows() + with pytest.raises(ValueError, match="in pairs"): + SUMIFS(m.d.amount) + with pytest.raises(ValueError, match="in pairs"): + SUMIFS(m.d.amount, m.d.region) + with pytest.raises(ValueError, match="same number of cells"): + SUMIFS(m.d.amount, mo.Variable([1, 2]), 1) + + def test_type_and_unit_follow_the_summed_range(self): + from modeleon.core.unit import unit_key + m = _rows() + out = SUMIFS(m.d.units, m.d.region, "North") + assert out.value == 4 and out.value_type == "int" and out.var_type == "scalar" + assert unit_key(out._unit) == unit_key(m.d.units._unit) + + +class TestCOUNTIF: + def test_counts_the_cells_that_meet_the_criteria(self): + m = _rows() + assert COUNTIF(m.d.region, "North").value == 2 + assert COUNTIF(m.d.region, "n*").value == 2 + assert COUNTIF(m.d.region, "<>North").value == 2 + assert COUNTIF(m.d.amount, ">=200").value == 3 + assert COUNTIF(m.d.amount, (">", m.i.floor)).value == 2 + assert COUNTIF(m.d.year, m.d.year[0]).value == 2 + + def test_blanks_numeric_text_and_errors(self): + row = mo.Variable([None, "", "10", 10, "#N/A", True]) + assert COUNTIF(row, "").value == 2 + assert COUNTIF(row, "<>").value == 4 + assert COUNTIF(row, 10).value == 2 + assert COUNTIF(row, "#N/A").value == 1 + assert COUNTIF(row, "<>10").value == 4 + assert COUNTIF(row, True).value == 1 + + def test_empty_text_typed_in_is_a_blank_cell_and_a_formula_s_is_not(self): + # Typed in, '' is written as nothing: a blank. A formula's '' is + # a cell the spreadsheet counts (and '' matches it too). + typed = mo.Variable(["", "x", None]) + assert (COUNTIF(typed, "").value, COUNTIF(typed, "<>").value) == (2, 1) + assert COUNTIF(typed, "*").value == 1 + formula = mo.IF(mo.Variable([True, False, True]), "", "x") + assert (COUNTIF(formula, "").value, COUNTIF(formula, "<>").value) == (2, 3) + assert COUNTIF(formula, "*").value == 3 + # One element of a row is the row's cell: a typed-in row's '' is blank. + assert COUNTIF([typed[0], typed[1]], "<>").value == 1 + assert COUNTIF([formula[0], formula[1]], "<>").value == 2 + + def test_none_is_the_number_zero_typed_or_in_a_blank_cell(self): + row = mo.Variable([0.0, None, 5.0, "", 0]) + assert COUNTIF(row, None).value == 2 + assert COUNTIF(row, mo.Variable(None)).value == 2 + assert COUNTIF(row, mo.Variable("")).value == 2 + assert SUMIF(row, None, mo.Variable([1, 2, 3, 4, 5])).value == 6 + assert COUNTIF(mo.Variable(["x", None, ""]), mo.Variable(None)).value == 0 + + def test_equals_alone_is_a_blank_cell_and_nothing_else(self): + # '' matches a blank cell and a formula's empty text; '=' only the + # blank; an ordering alone compares text with the empty text. + formula = mo.IF(mo.Variable([True, False, True]), "", "x") + typed = mo.Variable(["", "x", None]) + assert (COUNTIF(formula, "=").value, COUNTIF(formula, "").value) == (0, 2) + assert (COUNTIF(typed, "=").value, COUNTIF(typed, "").value) == (2, 2) + assert (COUNTIF(formula, ">=").value, COUNTIF(formula, ">").value) == (3, 1) + assert (COUNTIF(formula, "<=").value, COUNTIF(formula, "<").value) == (2, 0) + + def test_an_operator_joined_to_a_blank_cell_or_none_is_the_operator_alone(self): + row = mo.Variable(["a", None, 5.0]) + blank = mo.Variable(None) + assert COUNTIF(row, ("=", blank)).value == 1 + assert COUNTIF(row, ("<>", blank)).value == 2 + assert COUNTIF(row, (">=", blank)).value == 1 + assert COUNTIF(row, ("=", None)).value == 1 + + def test_numbers_are_equal_as_the_spreadsheet_compares_them(self): + share = mo.Variable([0.1, 0.2, 0.7]) + mo.Variable([0.2, 0.1, -0.4]) + assert share.value[0] != 0.3 and share.value[2] != 0.3 + assert COUNTIF(share, 0.3).value == 3 + assert COUNTIF(share, mo.Variable(0.3)).value == 3 + assert (COUNTIF(share, ">0.3").value, COUNTIF(share, ">=0.3").value) == (0, 3) + assert COUNTIF(share, "<>0.3").value == 0 + assert SUMIF(share, 0.3, mo.Variable([1.0, 2.0, 4.0])).value == 7.0 + + def test_numeric_text_meets_an_equality_and_never_an_ordering(self): + row = mo.Variable(["10", "3", 7.0, None, "abc", "-1"]) + amounts = mo.Variable([1.0, 2.0, 4.0, 8.0, 16.0, 32.0]) + assert (COUNTIF(row, ">5").value, COUNTIF(row, ">=0").value) == (1, 1) + assert COUNTIF(row, "<0").value == 0 + assert SUMIF(row, ">5", amounts).value == 4.0 + assert (COUNTIF(row, 10).value, COUNTIF(row, "10").value) == (1, 1) + assert COUNTIF(row, "<>10").value == 5 + + def test_the_text_true_meets_the_criteria_written_true(self): + row = mo.Variable(["TRUE", True, 1, "x"]) + assert (COUNTIF(row, "TRUE").value, COUNTIF(row, "<>TRUE").value) == (2, 2) + assert COUNTIF(row, True).value == 1 + assert COUNTIF(row, ("=", True)).value == 2 + + def test_a_criteria_cell_holding_an_error_is_the_result(self): + errs = mo.Variable([1.0, "#DIV/0!", 3.0, "#N/A"]) + flags = mo.Variable(["y", "y", "n", "y"]) + cell = mo.Variable("#DIV/0!") + assert COUNTIF(errs, cell).value == "#DIV/0!" + assert COUNTIF(errs, (">", cell)).value == "#DIV/0!" + assert SUMIF(flags, cell, errs).value == "#DIV/0!" + assert SUMIFS(errs, flags, "y", flags, mo.Variable("#N/A")).value == "#N/A" + assert COUNTIFS(flags, "y", errs, cell).value == "#DIV/0!" + assert AVERAGEIF(flags, cell).value == "#DIV/0!" + assert MAXIFS(errs, flags, mo.Variable("#N/A")).value == "#N/A" + # Typed into the call, an error's text is a criteria: it matches the error. + assert COUNTIF(errs, "#DIV/0!").value == 1 and COUNTIF(errs, "<>#N/A").value == 3 + # Text that only starts with '#' is text. + assert COUNTIF(mo.Variable(["#1", "#2"]), mo.Variable("#1")).value == 1 + + def test_the_count_is_an_int_without_a_unit(self): + m = _rows() + out = COUNTIF(m.d.amount, ">0") + assert out.value_type == "int" and out.var_type == "scalar" and out._unit is None + + +class TestCOUNTIFS: + def test_every_pair_must_hold(self): + m = _rows() + assert COUNTIFS(m.d.region, "North", m.d.year, 2026).value == 1 + assert COUNTIFS(m.d.region, "<>North", m.d.amount, ">100").value == 2 + assert COUNTIFS(m.d.region, "North").value == 2 + assert COUNTIFS(m.d.region, "East").value == 0 + + def test_the_pairs_are_checked(self): + m = _rows() + with pytest.raises(ValueError, match="in pairs"): + COUNTIFS(m.d.region) + with pytest.raises(ValueError, match="same number of cells"): + COUNTIFS(m.d.region, "North", mo.Variable([1]), 1) + assert COUNTIFS(m.d.region, "North", m.d.year, 2026)._unit is None + + +class TestAVERAGEIF: + def test_mean_of_the_picked_numbers(self): + m = _rows() + assert AVERAGEIF(m.d.region, "North", m.d.amount).value == 200.0 + assert AVERAGEIF(m.d.amount, ">100").value == 300.0 + assert AVERAGEIFS(m.d.amount, m.d.region, "<>North", m.d.year, 2027).value == 400.0 + + def test_blanks_and_text_are_not_counted_and_nothing_is_div0(self): + flags = mo.Variable(["y", "y", "y", "n"]) + assert AVERAGEIF(flags, "y", mo.Variable([10.0, None, "x", 99.0])).value == 10.0 + assert AVERAGEIF(flags, "z", mo.Variable([10.0, 20.0, 30.0, 40.0])).value == "#DIV/0!" + assert AVERAGEIFS(mo.Variable([None, None]), flags[0:2], "y").value == "#DIV/0!" + + def test_an_error_among_the_matched_cells_is_the_result(self): + flags = mo.Variable(["y", "n", "y"]) + assert AVERAGEIF(flags, "y", mo.Variable([1.0, "#DIV/0!", 3.0])).value == 2.0 + assert AVERAGEIF(flags, "y", mo.Variable([1.0, 2.0, "#NUM!"])).value == "#NUM!" + + def test_a_float_in_the_averaged_range_s_unit(self): + from modeleon.core.unit import unit_key + m = _rows() + out = AVERAGEIF(m.d.region, "North", m.d.units) + assert out.value == 2.0 and out.value_type == "float" + assert unit_key(out._unit) == unit_key(m.d.units._unit) + assert AVERAGEIFS(m.d.amount, m.d.units, ">1")._unit is not None + assert AVERAGEIFS(m.d.year, m.d.units, ">1")._unit is None + + +class TestMAXIFSAndMINIFS: + def test_the_largest_and_smallest_of_the_picked(self): + m = _rows() + assert MAXIFS(m.d.amount, m.d.region, "North").value == 300.0 + assert MINIFS(m.d.amount, m.d.region, "North").value == 100.0 + assert MAXIFS(m.d.amount, m.d.region, "<>North", m.d.year, 2026).value == 200.0 + assert MINIFS(m.d.units, m.d.year, 2027).value == 3 + + def test_negatives_rank_and_nothing_is_zero(self): + flags = mo.Variable(["y", "y", "n"]) + negatives = mo.Variable([-5.0, -3.0, 9.0]) + assert MAXIFS(negatives, flags, "y").value == -3.0 + assert MINIFS(negatives, flags, "y").value == -5.0 + assert MAXIFS(negatives, flags, "z").value == 0 + assert MINIFS(negatives, flags, "z").value == 0 + + def test_text_and_blanks_are_skipped_and_an_error_is_the_result(self): + flags = mo.Variable(["y", "y", "y", "n"]) + assert MAXIFS(mo.Variable(["x", None, 2.0, 9.0]), flags, "y").value == 2.0 + assert MAXIFS(mo.Variable([1.0, "#REF!", 2.0, 9.0]), flags, "y").value == "#REF!" + + def test_type_and_unit_follow_the_ranked_range(self): + from modeleon.core.unit import unit_key + m = _rows() + out = MAXIFS(m.d.units, m.d.region, "North") + assert out.value == 3 and out.value_type == "int" and out.var_type == "scalar" + assert unit_key(out._unit) == unit_key(m.d.units._unit) + # Over text nothing ranks: 0, a number. + nothing = MAXIFS(m.d.region, m.d.region, "N*") + assert nothing.value == 0 and nothing.value_type == "float" + with pytest.raises(ValueError, match="in pairs"): + MINIFS(m.d.units) + + +class TestDatesAreNumbers: + def _dates(self): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.d = mo.MultiVariable("D") + m.d.region = mo.Variable(["North", "South", "North", "West"], display_name="region") + m.d.closed = mo.Variable([dt.date(2026, 1, 31), dt.date(2026, 2, 28), + dt.date(2024, 2, 29), None], display_name="closed") + m.d.amount = mo.Variable([100.0, 200.0, 300.0, 400.0], display_name="amount") + return m + + def test_a_date_cell_meets_a_numeric_criteria_and_a_number_a_date_one(self): + m = self._dates() + assert COUNTIF(m.d.closed, ">45000").value == 3 + assert COUNTIF(m.d.closed, 46053).value == 1 + assert COUNTIF(m.d.closed, (">=", dt.date(2026, 1, 1))).value == 2 + assert COUNTIF(m.d.closed, dt.date(2024, 2, 29)).value == 1 + assert COUNTIF(mo.Variable([46053, 46053.5, "46053"]), dt.date(2026, 1, 31)).value == 2 + assert SUMIF(m.d.closed, (">", dt.date(2026, 1, 31)), m.d.amount).value == 200.0 + + def test_the_largest_and_the_smallest_date_come_back_as_dates(self): + m = self._dates() + latest = MAXIFS(m.d.closed, m.d.region, "North") + assert latest.value == dt.date(2026, 1, 31) and latest.value_type == "datetime" + assert MINIFS(m.d.closed, m.d.region, "North").value == dt.date(2024, 2, 29) + assert MAXIFS(m.d.closed, m.d.region, "West").value == 0 + assert MAXIFS(m.d.closed, m.d.region, "<>").value == dt.date(2026, 2, 28) + + def test_the_mean_of_dates_is_a_date(self): + m = self._dates() + mean = AVERAGEIF(m.d.region, "<>West", m.d.closed) + assert mean.value == dt.datetime(2025, 6, 20, 8) and mean.value_type == "datetime" + whole = AVERAGEIFS(m.d.closed, m.d.region, "<>North") + assert whole.value == dt.date(2026, 2, 28) and whole.value_type == "datetime" + assert AVERAGEIF(m.d.region, "West", m.d.closed).value == "#DIV/0!" + assert AVERAGEIF(m.d.region, "North", m.d.amount).value_type == "float" + + def test_a_sum_of_dates_is_a_number_of_days(self): + m = self._dates() + total = SUMIF(m.d.region, "North", m.d.closed) + assert total.value == 46053 + 45351 and total.value_type == "float" + assert SUMIFS(m.d.closed, m.d.region, "West").value == 0 + + def test_a_date_written_2026_01_31_is_that_date(self): + m = self._dates() + assert COUNTIF(m.d.closed, ">=2026-01-01").value == 2 + assert COUNTIF(m.d.closed, "2026-01-31").value == 1 + assert SUMIF(m.d.closed, "<2026-02-01", m.d.amount).value == 400.0 + noon = mo.Variable([dt.datetime(2026, 1, 31, 12), dt.date(2026, 1, 31)]) + assert COUNTIF(noon, "2026-01-31 12:00").value == 1 + assert COUNTIF(noon, ">2026-01-31").value == 1 + + def test_a_duration_is_its_days(self): + days = mo.Variable([dt.timedelta(days=1), dt.timedelta(hours=12), None]) + assert COUNTIF(days, ">=0.5").value == 2 + assert SUMIF(days, ">0").value == 1.5 + assert MAXIFS(days, days, ">0").value == dt.timedelta(days=1) + + +# ══════════════════════════════════════════════════════════════════════ +# The counts +# ══════════════════════════════════════════════════════════════════════ + + +class TestCOUNT: + def test_numbers_and_dates_in_cells_count_and_nothing_else_does(self): + row = mo.Variable([1, 2.5, None, "x", "3", True, "#N/A", dt.date(2026, 1, 1)]) + assert COUNT(row).value == 3 + + def test_values_typed_into_the_call_count_as_the_spreadsheet_counts_them(self): + # None typed in is written as 0, a number. + assert COUNT(1, 2.5, "3", True, "x", None).value == 5 + assert COUNT(dt.timedelta(days=1), mo.Variable(dt.timedelta(hours=1))).value == 2 + # Beside a series the call is one per period: the + # duration typed in counts in every period. + assert COUNT(dt.timedelta(days=1), + mo.Variable([dt.timedelta(hours=1), None])).value == [2, 1] + assert COUNT([1, 2, "3", True]).value == 2 + assert COUNT(mo.Variable(True), mo.Variable("3")).value == 0 + + def test_over_rows_cells_and_lists_of_lines(self): + m = _rows() + assert COUNT(m.d.amount).value == 4 + # Two rows are counted period by period, as a + b + # combines them; their cells over all periods are two counts. + assert COUNT(m.d.amount, m.d.region).value == [1, 1, 1, 1] + assert (COUNT(m.d.amount) + COUNT(m.d.region)).value == 4 + assert COUNT(m.i.floor, m.i.pick).value == 1 + assert COUNT([m.i.floor, m.i.pick, m.d.amount[0]]).value == 2 + + def test_an_int_without_a_unit_and_at_least_one_argument(self): + m = _rows() + out = COUNT(m.d.amount) + assert out.value_type == "int" and out.var_type == "scalar" and out._unit is None + with pytest.raises(ValueError, match="at least one argument"): + COUNT() + + +class TestCOUNTA: + def test_whatever_is_not_blank_counts(self): + row = mo.Variable([1, None, "x", "", True, "#N/A", 0]) + assert COUNTA(row).value == 5 + assert COUNTA(1, None, "", "x").value == 4 + assert COUNTA([mo.Variable(None), mo.Variable("")]).value == 0 + # A formula's empty text is a cell. + assert COUNTA(mo.IF(mo.Variable([True, False]), "", "x")).value == 2 + + def test_over_rows_and_cells(self): + m = _rows() + assert COUNTA(m.d.region).value == 4 + # Cells beside a row are counted with each period's cell. + assert COUNTA(m.i.floor, m.i.pick, m.d.amount).value == [3, 3, 3, 3] + assert (COUNTA(m.i.floor, m.i.pick) + COUNTA(m.d.amount)).value == 6 + out = COUNTA(m.d.region) + assert out.value_type == "int" and out.var_type == "scalar" and out._unit is None + with pytest.raises(ValueError, match="at least one argument"): + COUNTA() + + +# ══════════════════════════════════════════════════════════════════════ +# A list of scalar lines as a range +# ══════════════════════════════════════════════════════════════════════ + + +class Deal(mo.MultiVariableClass): + def compute(self, region, amount): + self.region = mo.Variable(region, display_name="Region") + self.amount = mo.Variable(amount, display_name="Amount", unit="USD") + self.flow = mo.Variable([amount * 0.1, amount * 0.2], display_name="Flow") + + +def _records(): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=2) + with m: + m.deals = mo.MultiVariable("Deals", excel_props={"orient": "records"}) + with m.deals as d: + d.a = Deal(region="North", amount=100.0, display_name="Alpha") + d.b = Deal(region="South", amount=200.0, display_name="Beta") + d.c = Deal(region="North", amount=300.0, display_name="Gamma") + return m + + +class TestAListOfLinesIsARange: + def test_the_values_reduce_as_a_row_does(self): + d = _records().deals + regions = [d.a.region, d.b.region, d.c.region] + amounts = [d.a.amount, d.b.amount, d.c.amount] + assert SUMIF(regions, "North", amounts).value == 400.0 + assert SUMIFS(amounts, regions, "North", amounts, ">100").value == 300.0 + assert COUNTIF(regions, "North").value == 2 + assert COUNTIFS(regions, "North", amounts, ">100").value == 1 + assert AVERAGEIF(regions, "North", amounts).value == 200.0 + assert AVERAGEIFS(amounts, regions, "North").value == 200.0 + assert MAXIFS(amounts, regions, "North").value == 300.0 + assert MINIFS(amounts, regions, "North").value == 100.0 + assert COUNT(amounts).value == 3 and COUNTA(regions).value == 3 + + def test_one_element_of_a_row_is_a_line_too(self): + d = _records().deals + first = [d.a.flow[0], d.b.flow[0], d.c.flow[0]] + out = SUMIFS(first, [d.a.region, d.b.region, d.c.region], "North") + assert out.value == pytest.approx(40.0) + assert out._source_code == "SUMIFS([a.flow[0], b.flow[0], c.flow[0]], [a.region, b.region, c.region], 'North')" + + def test_a_tuple_a_mixed_list_and_a_literal_list(self): + d = _records().deals + assert SUMIF((d.a.region, d.b.region), "North", (d.a.amount, d.b.amount)).value == 100.0 + assert SUMIF([d.a.region, "North"], "North", [d.a.amount, 7.0]).value == 107.0 + assert SUMIF([1, 2, 3], ">1").value == 5 + + def test_a_row_inside_the_list_and_an_empty_list_are_refused(self): + d = _records().deals + with pytest.raises(ValueError, match=r"a\.flow in the list is a series"): + SUMIF([d.a.flow, d.b.flow], ">1") + with pytest.raises(ValueError, match="at least one cell"): + SUMIF([], ">1") + + def test_the_unit_and_type_are_the_lines_shared_ones(self): + from modeleon.core.unit import unit_key + d = _records().deals + out = SUMIF([d.a.region, d.b.region], "North", [d.a.amount, d.b.amount]) + assert unit_key(out._unit) == unit_key(d.a.amount._unit) and out.value_type == "float" + # A line without a unit does not vote; a second unit is refused, + # as SUM refuses it. + assert SUMIF([d.a.region, d.b.region], "North", + [d.a.amount, mo.Variable(5.0)])._unit is not None + with pytest.raises(ValueError, match="mo.SUMIF takes its values in one unit"): + SUMIF([d.a.region, d.b.region], "North", + [d.a.amount, mo.Variable(5.0, unit="EUR")]) + + +# ══════════════════════════════════════════════════════════════════════ +# The AST, the formula and the registry +# ══════════════════════════════════════════════════════════════════════ + + +class TestTheAST: + def test_a_funccall_over_refs_literals_and_a_list(self): + m = _rows() + d = _records().deals + out = SUMIF(m.d.region, ">=10", m.d.amount) + assert isinstance(out._expr, FuncCall) and out._expr.func == "SUMIF" + assert [type(a) for a in out._expr.args] == [VarRef, Literal, VarRef] + assert out._expr.args[1].value == ">=10" + assert out.formula == "SUMIF(region, '>=10', amount)" + assert out._source_code == "SUMIF(region, '>=10', amount)" + listed = SUMIF([d.a.region, d.b.region], "North", [d.a.amount, d.b.amount]) + assert isinstance(listed._expr.args[0], ListExpr) + assert [type(i) for i in listed._expr.args[0].items] == [VarRef, VarRef] + assert listed._source_code == "SUMIF([a.region, b.region], 'North', [a.amount, b.amount])" + assert sorted(listed.dependencies) == sorted( + v.id for v in (d.a.region, d.b.region, d.a.amount, d.b.amount)) + + def test_a_criteria_cell_and_an_operator_joined_to_one(self): + m = _rows() + cell = SUMIF(m.d.region, m.i.pick, m.d.amount) + assert isinstance(cell._expr.args[1], VarRef) and cell._expr.args[1].var is m.i.pick + assert cell._source_code == "SUMIF(region, pick, amount)" + joined = SUMIF(m.d.amount, (">=", m.i.floor)) + node = joined._expr.args[1] + assert isinstance(node, FuncCall) and node.func == "CONCAT" + assert isinstance(node.args[0], Literal) and node.args[0].value == ">=" + assert isinstance(node.args[1], VarRef) and node.args[1].var is m.i.floor + assert joined._source_code == "SUMIF(amount, CONCAT('>=', floor))" + assert m.i.floor.id in joined.dependencies + plain = SUMIF(m.d.amount, (">=", 250)) + assert isinstance(plain._expr.args[1], Literal) and plain._expr.args[1].value == ">=250" + dated = COUNTIF(mo.Variable([dt.date(2026, 1, 1), dt.date(2026, 6, 1)]), + (">=", dt.date(2026, 3, 1))) + assert dated.value == 1 and isinstance(dated._expr.args[1], FuncCall) + + def test_a_pair_is_written_as_the_text_it_joins(self): + m = _rows() + alone = SUMIF(m.d.amount, ("=", None)) + assert isinstance(alone._expr.args[1], Literal) and alone._expr.args[1].value == "=" + flags = COUNTIF(mo.Variable([True, "TRUE", False]), ("=", True)) + assert flags._expr.args[1].value == "=TRUE" and flags.value == 2 + # A time of day cannot be written as DATE(...): the serial is. + noon = dt.datetime(2026, 1, 31, 12) + cells = mo.Variable([noon, dt.date(2026, 1, 31)]) + typed = COUNTIF(cells, noon) + assert typed._expr.args[1].value == 46053.5 and typed.value == 1 + later = COUNTIF(cells, (">=", noon)) + assert later._expr.args[1].value == ">=46053.5" and later.value == 1 + + def test_the_counts_keep_sum_s_source_shape(self): + m = _rows() + assert COUNT(m.d.amount, 5)._source_code == "COUNT(amount, 5)" + assert COUNTA(m.d.region).formula == "COUNTA(region)" + assert isinstance(COUNT([1, 2])._expr.args[0], ListExpr) + + def test_every_name_is_registered_as_a_range_function_and_written_as_given(self): + for name in ("SUMIF", "SUMIFS", "COUNTIF", "COUNTIFS", "AVERAGEIF", "AVERAGEIFS", + "MAXIFS", "MINIFS"): + assert spec(name).shape == "range" and range_positions(name) is None, name + assert is_range_taking(name) and is_passthrough(name), name + assert not admits_in_loop(name, 3), name + assert file_prefix("MAXIFS") == file_prefix("MINIFS") == "_xlfn." + assert file_prefix("SUMIFS") == file_prefix("COUNTIFS") == "" + assert spec("COUNTA").shape == "reduction" and is_range_taking("COUNTA") + assert spec("COUNT").shape == "reduction" + + def test_the_json_renderer_takes_the_list_argument(self): + d = _records().deals + out = SUMIF([d.a.region, d.b.region], "North", [d.a.amount, d.b.amount]) + from modeleon.compile.json import to_json + rendered = to_json(out._expr) + assert "SUMIF" in str(rendered) + assert {"kind": "list"}.items() <= rendered["args"][0].items() + assert len(rendered["args"][0]["items"]) == 2 diff --git a/tests/test_dates_extra.py b/tests/test_dates_extra.py new file mode 100644 index 0000000..bce21cd --- /dev/null +++ b/tests/test_dates_extra.py @@ -0,0 +1,647 @@ +# SPDX-License-Identifier: Apache-2.0 +"""``DATEDIF``, ``WEEKDAY``, ``WEEKNUM``, ``NETWORKDAYS`` and ``WORKDAY`` +compute what the spreadsheet computes, quirks included, with the +formula as the product. + +``DATEDIF`` keeps Excel's own arithmetic for the ``"MD"`` and ``"YD"`` +units — the ones Microsoft's help flags — so the workbook recalculates +to the engine's numbers: ``"MD"`` moves the start's day into the month +before the end's and lets it roll over as ``DATE()`` does (31 January +to 1 March is -1), ``"YD"`` moves the end's month and day into the +start's year, not the start's into the end's (3 January 2015 to 28 July +2016 is 206, not 207). Each function is one call per period through the +shared helper: a scalar date stands in every period, a series gives a +series, tracks lift, an error cell passes through; a blank or a value +that is not a date is refused by name at build time, as the other date +helpers refuse it. The holidays of ``NETWORKDAYS`` / ``WORKDAY`` are one +range every period reads, never a series. +""" + +from __future__ import annotations + +from datetime import date, datetime + +import pytest + +import modeleon as mo +from modeleon.core.expr import FuncCall, Literal, VarRef +from modeleon.functions.dates import DATEDIF, NETWORKDAYS, WEEKDAY, WEEKNUM, WORKDAY +from modeleon.functions.registry import ( + admits_in_loop, + excel_name, + file_prefix, + is_elementwise, + is_range_taking, + range_positions, + spec, +) + +# Microsoft's own DATEDIF examples. +S, E = date(2001, 6, 1), date(2002, 8, 15) + + +def V(value, **kwargs): + return mo.Variable(value, **kwargs) + + +# ══════════════════════════════════════════════════════════════════════ +# DATEDIF +# ══════════════════════════════════════════════════════════════════════ + + +class TestDatedif: + @pytest.mark.parametrize("unit, expected", [ + ("Y", 1), ("M", 14), ("D", 440), ("MD", 14), ("YM", 2), ("YD", 75), + ]) + def test_the_documented_examples(self, unit, expected): + assert DATEDIF(S, E, unit).value == expected + assert DATEDIF(V(S), V(E), unit).value == expected + + def test_whole_years(self): + assert DATEDIF(date(2001, 1, 1), date(2003, 1, 1), "Y").value == 2 + assert DATEDIF(date(2001, 1, 2), date(2003, 1, 1), "Y").value == 1 + # 29 February's anniversary is 1 March: a year is not complete on the 28th. + assert DATEDIF(date(2024, 2, 29), date(2025, 2, 28), "Y").value == 0 + assert DATEDIF(date(2024, 2, 29), date(2025, 3, 1), "Y").value == 1 + + def test_whole_months_need_the_day_reached(self): + # 31 January to 28 February is no whole month to the spreadsheet. + assert DATEDIF(date(2025, 1, 31), date(2025, 2, 28), "M").value == 0 + assert DATEDIF(date(2025, 3, 31), date(2025, 4, 30), "M").value == 0 + assert DATEDIF(date(2025, 1, 31), date(2025, 4, 30), "M").value == 2 + assert DATEDIF(date(2025, 1, 15), date(2025, 2, 15), "M").value == 1 + + def test_months_past_the_years(self): + assert DATEDIF(date(2001, 6, 15), date(2002, 6, 10), "YM").value == 11 + assert DATEDIF(date(2001, 6, 15), date(2002, 6, 15), "YM").value == 0 + assert DATEDIF(date(2001, 1, 15), date(2001, 12, 20), "YM").value == 11 + + @pytest.mark.parametrize("start, end, expected", [ + # The known quirk: the start's day lands in a February that is + # too short and rolls over into March, past the end. + (date(2016, 1, 31), date(2016, 3, 1), -1), + (date(2025, 1, 31), date(2025, 3, 1), -2), + (date(2025, 1, 29), date(2025, 3, 1), 0), + (date(2012, 1, 28), date(2012, 3, 1), 2), + (date(2011, 1, 29), date(2011, 3, 1), 0), + # The month before January is the December before. + (date(2023, 12, 31), date(2024, 1, 15), 15), + (date(2023, 11, 30), date(2024, 1, 15), 16), + # The end's day reached: a plain difference. + (date(2024, 1, 15), date(2024, 5, 20), 5), + (date(2024, 1, 15), date(2024, 5, 15), 0), + ]) + def test_days_past_the_months_quirks_included(self, start, end, expected): + assert DATEDIF(start, end, "MD").value == expected + + @pytest.mark.parametrize("start, end, expected", [ + # The end's month and day in the start's year: February 2015 has + # 28 days, so 206 where a count near the end date gives 207. + (date(2015, 1, 3), date(2016, 7, 28), 206), + # The same rule across 23 years: 323, where the count near the + # end date crosses 29 February 2012 and gives 324. + (date(1988, 6, 22), date(2012, 5, 11), 323), + # The end's day before the start's in the year: the year after. + (date(2010, 10, 11), date(2012, 3, 1), 141), + (date(2010, 10, 11), date(2011, 3, 1), 141), + # A whole number of years is 0 days, leap years included. + (date(2003, 3, 1), date(2004, 3, 1), 0), + (date(2004, 1, 1), date(2005, 1, 1), 0), + (date(2004, 3, 1), date(2005, 3, 1), 0), + (date(2020, 2, 29), date(2021, 2, 28), 365), + # 29 February placed in a common year rolls over to 1 March. + (date(2015, 1, 3), date(2016, 2, 29), 57), + # A 29 February at either end lies outside the values checked in Excel: + # the rule applied as written (LibreOffice gives 0 and 2 here). + (date(2024, 2, 29), date(2025, 3, 1), 1), + (date(2015, 2, 28), date(2016, 3, 1), 1), + ]) + def test_days_past_the_years_quirks_included(self, start, end, expected): + assert DATEDIF(start, end, "YD").value == expected + + @pytest.mark.parametrize("unit", ["Y", "M", "D", "MD", "YM", "YD"]) + def test_equal_dates_are_zero(self, unit): + assert DATEDIF(S, S, unit).value == 0 + + def test_the_unit_is_read_in_any_case_and_written_as_given(self): + assert DATEDIF(S, E, "md").value == 14 + assert DATEDIF(S, E, "yd").value == 75 + start, end = V(S, display_name="start"), V(E, display_name="end") + assert DATEDIF(start, end, "md").formula == "DATEDIF(start, end, 'md')" + assert DATEDIF(start, end, V("YM", display_name="unit")).value == 2 + + def test_a_start_after_the_end_is_the_period_s_num(self): + assert DATEDIF(E, S, "D").value == "#NUM!" + ends = V([date(2001, 5, 1), date(2001, 7, 1), date(2002, 8, 15)]) + assert DATEDIF(S, ends, "M").value == ["#NUM!", 1, 14] + guarded = mo.IF(ends >= S, DATEDIF(S, ends, "M"), 0) + assert guarded.value == [0, 1, 14] + + @pytest.mark.parametrize("unit", ["X", "DM", "", 3, None]) + def test_a_unit_the_spreadsheet_lacks_is_refused(self, unit): + with pytest.raises((ValueError, TypeError), match="DATEDIF"): + DATEDIF(S, E, unit) + + def test_element_wise_over_series(self): + ends = V([date(2024, 1, 31), date(2024, 2, 29), date(2024, 3, 31)]) + starts = V([date(2024, 1, 1), date(2024, 1, 1), date(2024, 2, 1)]) + start = V(date(2024, 1, 1)) + assert DATEDIF(start, ends, "D").value == [30, 59, 90] + assert DATEDIF(starts, ends, "D").value == [30, 59, 59] + assert DATEDIF(date(2024, 1, 1), ends, "D").var_type == 'list' + assert DATEDIF(V([date(2024, 1, 1)]), ends, "D").value == [30, 59, 90] + assert DATEDIF(starts, ends, ["D", "M", "MD"]).value == [30, 1, 30] + + def test_series_of_different_lengths_are_refused_as_the_operators_refuse_them(self): + with pytest.raises(ValueError, match="different lengths"): + DATEDIF(V([S, S, S]), V([E, E]), "D") + + def test_an_error_cell_is_the_period_s_result(self): + ends = V([E, "#N/A", "#DIV/0!"]) + assert DATEDIF(S, ends, "D").value == [440, "#N/A", "#DIV/0!"] + assert DATEDIF(V("#N/A"), E, "D").value == "#N/A" + + def test_a_blank_is_refused_by_name_with_its_period(self): + with pytest.raises(TypeError, match="DATEDIF needs a date: 'leave' is blank"): + DATEDIF(S, V(None, display_name="leave"), "M") + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=2) + with m: + m.leave = mo.Variable([date(2026, 1, 31), None], display_name="leave") + with pytest.raises(TypeError, match="'leave' is blank in Feb 2026"): + DATEDIF(date(2026, 1, 1), m.leave, "D") + + def test_a_value_that_is_not_a_date_is_refused_at_build_time(self): + with pytest.raises(TypeError, match="Expected a date"): + DATEDIF(45000, E, "D") + with pytest.raises(ValueError): + DATEDIF("not a date", E, "D") + + def test_a_datetime_counts_by_its_date_and_an_iso_string_is_a_date(self): + assert DATEDIF(datetime(2001, 6, 1, 23, 30), datetime(2002, 8, 15, 1), "D").value == 440 + assert DATEDIF("2001-06-01", "2002-08-15", "YD").value == 75 + + def test_tracked_operands_lift_per_track(self): + m = mo.Model("m", tracks=mo.Tracks("plan", "actual"), + default_grain="month", default_start="2026-01", default_periods=2) + with m: + m.paid = mo.Variable(plan=[date(2026, 1, 31), date(2026, 2, 28)], + actual=[date(2026, 2, 3), date(2026, 3, 2)]) + m.late = DATEDIF(date(2026, 1, 1), m.paid, "D") + assert dict(m.late._value.items()) == {"plan": [30, 58], "actual": [33, 60]} + assert m.late.var_type == 'list' + + def test_a_whole_number_without_a_unit_and_the_ast_is_a_call(self): + m = mo.Model("m") + start = m.start = V(S) + end = m.end = V(E, unit="$") + out = DATEDIF(start, end, "M") + assert out.value_type == 'int' and out.var_type == 'scalar' + assert out._unit is None + assert out.formula == "DATEDIF(start, end, 'M')" + assert out.source_code == "DATEDIF(start, end, 'M')" + expr = out._expr + assert isinstance(expr, FuncCall) and expr.func == "DATEDIF" + assert isinstance(expr.args[0], VarRef) and expr.args[0].var is start + assert isinstance(expr.args[1], VarRef) and expr.args[1].var is end + assert isinstance(expr.args[2], Literal) and expr.args[2].value == "M" + assert expr.render_backends == frozenset({'excel'}) + assert out.dependencies == {start.id, end.id} + + def test_an_axised_operand_is_refused(self): + region = mo.Variable(["N", "S"], display_name="region") + by_region = mo.Variable([S, E], indexed_by=[region]) + with pytest.raises(ValueError, match="DATEDIF"): + DATEDIF(by_region, E, "D") + + +# ══════════════════════════════════════════════════════════════════════ +# WEEKDAY and WEEKNUM +# ══════════════════════════════════════════════════════════════════════ + +# Monday 1 January 2024 to Sunday 7 January 2024. +WEEK = [date(2024, 1, d) for d in range(1, 8)] + + +class TestWeekday: + @pytest.mark.parametrize("return_type, expected", [ + (1, [2, 3, 4, 5, 6, 7, 1]), + (2, [1, 2, 3, 4, 5, 6, 7]), + (3, [0, 1, 2, 3, 4, 5, 6]), + (11, [1, 2, 3, 4, 5, 6, 7]), + (12, [7, 1, 2, 3, 4, 5, 6]), + (13, [6, 7, 1, 2, 3, 4, 5]), + (14, [5, 6, 7, 1, 2, 3, 4]), + (15, [4, 5, 6, 7, 1, 2, 3]), + (16, [3, 4, 5, 6, 7, 1, 2]), + (17, [2, 3, 4, 5, 6, 7, 1]), + ]) + def test_every_return_type(self, return_type, expected): + assert WEEKDAY(V(WEEK), return_type).value == expected + assert [WEEKDAY(d, return_type).value for d in WEEK] == expected + + def test_sunday_is_one_when_no_type_is_given(self): + assert WEEKDAY(date(2024, 1, 7)).value == 1 + assert WEEKDAY(V(WEEK)).value == [2, 3, 4, 5, 6, 7, 1] + assert WEEKDAY(V(WEEK), 1.9).value == [2, 3, 4, 5, 6, 7, 1] + + def test_the_type_is_written_only_when_given(self): + d = V(date(2024, 1, 7), display_name="d") + assert WEEKDAY(d).formula == "WEEKDAY(d)" + assert WEEKDAY(d, 2).formula == "WEEKDAY(d, 2)" + assert len(WEEKDAY(d)._expr.args) == 1 + + @pytest.mark.parametrize("return_type", [0, 4, 10, 18, 21, -1]) + def test_a_type_the_spreadsheet_lacks_is_num(self, return_type): + with pytest.raises(ValueError, match="WEEKDAY.*#NUM!"): + WEEKDAY(date(2024, 1, 7), return_type) + + def test_a_type_that_is_not_a_number_is_value(self): + with pytest.raises(TypeError, match="WEEKDAY.*#VALUE!"): + WEEKDAY(date(2024, 1, 7), "two") + with pytest.raises(TypeError, match="WEEKDAY.*#VALUE!"): + WEEKDAY(date(2024, 1, 7), True) + + def test_a_datetime_counts_by_its_date(self): + assert WEEKDAY(datetime(2024, 1, 7, 23, 59)).value == 1 + + def test_a_blank_and_an_error_cell(self): + with pytest.raises(TypeError, match="WEEKDAY needs a date"): + WEEKDAY(V(None)) + assert WEEKDAY(V([date(2024, 1, 7), "#N/A"])).value == [1, "#N/A"] + + def test_a_whole_number_without_a_unit(self): + out = WEEKDAY(V(WEEK, unit="$")) + assert out.value_type == 'int' and out.var_type == 'list' and out._unit is None + + +class TestWeeknum: + def test_week_one_holds_the_first_of_january(self): + # 1 January 2024 is a Monday: Sunday the 7th begins week 2 of a + # Sunday-started count and ends week 1 of a Monday-started one. + assert WEEKNUM(date(2024, 1, 1)).value == 1 + assert WEEKNUM(date(2024, 1, 6)).value == 1 + assert WEEKNUM(date(2024, 1, 7)).value == 2 + assert WEEKNUM(date(2024, 1, 7), 1).value == 2 + assert WEEKNUM(date(2024, 1, 7), 2).value == 1 + assert WEEKNUM(date(2024, 1, 8), 2).value == 2 + # 1 January 2021 is a Friday: Sunday the 3rd begins week 2. + assert WEEKNUM(date(2021, 1, 1)).value == 1 + assert WEEKNUM(date(2021, 1, 2)).value == 1 + assert WEEKNUM(date(2021, 1, 3)).value == 2 + assert WEEKNUM(date(2021, 1, 3), 2).value == 1 + assert WEEKNUM(date(2021, 1, 4), 2).value == 2 + + def test_the_last_days_of_a_year_reach_week_53_or_54(self): + # 365 days are 52 weeks and a day: 31 December is always past + # week 52, and a leap year that opens on the last day of a week + # reaches 54 (1 January 2000 was a Saturday). + assert WEEKNUM(date(2024, 12, 29)).value == 53 + assert WEEKNUM(date(2024, 12, 31)).value == 53 + assert WEEKNUM(date(2024, 12, 31), 2).value == 53 + assert WEEKNUM(date(2023, 12, 31)).value == 53 + assert WEEKNUM(date(2023, 12, 31), 2).value == 53 + assert WEEKNUM(date(2023, 12, 24), 2).value == 52 + assert WEEKNUM(date(2000, 12, 31)).value == 54 + assert WEEKNUM(date(2000, 12, 31), 2).value == 53 + + def test_the_last_days_of_a_year_in_every_mode(self): + # 2021 opened and closed on a Friday: whichever day a week begins + # on, 31 December lies past 52 whole weeks from the week of 1 + # January. LibreOffice numbers these days 1 (see test_excel_dates). + for return_type in (1, 2, 11, 12, 13, 14, 15, 16, 17): + assert WEEKNUM(date(2021, 12, 31), return_type).value == 53, return_type + assert WEEKNUM(date(2021, 12, 31), 21).value == 52 + # 2028 is a leap year opening on a Saturday: 54 for a Sunday-started + # count, 53 when the week begins on Saturday or Monday. + assert WEEKNUM(date(2028, 12, 31), 1).value == 54 + assert WEEKNUM(date(2028, 12, 31), 17).value == 54 + assert WEEKNUM(date(2028, 12, 31), 16).value == 53 + assert WEEKNUM(date(2028, 12, 31), 2).value == 53 + assert WEEKNUM(date(2028, 12, 31), 21).value == 52 + # A Thursday-started week 53 of 2024 opens on Thursday 26 December. + assert WEEKNUM(date(2024, 12, 25), 14).value == 52 + assert WEEKNUM(date(2024, 12, 26), 14).value == 53 + + @pytest.mark.parametrize("return_type, expected", [ + (11, [1, 1, 1, 1, 1, 1, 1]), + (12, [1, 2, 2, 2, 2, 2, 2]), + (13, [1, 1, 2, 2, 2, 2, 2]), + (14, [1, 1, 1, 2, 2, 2, 2]), + (15, [1, 1, 1, 1, 2, 2, 2]), + (16, [1, 1, 1, 1, 1, 2, 2]), + (17, [1, 1, 1, 1, 1, 1, 2]), + ]) + def test_a_week_that_begins_on_any_day(self, return_type, expected): + assert WEEKNUM(V(WEEK), return_type).value == expected + + def test_type_21_is_the_iso_week(self): + assert WEEKNUM(date(2021, 1, 1), 21).value == 53 + assert WEEKNUM(date(2021, 1, 4), 21).value == 1 + assert WEEKNUM(date(2024, 12, 30), 21).value == 1 + assert WEEKNUM(date(2024, 12, 29), 21).value == 52 + assert WEEKNUM(V(WEEK), 21).value == [1, 1, 1, 1, 1, 1, 1] + + def test_the_type_is_written_only_when_given(self): + d = V(date(2024, 1, 7), display_name="d") + assert WEEKNUM(d).formula == "WEEKNUM(d)" + assert WEEKNUM(d, 21).formula == "WEEKNUM(d, 21)" + + @pytest.mark.parametrize("return_type", [0, 3, 10, 18, 20, 22, 150]) + def test_a_type_the_spreadsheet_lacks_is_num(self, return_type): + with pytest.raises(ValueError, match="WEEKNUM.*#NUM!"): + WEEKNUM(date(2024, 1, 7), return_type) + + def test_a_blank_an_error_cell_and_the_result_s_type(self): + with pytest.raises(TypeError, match="WEEKNUM needs a date"): + WEEKNUM(V([date(2024, 1, 7), None])) + out = WEEKNUM(V([date(2024, 1, 7), "#VALUE!"], unit="$")) + assert out.value == [2, "#VALUE!"] + assert out.value_type == 'int' and out._unit is None + + +# ══════════════════════════════════════════════════════════════════════ +# NETWORKDAYS and WORKDAY +# ══════════════════════════════════════════════════════════════════════ + +# Microsoft's own examples. +NW_START, NW_END = date(2012, 10, 1), date(2013, 3, 1) +NW_HOLIDAYS = [date(2012, 11, 22), date(2012, 12, 4), date(2013, 1, 21)] +WD_START, WD_DAYS = date(2008, 10, 1), 151 +WD_HOLIDAYS = [date(2008, 11, 26), date(2008, 12, 4), date(2009, 1, 21)] + + +class TestNetworkdays: + def test_the_documented_examples(self): + assert NETWORKDAYS(NW_START, NW_END).value == 110 + assert NETWORKDAYS(NW_START, NW_END, NW_HOLIDAYS[0]).value == 109 + assert NETWORKDAYS(NW_START, NW_END, NW_HOLIDAYS).value == 107 + assert NETWORKDAYS(V(NW_START), V(NW_END), V(NW_HOLIDAYS)).value == 107 + + def test_both_ends_count_and_weekends_do_not(self): + jan = (date(2024, 1, 1), date(2024, 1, 31)) # Monday to Wednesday + assert NETWORKDAYS(*jan).value == 23 + assert NETWORKDAYS(date(2024, 1, 1), date(2024, 1, 1)).value == 1 + assert NETWORKDAYS(date(2024, 1, 6), date(2024, 1, 6)).value == 0 + assert NETWORKDAYS(date(2024, 1, 6), date(2024, 1, 7)).value == 0 + assert NETWORKDAYS(date(2024, 1, 5), date(2024, 1, 8)).value == 2 + + def test_a_start_after_the_end_counts_negative(self): + assert NETWORKDAYS(date(2024, 1, 31), date(2024, 1, 1)).value == -23 + assert NETWORKDAYS(NW_END, NW_START, NW_HOLIDAYS).value == -107 + + def test_holidays_count_once_on_weekdays_inside_the_span(self): + jan = (date(2024, 1, 1), date(2024, 1, 31)) + assert NETWORKDAYS(*jan, [date(2024, 1, 1), date(2024, 1, 1)]).value == 22 + assert NETWORKDAYS(*jan, [date(2024, 1, 6), date(2024, 1, 7)]).value == 23 + assert NETWORKDAYS(*jan, [date(2023, 12, 29), date(2024, 2, 1)]).value == 23 + assert NETWORKDAYS(*jan, [date(2024, 1, 1), date(2024, 1, 31)]).value == 21 + assert NETWORKDAYS(*jan, ()).value == 23 + assert NETWORKDAYS(*jan, []).value == 23 + + def test_holidays_may_be_blank_dated_text_or_a_line_of_dates(self): + jan = (date(2024, 1, 1), date(2024, 1, 31)) + assert NETWORKDAYS(*jan, [date(2024, 1, 1), None]).value == 22 + assert NETWORKDAYS(*jan, ["2024-01-01", datetime(2024, 1, 2, 9)]).value == 21 + assert NETWORKDAYS(*jan, V([date(2024, 1, 1), None, date(2024, 1, 2)])).value == 21 + assert NETWORKDAYS(*jan, V(date(2024, 1, 1))).value == 22 + + def test_a_zero_among_the_holidays_is_skipped_as_the_spreadsheet_skips_it(self): + jan = (date(2024, 1, 1), date(2024, 1, 31)) + assert NETWORKDAYS(*jan, [date(2024, 1, 1), 0, 0.0]).value == 22 + assert WORKDAY(jan[0], 1, [0, date(2024, 1, 2)]).value == date(2024, 1, 3) + # Holidays in a longer window: a line extended with mo.zero() + # fills the rest of the window with 0.0, which Excel reads as a + # serial before every date. + m = mo.Model("m", default_grain="month", default_start="2024-01", default_periods=7) + with m: + m.hol = mo.Variable([date(2024, 1, 1), date(2024, 1, 2)], extend=mo.zero()) + assert m.hol.value[2:] == [0.0] * 5 + assert NETWORKDAYS(*jan, m.hol).value == 21 + + def test_a_shorter_holidays_line_keeps_its_own_length_with_extend_none(self): + m = mo.Model("m", default_grain="month", default_start="2024-01", default_periods=7) + with m: + m.hol = mo.Variable([date(2024, 1, 1), date(2024, 1, 2)], extend=mo.none()) + m.n = NETWORKDAYS(date(2024, 1, 1), date(2024, 1, 31), m.hol) + assert m.hol.value == [date(2024, 1, 1), date(2024, 1, 2)] + assert m.n.value == 21 + assert isinstance(m.n._expr.args[2], VarRef) and m.n.dependencies == {m.hol.id} + + def test_a_non_date_among_the_holidays_is_refused_by_name(self): + jan = (date(2024, 1, 1), date(2024, 1, 31)) + m = mo.Model("m", default_grain="month", default_start="2024-01", default_periods=3) + with m: + m.hol = mo.Variable([date(2024, 1, 1), 45000, None], display_name="hol") + with pytest.raises(TypeError, match="NETWORKDAYS holidays must be dates: 'hol' holds " + "45000 in Feb 2024"): + NETWORKDAYS(*jan, m.hol) + with pytest.raises(TypeError, match="the list holds 'x' at position 2"): + NETWORKDAYS(*jan, [date(2024, 1, 1), "x"]) + with pytest.raises(TypeError, match="WORKDAY holidays must be dates: the holidays holds " + "45000"): + WORKDAY(jan[0], 1, 45000) + with pytest.raises(TypeError, match="holds True"): + NETWORKDAYS(*jan, [True]) + + def test_a_list_of_lines_is_refused_with_a_pointer_to_one_line(self): + a, b = V(date(2024, 1, 1)), V(date(2024, 1, 2)) + with pytest.raises(TypeError, match="a list of lines is not a range"): + NETWORKDAYS(date(2024, 1, 1), date(2024, 1, 31), [a, b]) + with pytest.raises(TypeError, match="a list of lines is not a range"): + WORKDAY(date(2024, 1, 1), 1, (a,)) + + def test_an_error_among_the_holidays_is_the_result(self): + jan = (date(2024, 1, 1), date(2024, 1, 31)) + assert NETWORKDAYS(*jan, V([date(2024, 1, 1), "#N/A"])).value == "#N/A" + assert NETWORKDAYS(V([jan[0], jan[0]]), jan[1], ["#REF!"]).value == ["#REF!", "#REF!"] + + def test_element_wise_over_the_dates_with_one_set_of_holidays(self): + ends = V([date(2024, 1, 31), date(2024, 2, 29), date(2024, 3, 29)]) + hol = V([date(2024, 1, 1), date(2024, 2, 5), date(2024, 3, 15)]) + out = NETWORKDAYS(date(2024, 1, 1), ends, hol) + assert out.value == [22, 42, 62] + assert out.var_type == 'list' + starts = V([date(2024, 1, 1), date(2024, 2, 1), date(2024, 3, 1)]) + assert NETWORKDAYS(starts, ends, hol).value == [22, 20, 20] + assert NETWORKDAYS(ends, date(2024, 1, 1)).value == [-23, -44, -65] + assert NETWORKDAYS(date(2024, 1, 1), V([date(2024, 1, 31), "#N/A"])).value == [23, "#N/A"] + + def test_blanks_and_non_dates_in_the_span_are_refused_by_name(self): + with pytest.raises(TypeError, match="NETWORKDAYS needs a date: 'end' is blank"): + NETWORKDAYS(date(2024, 1, 1), V(None, display_name="end")) + with pytest.raises(TypeError, match="Expected a date"): + NETWORKDAYS(date(2024, 1, 1), 45000) + + def test_tracked_dates_lift_per_track(self): + m = mo.Model("m", tracks=mo.Tracks("plan", "actual"), + default_grain="month", default_start="2024-01", default_periods=2) + with m: + m.paid = mo.Variable(plan=[date(2024, 1, 31), date(2024, 2, 29)], + actual=[date(2024, 2, 2), date(2024, 3, 1)]) + m.hol = mo.Variable([date(2024, 1, 1)]) + m.days = NETWORKDAYS(date(2024, 1, 1), m.paid, m.hol) + assert dict(m.days._value.items()) == {"plan": [22, 43], "actual": [24, 44]} + + def test_the_holidays_are_one_operand_of_the_call_not_a_series(self): + m = mo.Model("m") + start = m.start = V(date(2024, 1, 1)) + ends = m.ends = V([date(2024, 1, 31), date(2024, 2, 29)], unit="$") + hol = m.hol = V([date(2024, 1, 1), date(2024, 2, 5)]) + out = NETWORKDAYS(start, ends, hol) + assert out.formula == "NETWORKDAYS(start, ends, hol)" + assert out.source_code == "NETWORKDAYS(start, ends, hol)" + expr = out._expr + assert isinstance(expr, FuncCall) and expr.func == "NETWORKDAYS" + assert [type(a) for a in expr.args] == [VarRef, VarRef, VarRef] + assert expr.args[2].var is hol + assert expr.render_backends == frozenset({'excel'}) + assert out.dependencies == {start.id, ends.id, hol.id} + assert out.value_type == 'int' and out._unit is None + bare = NETWORKDAYS(start, ends) + assert bare.formula == "NETWORKDAYS(start, ends)" and len(bare._expr.args) == 2 + literal = NETWORKDAYS(start, ends, [date(2024, 1, 1)]) + assert isinstance(literal._expr.args[2], Literal) + assert literal._expr.args[2].value == [date(2024, 1, 1)] + assert literal.dependencies == {start.id, ends.id} + + def test_an_axised_operand_is_refused(self): + region = mo.Variable(["N", "S"], display_name="region") + by_region = mo.Variable([date(2024, 1, 1), date(2024, 2, 1)], indexed_by=[region]) + with pytest.raises(ValueError, match="NETWORKDAYS"): + NETWORKDAYS(by_region, date(2024, 3, 1)) + with pytest.raises(ValueError, match="NETWORKDAYS"): + NETWORKDAYS(date(2024, 1, 1), date(2024, 3, 1), by_region) + + +class TestWorkday: + def test_the_documented_examples(self): + assert WORKDAY(WD_START, WD_DAYS).value == date(2009, 4, 30) + assert WORKDAY(WD_START, WD_DAYS, WD_HOLIDAYS).value == date(2009, 5, 5) + assert WORKDAY(V(WD_START), V(WD_DAYS), V(WD_HOLIDAYS)).value == date(2009, 5, 5) + + def test_steps_over_weekends_and_holidays_in_either_direction(self): + fri = date(2024, 1, 5) + assert WORKDAY(fri, 1).value == date(2024, 1, 8) + assert WORKDAY(fri, 1, [date(2024, 1, 8)]).value == date(2024, 1, 9) + assert WORKDAY(fri, 5).value == date(2024, 1, 12) + assert WORKDAY(date(2024, 1, 8), -1).value == date(2024, 1, 5) + assert WORKDAY(date(2024, 1, 8), -1, [date(2024, 1, 5)]).value == date(2024, 1, 4) + assert WORKDAY(date(2024, 1, 6), 1).value == date(2024, 1, 8) # from a Saturday + assert WORKDAY(date(2024, 1, 7), -1).value == date(2024, 1, 5) # from a Sunday + assert WORKDAY(fri, 1, [date(2024, 1, 6), date(2024, 1, 7)]).value == date(2024, 1, 8) + + def test_zero_days_is_the_start_itself_even_on_a_weekend(self): + assert WORKDAY(date(2024, 1, 6), 0).value == date(2024, 1, 6) + assert WORKDAY(date(2024, 1, 6), 0, [date(2024, 1, 6)]).value == date(2024, 1, 6) + + def test_days_are_truncated_toward_zero(self): + fri = date(2024, 1, 5) + assert WORKDAY(fri, 1.9).value == date(2024, 1, 8) + assert WORKDAY(fri, -1.9).value == date(2024, 1, 4) + assert WORKDAY(fri, 0.5).value == fri + with pytest.raises(TypeError, match="WORKDAY.*#VALUE!"): + WORKDAY(fri, "ten") + with pytest.raises(TypeError, match="WORKDAY needs a number"): + WORKDAY(fri, V(None)) + + def test_the_result_is_a_date(self): + out = WORKDAY(V(date(2024, 1, 5), display_name="start"), V(3, unit="day")) + assert out.value == date(2024, 1, 10) and type(out.value) is date + assert out.value_type == 'datetime' and out._unit is None + assert WORKDAY(datetime(2024, 1, 5, 15), 1).value == date(2024, 1, 8) + + def test_element_wise_over_the_start_or_the_days(self): + starts = V([date(2024, 1, 5), date(2024, 1, 8), date(2024, 1, 9)]) + assert WORKDAY(starts, 1).value == [date(2024, 1, 8), date(2024, 1, 9), date(2024, 1, 10)] + assert WORKDAY(date(2024, 1, 5), [1, 2, 3]).value == \ + [date(2024, 1, 8), date(2024, 1, 9), date(2024, 1, 10)] + assert WORKDAY(starts, V([1, 2, 3]), [date(2024, 1, 9)]).value == \ + [date(2024, 1, 8), date(2024, 1, 11), date(2024, 1, 12)] + assert WORKDAY(V([date(2024, 1, 5), "#N/A"]), 1).value == [date(2024, 1, 8), "#N/A"] + assert WORKDAY(date(2024, 1, 5), V([1, "#DIV/0!"])).value == [date(2024, 1, 8), "#DIV/0!"] + + def test_the_holidays_are_one_operand_of_the_call_not_a_series(self): + start = V(date(2024, 1, 5), display_name="start") + hol = V([date(2024, 1, 8), date(2024, 1, 9)], display_name="hol") + out = WORKDAY(start, 1, hol) + assert out.value == date(2024, 1, 10) + assert out.formula == "WORKDAY(start, 1, hol)" + expr = out._expr + assert isinstance(expr, FuncCall) and expr.func == "WORKDAY" + assert isinstance(expr.args[2], VarRef) and expr.args[2].var is hol + assert out.dependencies == {start.id, hol.id} + assert WORKDAY(start, 1).formula == "WORKDAY(start, 1)" + + +# ══════════════════════════════════════════════════════════════════════ +# The registry and a loop container +# ══════════════════════════════════════════════════════════════════════ + + +class TestTheRegistry: + @pytest.mark.parametrize("name", ["DATEDIF", "WEEKDAY", "WEEKNUM"]) + def test_one_call_per_period_under_a_bare_classic_name(self, name): + assert is_elementwise(name) and not is_range_taking(name) + assert spec(name).passthrough and not spec(name).bare + assert file_prefix(name) == "" and excel_name(name) == name + assert admits_in_loop(name, 1) and admits_in_loop(name, 3) + + @pytest.mark.parametrize("name", ["NETWORKDAYS", "WORKDAY"]) + def test_the_holidays_are_the_range_position(self, name): + assert spec(name).shape == "range" and is_range_taking(name) + assert range_positions(name) == (2,) + assert not is_elementwise(name) and not admits_in_loop(name, 3) + assert spec(name).passthrough and not spec(name).bare + assert file_prefix(name) == "" and excel_name(name) == name + + +class TestOtherRenderers: + """Excel-native calls: the JSON renderer keeps the structure and the + backend hint, every other renderer inlines the value.""" + + @pytest.mark.parametrize("build, name", [ + (lambda d: DATEDIF(d, date(2024, 3, 1), "D"), "DATEDIF"), + (lambda d: WEEKDAY(d), "WEEKDAY"), + (lambda d: WEEKNUM(d), "WEEKNUM"), + (lambda d: NETWORKDAYS(d, date(2024, 3, 1), [date(2024, 1, 1)]), "NETWORKDAYS"), + (lambda d: WORKDAY(d, 5), "WORKDAY"), + ]) + def test_the_json_renderer_keeps_the_call_and_its_hint(self, build, name): + out = mo.to_json(build(mo.Variable(date(2024, 1, 15)))._expr) + assert out["kind"] == "funccall" and out["func"] == name + assert out["render_backends"] == ["excel"] + + +class TestInsideALoop: + def test_an_element_wise_function_runs_period_by_period(self): + m = mo.Model("m", default_grain="month", default_start="2024-01", default_periods=3) + with m: + m.i = mo.MultiVariable("I") + m.i.start = mo.Variable(date(2024, 1, 15)) # a Monday + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + d.k = mo.lag(d.next, fill=0.0) + d.due = m.i.start + d.k * 7 + d.w = WEEKDAY(d.due, 2) + d.wk = WEEKNUM(d.due, 2) + d.gap = DATEDIF(m.i.start, d.due, "D") + d.next = d.k + d.w + assert d.k.value == [0.0, 1.0, 2.0] + assert d.due.value == [date(2024, 1, 15), date(2024, 1, 22), date(2024, 1, 29)] + assert d.w.value == [1, 1, 1] + assert d.wk.value == [3, 4, 5] + assert d.gap.value == [0, 7, 14] + assert d.next.value == [1.0, 2.0, 3.0] + + def test_a_holidays_range_is_refused_and_points_past_the_loop(self): + m = mo.Model("m", default_grain="month", default_start="2024-01", default_periods=3) + with pytest.raises(ValueError, match="NETWORKDAYS reads a whole series"): + with m: + m.i = mo.MultiVariable("I") + m.i.start = mo.Variable(date(2024, 1, 15)) + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + d.k = mo.lag(d.next, fill=0.0) + d.due = m.i.start + d.k * 7 + d.n = NETWORKDAYS(m.i.start, d.due) + d.next = d.k + d.n diff --git a/tests/test_day_month_totals.py b/tests/test_day_month_totals.py new file mode 100644 index 0000000..035c77c --- /dev/null +++ b/tests/test_day_month_totals.py @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A day sheet totals by month — "30 31 | Total Jan | 01 02 03 | Total Feb". + +A plan keys the sheet's OWN periods ``('period', i)`` and a bucket by +its ruler (``('month', bi)``); they used to share the spelling +``'month'``, so a month total over days took a day's address. Now the +month is a bucket like a quarter: its own columns, its own tier, its +own outline group, a live sum over its days. +""" +from openpyxl import load_workbook + +import modeleon as mo +from modeleon.compile.excel.totals import NATIVE, build_totals_plan, header_rows +from modeleon.core.time import DefaultWindow, _period_labels + + +def _book(tmp_path, kinds): + m = mo.Model('m', default_grain='day', default_start='2026-01-30', default_periods=5, + default_excel_view=mo.ExcelView(timeline={'totals': kinds})) + with m: + m.units = mo.Variable([1.0, 2.0, 3.0, 4.0, 5.0], regrain=mo.up('sum'), display_name='Units') + m.price = mo.Variable(10.0, display_name='Price') + m.rev = (m.units * m.price).set_regrain(mo.up('sum')).set_display_name('Revenue') + m.stock = mo.Variable([7.0, 8.0, 9.0, 10.0, 11.0], regrain=mo.up('last'), display_name='Stock') + path = tmp_path / 'day.xlsx' + m.to_excel(path) + return load_workbook(path)['M'] + + +class TestThePlan: + def test_native_columns_and_month_buckets_never_share_a_key(self): + plan = build_totals_plan(('month',), 'day', '2026-01-30', 5, 2) + assert plan.columns == [ + (NATIVE, 0), (NATIVE, 1), ('month', 0), + (NATIVE, 2), (NATIVE, 3), (NATIVE, 4), ('month', 1), + ] + assert len(plan.col_of) == len(plan.columns) == 7 + assert plan.native_col(2) == 5 and plan.month_col(2) == 5 # the old name still answers + + def test_the_month_tier_stands_over_the_days(self): + plan = build_totals_plan(('month', 'year'), 'day', '2026-01-30', 5, 2) + rows = header_rows(plan, _period_labels('2026-01-30', None, 5, 'day', 'finance')) + assert [r['cells'][0]['kind'] for r in rows] == ['year', 'month', NATIVE] + month_tier = [(c['label'], c['col'], c['colspan']) for c in rows[1]['cells']] + assert month_tier == [('Jan', 2, 2), ('Total Jan', 4, 1), ('Feb', 5, 3), ('Total Feb', 8, 1)] + assert [c['label'] for c in rows[2]['cells']] == ['30', '31', '01', '02', '03'] + + +class TestTheBook: + def test_month_totals_are_live_sums_over_their_days(self, tmp_path): + ws = _book(tmp_path, ['month']) + # Column B holds the constants (Price); the periods start at C. + assert [c.value for c in ws[1]][:9] == [None, None, 'Jan', None, 'Total Jan', 'Feb', None, None, 'Total Feb'] + assert [c.value for c in ws[2]][:9] == [None, None, '30', '31', None, '01', '02', '03', None] + rows = {r[0].value: [c.value for c in r[2:9]] for r in ws.iter_rows(min_row=3) if r[0].value} + assert rows['Units'] == [1, 2, '=SUM(C3,D3)', 3, 4, 5, '=SUM(F3,G3,H3)'] + assert rows['Revenue'][2] == '=SUM(C5,D5)' + # A stock carries the month's last day. + assert rows['Stock'][2] == '=D6' and rows['Stock'][6] == '=H6' + + def test_the_days_group_under_their_month(self, tmp_path): + ws = _book(tmp_path, ['month', 'year']) + levels = {k: v.outline_level for k, v in ws.column_dimensions.items() if v.outline_level} + # Days sit inside their month and the year; a month's total inside the year. + assert levels['C'] == 2 and levels['F'] == 2 + assert levels['E'] == 1 and levels['I'] == 1 + # The year total sums the days themselves. + row = next(r for r in ws.iter_rows() if r[0].value == 'Units') + assert row[9].value == '=SUM(C4,D4,F4,G4,H4)' + + +class TestGrowthByTheCalendar: + def test_a_day_grows_on_the_same_day_a_year_back(self): + from modeleon.compile.excel.writer import _growth_by_calendar + child = mo.Variable([float(i + 1) for i in range(400)]) + window = DefaultWindow('2025-12-30', 'day', 400) + out = _growth_by_calendar(child, window, 'day', 400, child.value) + assert out is not None + # 2026-12-30 is index 365; a year back is index 0. + assert out.value[365] == 366.0 / 1.0 + assert all(v is None for v in out.value[:365]) + assert out._metric_no_bucket + + def test_a_window_shorter_than_a_year_has_no_growth(self): + from modeleon.compile.excel.writer import _growth_by_calendar + child = mo.Variable([1.0] * 30) + assert _growth_by_calendar(child, DefaultWindow('2026-01-01', 'day', 30), 'day', 30, child.value) is None diff --git a/tests/test_entries.py b/tests/test_entries.py new file mode 100644 index 0000000..6cf831a --- /dev/null +++ b/tests/test_entries.py @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Which entered rows a derived track is computed from, read once per +model through the shared readings - and the same answer a fresh walk +gives, a chain deeper than the walk reads included.""" + +from __future__ import annotations + +import pytest + +import modeleon as mo +from modeleon.core import entries + +P = 12 + + +def _chain(depth: int): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=P, + tracks=mo.Tracks("plan", "actual", blend=mo.blend( + given="actual", follow="plan", until="2026-06"))) + with m: + m.inp = mo.MultiVariable("Inputs") + m.inp.rev = mo.Variable(plan=[100.0] * P, actual=[110.0] * 6 + [0.0] * (P - 6)) + m.s = mo.MultiVariable("Section") + prev = m.inp.rev + for j in range(depth): + setattr(m.s, f"l{j}", prev * 0.5 + m.inp.rev) + prev = getattr(m.s, f"l{j}") + return [getattr(m.s, f"l{j}")._value.as_dict() for j in range(depth)] + + +@pytest.mark.parametrize("depth", [entries.MAX_DEPTH - 5, entries.MAX_DEPTH + 5, 90]) +def test_the_shared_readings_answer_as_a_fresh_walk(depth, monkeypatch): + shared = _chain(depth) + monkeypatch.setattr(entries, "readings_under", lambda node: None) + assert _chain(depth) == shared + + +def test_a_chain_deeper_than_the_walk_is_read_once_per_line(monkeypatch): + # Past MAX_DEPTH the shared walk gives up and the line is read afresh; + # what it learnt on the way down tells the next line to give up at + # once, so the shared walk reads a line or two per line joined, not + # MAX_DEPTH of them. + calls = [] + read = entries._Walk._line + + def counting(self, var, depth, own): + if self.kept is not None: + calls.append(depth) + return read(self, var, depth, own) + + monkeypatch.setattr(entries._Walk, "_line", counting) + n = 3 * entries.MAX_DEPTH + _chain(n) + assert len(calls) < 5 * n diff --git a/tests/test_error_cells_in_functions.py b/tests/test_error_cells_in_functions.py index 18d9a7c..4e2faa7 100644 --- a/tests/test_error_cells_in_functions.py +++ b/tests/test_error_cells_in_functions.py @@ -39,12 +39,12 @@ def test_an_aggregate_over_an_error_cell_is_that_error() -> None: def test_the_first_error_in_reading_order_wins(fn) -> None: assert fn(mo.Variable(["#N/A", "#DIV/0!", 3.0])).value == "#N/A" assert fn(mo.Variable(["#DIV/0!", "#N/A", 3.0])).value == "#DIV/0!" + # Two series are read period by period, as every aggregate reads + # them: each period's first error. two = fn(mo.Variable([1.0, 2.0]), mo.Variable(["#VALUE!", "#N/A"])).value - if fn in (mo.MIN, mo.MAX): - # Two series are read period by period: each period's first error. - assert two == ["#VALUE!", "#N/A"] - else: - assert two == "#VALUE!" + assert two == ["#VALUE!", "#N/A"] + two = fn(mo.Variable(["#N/A", 2.0]), mo.Variable(["#VALUE!", "#DIV/0!"])).value + assert two == ["#N/A", "#DIV/0!"] def test_a_hole_is_skipped_as_a_blank_is() -> None: diff --git a/tests/test_excel_board_list_field.py b/tests/test_excel_board_list_field.py new file mode 100644 index 0000000..f5bf387 --- /dev/null +++ b/tests/test_excel_board_list_field.py @@ -0,0 +1,194 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A field of several cells inside an instance of an ``orient='columns'`` +board is refused at layout — never skipped. + +The matrix gives every field ONE cell per instance. A per-period line, +a keyed line or a line on an axis declared inside an instance fitted +no cell, and was left out without a word: its row vanished from the +sheet, and every aggregate reading it — ``mo.SUM(self.revenue)`` — was +written as its computed number, a dead cell with a plausible look. The +layout now refuses such a field, naming the field, the instance and +the section, and says where the line does fit: outside the section, as +a row of the sheet, or — for a per-period field — under +``orient='records'``, where it gets a block of its own beneath the +table. The board's own direct lines, scalar and series alike, keep +their rows; a board of scalar fields lays out exactly as before. +""" + +import warnings + +import pytest +from openpyxl import load_workbook + +import modeleon as mo + + +def _grid(model, sheet, tmp_path, cols=5): + path = tmp_path / "t.xlsx" + model.to_excel(str(path)) + ws = load_workbook(str(path))[sheet] + return [ + [ws.cell(r, c).value for c in range(1, cols + 1)] + for r in range(1, ws.max_row + 1) + ] + + +def _labels(grid): + return [row[0] for row in grid] + + +def _first_instance_column(grid): + captions = next(row for row in grid if "P1" in row) + first = captions.index("P1") + assert captions[first:first + 2] == ["P1", "P2"] + return first + + +def _model(orient="columns", *, view=False, field="period"): + m = mo.Model("m", display_name="M", default_grain="month", default_start="2026-01", + default_periods=3) + with m: + if field == "axis": + m.scen = mo.Variable(["Bear", "Base", "Bull"]) + m.sheet = mo.MultiVariable("Sheet", excel_props={"tab": True}) + with m.sheet as s: + if view: + s.board = mo.MultiVariable("Board") + s.board.default_excel_view = mo.ExcelView(orient=orient) + else: + s.board = mo.MultiVariable("Board", excel_props={"orient": orient}) + with s.board as b: + b.p1 = mo.MultiVariable("P1") + b.p1.price = mo.Variable(10.0, display_name="Price") + if field == "period": + b.p1.revenue = mo.Variable([1.0, 2.0, 3.0], display_name="Revenue") + b.p1.total = mo.Variable(mo.SUM(b.p1.revenue), display_name="Total") + elif field == "keyed": + b.p1.growth = mo.Variable( + {"Bear": 0.9, "Base": 1.0, "Bull": 1.1}, display_name="Growth" + ) + elif field == "axis": + b.p1.growth = mo.Variable( + [0.9, 1.0, 1.1], indexed_by=[m.scen], display_name="Growth" + ) + b.p2 = mo.MultiVariable("P2") + b.p2.price = mo.Variable(20.0, display_name="Price") + return m + + +def _tracked_model(*, vol=True): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=3, + tracks=mo.Tracks("plan", "actual")) + with m: + m.board = mo.MultiVariable("Board", excel_props={"tab": True, "orient": "columns"}) + with m.board as b: + b.p1 = mo.MultiVariable("P1") + b.p1.rate = mo.Variable(plan=5, display_name="Rate") + if vol: + b.p1.vol = mo.Variable(plan=[1, 2, 3], display_name="Vol") + b.p2 = mo.MultiVariable("P2") + b.p2.rate = mo.Variable(plan=6, display_name="Rate") + return m + + +def test_a_per_period_field_in_an_instance_is_refused_with_both_fixes(tmp_path): + path = tmp_path / "t.xlsx" + with warnings.catch_warnings(record=True) as rec: + warnings.simplefilter("always") + with pytest.raises(ValueError) as exc: + _model().to_excel(str(path)) + msg = str(exc.value) + assert "'Revenue' in 'P1' is a line of 3 cell(s), one per period" in msg + assert "'Board' is laid out with orient='columns'" in msg + assert "outside 'Board'" in msg + assert "orient='records'" in msg + # Refused at layout, before the workbook exists: nothing is written. + assert not path.exists() + # ...and before any aggregate over the line is rendered as a dead + # number — the warning that path fires never does. + assert not [w for w in rec if w.category.__name__ == "CrossScopeReferenceWarning"] + + +@pytest.mark.parametrize("field", ["keyed", "axis"]) +def test_a_keyed_field_in_an_instance_is_refused_toward_the_sheet(tmp_path, field): + with pytest.raises(ValueError) as exc: + _model(field=field).to_excel(str(tmp_path / "t.xlsx")) + msg = str(exc.value) + assert "'Growth' in 'P1' is a line of 3 cell(s), one per key" in msg + assert "under a caption row of its keys" in msg + # Under orient='records' a keyed field gets a block of bare values + # with no caption row of its keys — a half layout, so not offered. + assert "orient='records'" not in msg + + +def test_a_tracked_series_field_in_an_instance_is_refused_too(tmp_path): + with pytest.raises(ValueError) as exc: + _tracked_model().to_excel(str(tmp_path / "t.xlsx")) + # After the writer expands the tracks, the head row keeps the name + # 'Vol' and holds the list. + assert "'Vol' in 'P1' is a line of 3 cell(s), one per period" in str(exc.value) + + +def test_the_view_spelling_of_the_block_is_refused_the_same_way(tmp_path): + with pytest.raises(ValueError) as exc: + _model(view=True).to_excel(str(tmp_path / "t.xlsx")) + msg = str(exc.value) + assert "'Revenue' in 'P1'" in msg + assert "orient='columns'" in msg + + +def test_a_board_of_scalar_fields_lays_out_as_before(tmp_path): + g = _grid(_model(field="none"), "Sheet", tmp_path) + first = _first_instance_column(g) + assert g[_labels(g).index("Price")][first:first + 2] == [10, 20] + + +def test_a_tracked_scalar_field_stays_a_cell(tmp_path): + # A tracked SCALAR is a number once the writer expands the tracks, + # not a list: it keeps its cell in the matrix. + g = _grid(_tracked_model(vol=False), "Board", tmp_path) + assert g[0][:3] == [None, "P1", "P2"] + assert g[1][:3] == ["Rate", 5, 6] + + +def test_the_same_records_lay_out_with_orient_records(tmp_path): + # The advice the refusal gives is true: under orient='records' the + # per-period field gets a block of its own beneath the table — its + # caption, then one row per instance, the values past the table's + # columns — and the table's Total is a live SUM over that block, + # not the dead number 6. + g = _grid(_model(orient="records"), "Sheet", tmp_path, cols=7) + labels = _labels(g) + i = labels.index("Revenue") + assert labels[i + 1] == "P1" + assert g[i + 1][4:7] == [1, 2, 3] + assert g[labels.index("P1")][2] == f"=SUM(E{i + 2}:G{i + 2})" + + +def test_the_layout_records_the_refusal_and_other_readers_keep_their_addresses(tmp_path): + from modeleon.compile.excel.layout import LayoutEngine + m = _model() + engine = LayoutEngine([m.sheet], flatten_nested_sheets=True, header_rows=True) + addresses = engine.compute_addresses() + assert addresses, "the rest of the model keeps its addresses" + assert len(engine.refusals) == 1 and "'Revenue' in 'P1'" in engine.refusals[0] + # the notebook repr still renders (it never raises on a layout refusal) + assert "P1" in m._repr_html_() + # the compatibility check reports it instead of raising + issues = mo.check_compat(m, "excel") + layout_issues = [i for i in issues if i.kind == "layout"] + assert len(layout_issues) == 1 and "'Revenue' in 'P1'" in str(layout_issues[0]) + + +def test_a_one_value_bare_list_field_is_a_cell_like_everywhere_else(tmp_path): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=3) + m.board = mo.MultiVariable("Board", excel_props={"tab": True, "orient": "columns"}) + m.board.p1 = mo.MultiVariable("P1") + m.board.p1.fee = mo.Variable([5.0], display_name="Fee") + m.board.p1.fee2 = m.board.p1.fee + 1 + m.board.p2 = mo.MultiVariable("P2") + m.board.p2.fee = mo.Variable([6.0], display_name="Fee") + m.board.p2.fee2 = m.board.p2.fee + 1 + g = _grid(m, "Board", tmp_path, cols=3) + assert g[1][:3] == ["Fee", 5, 6] + assert g[2][:3] == ["Fee2", "=B2 + 1", "=C2 + 1"] diff --git a/tests/test_excel_criteria.py b/tests/test_excel_criteria.py new file mode 100644 index 0000000..83853d8 --- /dev/null +++ b/tests/test_excel_criteria.py @@ -0,0 +1,525 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Conditional aggregates in the workbook. + +A row is its cells' range (``Data!B2:D2``); a list of scalar lines whose +cells stand one under another is a vertical range (``Deals!B3:B5``) — +the field down the records of a table, or one period's column of their +blocks (``Deals!D8:D10``); cells that form no run cannot be one range, +so the cell carries its computed value and ``check_compat`` lists it. +The criteria is written as the spreadsheet writes it: ``">=10"``, a +cell, ``">=" & Inputs!B1``. ``MAXIFS`` / ``MINIFS`` are stored with the +``_xlfn.`` prefix. Every function recalculates in LibreOffice to the +engine's numbers, except where LibreOffice departs from Excel (a +boolean cell, numeric text, a wildcard over a number), pinned as an +expected failure. +""" + +import datetime as dt +import shutil +import subprocess +from pathlib import Path + +import pytest +from openpyxl import load_workbook + +import modeleon as mo +from modeleon.compile import check_compat +from modeleon.compile.excel.renderer import _contiguous_run +from modeleon.functions.criteria import ( + AVERAGEIF, + AVERAGEIFS, + COUNT, + COUNTA, + COUNTIF, + COUNTIFS, + MAXIFS, + MINIFS, + SUMIF, + SUMIFS, +) + + +def _rows(ws): + out = {} + for row in ws.iter_rows(): + label = row[0].value + if label: + out[str(label).split(" (")[0]] = [c.value for c in row[1:]] + return out + + +def _book(model, tmp_path): + path = tmp_path / "book.xlsx" + model.to_excel(path) + return load_workbook(path) + + +class Deal(mo.MultiVariableClass): + def compute(self, region, amount): + self.region = mo.Variable(region, display_name="Region") + self.amount = mo.Variable(amount, display_name="Amount", unit="USD") + self.flow = mo.Variable([amount * 0.1, amount * 0.2, amount * 0.3], display_name="Flow") + + +class Close(mo.MultiVariableClass): + def compute(self, region, closed): + self.region = mo.Variable(region, display_name="Region") + self.closed = mo.Variable(closed, display_name="Closed") + + +def _model(): + m = mo.Model("m", display_name="M", default_grain="month", default_start="2026-01", + default_periods=3) + with m: + m.inputs = mo.MultiVariable("Inputs", excel_props={"tab": True}) + m.inputs.floor = mo.Variable(150.0, display_name="Floor") + m.inputs.pick = mo.Variable("north", display_name="Pick") + m.inputs.rule = mo.Variable("<>100", display_name="Rule") + m.data = mo.MultiVariable("Data", excel_props={"tab": True}) + m.data.region = mo.Variable(["North", "South", "Northeast"], display_name="Region") + m.data.amount = mo.Variable([100.0, 200.0, 300.0], display_name="Amount", unit="USD") + m.data.year = mo.Variable([2026, 2026, 2027], display_name="Year") + m.deals = mo.MultiVariable("Deals", excel_props={"tab": True, "orient": "records"}) + with m.deals as d: + d.a = Deal(region="North", amount=100.0, display_name="Alpha") + d.b = Deal(region="South", amount=200.0, display_name="Beta") + d.c = Deal(region="North", amount=300.0, display_name="Gamma") + m.closes = mo.MultiVariable("Closes", excel_props={"tab": True, "orient": "records"}) + with m.closes as c: + c.a = Close(region="North", closed=dt.date(2026, 3, 31), display_name="A") + c.b = Close(region="South", closed=dt.date(2026, 6, 30), display_name="B") + c.c = Close(region="North", closed=dt.date(2024, 2, 29), display_name="C") + m.s = mo.MultiVariable("Summary", excel_props={"tab": True}) + return m + + +# ══════════════════════════════════════════════════════════════════════ +# Rows are ranges, the criteria written as the spreadsheet writes it +# ══════════════════════════════════════════════════════════════════════ + + +class TestRowsAreRanges: + def test_every_function_over_rows(self, tmp_path): + m = _model() + d = m.data + with m.s as s: + s.sumif = SUMIF(d.region, "North", d.amount).set_display_name("sumif") + s.sumif_self = SUMIF(d.amount, ">=150").set_display_name("sumif_self") + s.sumifs = SUMIFS(d.amount, d.region, "North", d.year, 2026).set_display_name("sumifs") + s.countif = COUNTIF(d.region, "n*").set_display_name("countif") + s.countifs = COUNTIFS(d.region, "<>South", d.amount, ">100").set_display_name("countifs") + s.averageif = AVERAGEIF(d.region, "N*", d.amount).set_display_name("averageif") + s.averageifs = AVERAGEIFS(d.amount, d.year, 2026).set_display_name("averageifs") + s.maxifs = MAXIFS(d.amount, d.region, "N*").set_display_name("maxifs") + s.minifs = MINIFS(d.amount, d.region, "N*").set_display_name("minifs") + s.count = (COUNT(d.amount) + COUNT(d.region)).set_display_name("count") + s.counta = COUNTA(d.region).set_display_name("counta") + # Two rows are counted period by period: one call per + # column, after the scalars' column. + s.count_pp = COUNT(d.amount, d.region).set_display_name("count_pp") + rows = _rows(_book(m, tmp_path)["Summary"]) + assert rows["sumif"][0] == '=SUMIF(Data!B2:D2, "North", Data!B3:D3)' + assert rows["sumif_self"][0] == '=SUMIF(Data!B3:D3, ">=150")' + assert rows["sumifs"][0] == '=SUMIFS(Data!B3:D3, Data!B2:D2, "North", Data!B4:D4, 2026)' + assert rows["countif"][0] == '=COUNTIF(Data!B2:D2, "n*")' + assert rows["countifs"][0] == '=COUNTIFS(Data!B2:D2, "<>South", Data!B3:D3, ">100")' + assert rows["averageif"][0] == '=AVERAGEIF(Data!B2:D2, "N*", Data!B3:D3)' + assert rows["averageifs"][0] == '=AVERAGEIFS(Data!B3:D3, Data!B4:D4, 2026)' + assert rows["maxifs"][0] == '=_xlfn.MAXIFS(Data!B3:D3, Data!B2:D2, "N*")' + assert rows["minifs"][0] == '=_xlfn.MINIFS(Data!B3:D3, Data!B2:D2, "N*")' + assert rows["count"][0] == "=COUNT(Data!B3:D3) + COUNT(Data!B2:D2)" + assert rows["counta"][0] == "=COUNTA(Data!B2:D2)" + assert rows["count_pp"][1:4] == ["=COUNT(Data!B3, Data!B2)", "=COUNT(Data!C3, Data!C2)", + "=COUNT(Data!D3, Data!D2)"] + assert m.s.count_pp.value == [1, 1, 1] + # The values the cells cache are the engine's. + assert (m.s.sumif.value, m.s.sumif_self.value, m.s.sumifs.value) == (100.0, 500.0, 100.0) + assert (m.s.countif.value, m.s.countifs.value) == (2, 1) + assert (m.s.averageif.value, m.s.averageifs.value) == (200.0, 150.0) + assert (m.s.maxifs.value, m.s.minifs.value, m.s.count.value, m.s.counta.value) == ( + 300.0, 100.0, 3, 3) + + def test_every_criteria_kind(self, tmp_path): + m = _model() + d, i = m.data, m.inputs + with m.s as s: + s.text = SUMIF(d.region, "North", d.amount).set_display_name("text") + s.number = SUMIF(d.year, 2026, d.amount).set_display_name("number") + s.cell = SUMIF(d.region, i.pick, d.amount).set_display_name("cell") + s.rule = SUMIF(d.amount, i.rule).set_display_name("rule") + s.joined = SUMIF(d.amount, (">=", i.floor)).set_display_name("joined") + s.concat = SUMIF(d.amount, mo.CONCAT(">=", i.floor)).set_display_name("concat") + s.pair = SUMIF(d.amount, (">", 150)).set_display_name("pair") + s.quoted = COUNTIF(d.region, 'say "hi"').set_display_name("quoted") + rows = _rows(_book(m, tmp_path)["Summary"]) + assert rows["text"][0] == '=SUMIF(Data!B2:D2, "North", Data!B3:D3)' + assert rows["number"][0] == "=SUMIF(Data!B4:D4, 2026, Data!B3:D3)" + assert rows["cell"][0] == "=SUMIF(Data!B2:D2, Inputs!B2, Data!B3:D3)" + assert rows["rule"][0] == "=SUMIF(Data!B3:D3, Inputs!B3)" + assert rows["joined"][0] == '=SUMIF(Data!B3:D3, ">=" & Inputs!B1)' + assert rows["concat"][0] == '=SUMIF(Data!B3:D3, ">=" & Inputs!B1)' + assert rows["pair"][0] == '=SUMIF(Data!B3:D3, ">150")' + assert rows["quoted"][0] == '=COUNTIF(Data!B2:D2, "say ""hi""")' + assert (m.s.cell.value, m.s.rule.value, m.s.joined.value, m.s.pair.value) == ( + 100.0, 500.0, 500.0, 500.0) + + def test_on_the_rows_own_sheet_and_inside_a_larger_formula(self, tmp_path): + m = _model() + with m.data as d: + d.north = SUMIF(d.region, "North", d.amount).set_display_name("north") + d.share = (SUMIF(d.region, "North", d.amount) / mo.SUM(d.amount)).set_display_name("share") + rows = _rows(_book(m, tmp_path)["Data"]) + # The scalar lines take column B; the periods move to C. + assert rows["north"][0] == '=SUMIF(C2:E2, "North", C3:E3)' + assert rows["share"][0] == '=SUMIF(C2:E2, "North", C3:E3) / SUM(C3:E3)' + + +# ══════════════════════════════════════════════════════════════════════ +# A list of scalar lines is a vertical range +# ══════════════════════════════════════════════════════════════════════ + + +class TestRecordsAreVerticalRanges: + def test_a_field_down_the_records(self, tmp_path): + m = _model() + d = m.deals + regions = [d.a.region, d.b.region, d.c.region] + amounts = [d.a.amount, d.b.amount, d.c.amount] + with m.s as s: + s.sumif = SUMIF(regions, "North", amounts).set_display_name("sumif") + s.sumifs = SUMIFS(amounts, regions, "North", amounts, ">100").set_display_name("sumifs") + s.countif = COUNTIF(regions, "North").set_display_name("countif") + s.countifs = COUNTIFS(regions, "North", amounts, ">100").set_display_name("countifs") + s.averageif = AVERAGEIF(regions, "North", amounts).set_display_name("averageif") + s.averageifs = AVERAGEIFS(amounts, regions, "North").set_display_name("averageifs") + s.maxifs = MAXIFS(amounts, regions, "North").set_display_name("maxifs") + s.minifs = MINIFS(amounts, regions, "North").set_display_name("minifs") + s.count = COUNT(amounts).set_display_name("count") + s.counta = COUNTA(regions).set_display_name("counta") + s.one = SUMIF([d.a.region], "North", [d.a.amount]).set_display_name("one") + rows = _rows(_book(m, tmp_path)["Summary"]) + assert rows["sumif"][0] == '=SUMIF(Deals!B3:B5, "North", Deals!C3:C5)' + assert rows["sumifs"][0] == '=SUMIFS(Deals!C3:C5, Deals!B3:B5, "North", Deals!C3:C5, ">100")' + assert rows["countif"][0] == '=COUNTIF(Deals!B3:B5, "North")' + assert rows["countifs"][0] == '=COUNTIFS(Deals!B3:B5, "North", Deals!C3:C5, ">100")' + assert rows["averageif"][0] == '=AVERAGEIF(Deals!B3:B5, "North", Deals!C3:C5)' + assert rows["averageifs"][0] == '=AVERAGEIFS(Deals!C3:C5, Deals!B3:B5, "North")' + assert rows["maxifs"][0] == '=_xlfn.MAXIFS(Deals!C3:C5, Deals!B3:B5, "North")' + assert rows["minifs"][0] == '=_xlfn.MINIFS(Deals!C3:C5, Deals!B3:B5, "North")' + assert rows["count"][0] == "=COUNT(Deals!C3:C5)" + assert rows["counta"][0] == "=COUNTA(Deals!B3:B5)" + assert rows["one"][0] == '=SUMIF(Deals!B3, "North", Deals!C3)' + assert (m.s.sumif.value, m.s.sumifs.value, m.s.countif.value) == (400.0, 300.0, 2) + + def test_one_period_s_column_of_the_records_blocks(self, tmp_path): + m = _model() + d = m.deals + regions = [d.a.region, d.b.region, d.c.region] + with m.s as s: + s.jan = SUMIFS([d.a.flow[0], d.b.flow[0], d.c.flow[0]], regions, "North").set_display_name("jan") + s.mar = SUMIFS([d.a.flow[-1], d.b.flow[-1], d.c.flow[-1]], regions, "North").set_display_name("mar") + rows = _rows(_book(m, tmp_path)["Summary"]) + assert rows["jan"][0] == '=SUMIFS(Deals!D8:D10, Deals!B3:B5, "North")' + assert rows["mar"][0] == '=SUMIFS(Deals!F8:F10, Deals!B3:B5, "North")' + assert m.s.jan.value == pytest.approx(40.0) and m.s.mar.value == pytest.approx(120.0) + + def test_on_the_table_s_own_sheet(self, tmp_path): + m = _model() + with m.deals as d: + d.north = SUMIF([d.a.region, d.b.region, d.c.region], "North", + [d.a.amount, d.b.amount, d.c.amount]).set_display_name("North total") + rows = _rows(_book(m, tmp_path)["Deals"]) + assert rows["North total"][0] == '=SUMIF(B3:B5, "North", C3:C5)' + + def test_a_row_of_scalar_lines_across_a_board_is_a_horizontal_range(self, tmp_path): + m = mo.Model("m", display_name="M") + with m: + m.board = mo.MultiVariable("Board", excel_props={"tab": True, "orient": "columns"}) + with m.board as b: + for name, cost in (("p1", 50.0), ("p2", 70.0), ("p3", 90.0)): + col = mo.MultiVariable(name.upper()) + setattr(b, name, col) + col.cost = mo.Variable(cost, display_name="Cost") + m.s = mo.MultiVariable("Summary", excel_props={"tab": True}) + m.s.big = COUNTIF([m.board.p1.cost, m.board.p2.cost, m.board.p3.cost], + ">60").set_display_name("big") + rows = _rows(_book(m, tmp_path)["Summary"]) + assert rows["big"][0] == '=COUNTIF(Board!B2:D2, ">60")' and m.s.big.value == 2 + + +class TestNoRunShipsTheValue: + def test_a_gap_two_sheets_or_a_literal_among_the_cells(self, tmp_path): + m = _model() + d = m.deals + with m.s as s: + s.gap = SUMIF([d.a.region, d.c.region], "North", + [d.a.amount, d.c.amount]).set_display_name("gap") + s.sheets = SUMIF([d.a.region, m.inputs.pick], "North", + [d.a.amount, m.inputs.floor]).set_display_name("sheets") + s.literal = SUMIF([d.a.region, "North"], "North", + [d.a.amount, 7.0]).set_display_name("literal") + s.backwards = SUMIF([d.c.region, d.b.region, d.a.region], "North", + [d.c.amount, d.b.amount, d.a.amount]).set_display_name("backwards") + rows = _rows(_book(m, tmp_path)["Summary"]) + # (``pick`` holds "north": the text criteria is case-insensitive.) + assert rows["gap"][0] == 400 and rows["sheets"][0] == 250 + assert rows["literal"][0] == 107 and rows["backwards"][0] == 400 + issues = check_compat(m, "excel") + assert [(i.kind, i.qpath) for i in issues] == [ + ("value", m.s.gap.id), ("value", m.s.sheets.id), + ("value", m.s.literal.id), ("value", m.s.backwards.id)] + assert all(i.literal is None and i.inlined.startswith("[") for i in issues), issues + + def test_a_run_is_listed_by_nobody(self, tmp_path): + m = _model() + d = m.deals + with m.s as s: + s.ok = SUMIF([d.a.region, d.b.region, d.c.region], "North", + [d.a.amount, d.b.amount, d.c.amount]).set_display_name("ok") + _book(m, tmp_path) + assert check_compat(m, "excel") == [] + + def test_the_counts_list_the_cells(self, tmp_path): + # COUNT and COUNTA take any number of arguments: a comma list of + # the cells reads as their range would. + m = _model() + d = m.deals + with m.s as s: + s.count = COUNT([d.a.amount, d.c.amount]).set_display_name("count") + s.counta = COUNTA([d.a.amount, d.c.amount]).set_display_name("counta") + rows = _rows(_book(m, tmp_path)["Summary"]) + assert rows["count"][0] == "=COUNT(Deals!C3, Deals!C5)" + assert rows["counta"][0] == "=COUNTA(Deals!C3, Deals!C5)" + assert check_compat(m, "excel") == [] + + def test_a_row_the_book_has_no_cells_for(self, tmp_path): + m = _model() + with m.s as s: + s.scaled = SUMIF(m.data.region, "North", m.data.amount * 2.0).set_display_name("scaled") + with pytest.warns(mo.CrossScopeReferenceWarning, match="SUMIF"): + rows = _rows(_book(m, tmp_path)["Summary"]) + assert rows["scaled"][0] == 200 + + +class TestContiguousRun: + @pytest.mark.parametrize("refs, run", [ + (["B3", "B4", "B5"], True), + (["Deals!B3", "Deals!B4"], True), + (["$B$3", "$B$4"], True), + (["B3"], True), + (["B3", "C3", "D3"], True), + (["B3", "B5"], False), + (["B3", "C4"], False), + (["B5", "B4", "B3"], False), + (["D3", "C3"], False), + (["B3", "B4", "C4"], False), + ]) + def test_one_row_or_one_column_in_order(self, refs, run): + assert _contiguous_run(refs) is run + + +# ══════════════════════════════════════════════════════════════════════ +# LibreOffice recalculates these functions to the engine's numbers +# ══════════════════════════════════════════════════════════════════════ + + +def _soffice(): + for candidate in ("soffice", "libreoffice"): + found = shutil.which(candidate) + if found: + return found + mac = Path("/Applications/LibreOffice.app/Contents/MacOS/soffice") + return str(mac) if mac.exists() else None + + +def _recalculated(model, tmp_path): + """The Summary sheet after LibreOffice recalculates the written + workbook, by line label.""" + soffice = _soffice() + if soffice is None: + pytest.skip("LibreOffice not installed") + path = tmp_path / "criteria.xlsx" + model.to_excel(path) + subprocess.run( + [soffice, f"-env:UserInstallation={(tmp_path / 'prof').as_uri()}", + "--headless", "--convert-to", "xlsx", "--outdir", + str(tmp_path / "calc"), str(path)], + check=True, capture_output=True, + ) + return _rows(load_workbook(tmp_path / "calc" / "criteria.xlsx", data_only=True)["Summary"]) + + +def _serial(value): + """A date as the cell holds it, so a recalculated date (read back as + a datetime) compares with the engine's date.""" + if isinstance(value, dt.datetime): + return (value - dt.datetime(1899, 12, 30)) / dt.timedelta(days=1) + if isinstance(value, dt.date): + return (value - dt.date(1899, 12, 30)).days + return value + + +@pytest.mark.slow +def test_the_workbook_recalculates_to_the_python_values(tmp_path): + m = _model() + d, i, t, c = m.data, m.inputs, m.deals, m.closes + regions = [t.a.region, t.b.region, t.c.region] + amounts = [t.a.amount, t.b.amount, t.c.amount] + closes = [c.a.closed, c.b.closed, c.c.closed] + close_regions = [c.a.region, c.b.region, c.c.region] + i.blank = mo.Variable(None, display_name="Blank") + i.empty = mo.Variable("", display_name="Empty") + d.closed = mo.Variable([dt.date(2026, 1, 31), dt.date(2026, 2, 28), dt.date(2024, 2, 29)], + display_name="Closed") + d.note = mo.Variable(["ab", "a~b", ""], display_name="Note") + d.mix = mo.Variable([0.0, None, 5.0], display_name="Mix") + d.flag = mo.Variable([True, False, True], display_name="Flag") + d.gap = mo.IF(d.flag, "", "x").set_display_name("Gap") + d.numtxt = mo.Variable(["10", 7.0, "-1"], display_name="Numtxt") + d.p1 = mo.Variable([0.1, 0.2, 0.7], display_name="P1") + d.p2 = mo.Variable([0.2, 0.1, -0.4], display_name="P2") + d.share = (d.p1 + d.p2).set_display_name("Share") + d.tf = mo.Variable(["TRUE", True, "x"], display_name="Tf") + d.when = mo.Variable([dt.datetime(2026, 1, 31, 12), dt.datetime(2026, 1, 31, 6), + dt.datetime(2026, 1, 31, 18)], display_name="When") + i.thr = mo.Variable(0.3, display_name="Thr") + i.err = mo.Variable("#DIV/0!", display_name="Err") + with m.s as s: + s.sumif = SUMIF(d.region, "North", d.amount).set_display_name("sumif") + s.sumif_ge = SUMIF(d.amount, ">=150").set_display_name("sumif_ge") + s.sumif_cell = SUMIF(d.region, i.pick, d.amount).set_display_name("sumif_cell") + s.sumif_rule = SUMIF(d.amount, i.rule).set_display_name("sumif_rule") + s.sumif_joined = SUMIF(d.amount, (">=", i.floor)).set_display_name("sumif_joined") + s.sumifs = SUMIFS(d.amount, d.region, "N*", d.year, 2026).set_display_name("sumifs") + s.countif = COUNTIF(d.region, "n*").set_display_name("countif") + s.countif_ne = COUNTIF(d.region, "<>South").set_display_name("countif_ne") + s.countifs = COUNTIFS(d.region, "<>South", d.amount, ">100").set_display_name("countifs") + s.averageif = AVERAGEIF(d.region, "N*", d.amount).set_display_name("averageif") + s.averageifs = AVERAGEIFS(d.amount, d.year, 2026).set_display_name("averageifs") + s.maxifs = MAXIFS(d.amount, d.region, "N*").set_display_name("maxifs") + s.minifs = MINIFS(d.amount, d.region, "N*", d.year, "<>2026").set_display_name("minifs") + s.count = (COUNT(d.amount) + COUNT(d.region)).set_display_name("count") + s.counta = (COUNTA(d.region) + COUNTA(d.amount)).set_display_name("counta") + # Two rows and a value typed in, counted period by period. + s.count_pp = COUNT(d.amount, d.region, d.mix).set_display_name("count_pp") + s.counta_pp = COUNTA(d.region, d.note, d.gap, "x").set_display_name("counta_pp") + s.v_sumif = SUMIF(regions, "North", amounts).set_display_name("v_sumif") + s.v_sumifs = SUMIFS(amounts, regions, "North", amounts, ">100").set_display_name("v_sumifs") + s.v_countifs = COUNTIFS(regions, "North", amounts, "<300").set_display_name("v_countifs") + s.v_averageif = AVERAGEIF(regions, "North", amounts).set_display_name("v_averageif") + s.v_maxifs = MAXIFS(amounts, regions, "North").set_display_name("v_maxifs") + s.v_minifs = MINIFS(amounts, regions, "North").set_display_name("v_minifs") + s.v_jan = SUMIFS([t.a.flow[0], t.b.flow[0], t.c.flow[0]], regions, "North").set_display_name("v_jan") + s.v_count = COUNT(amounts).set_display_name("v_count") + # Empty text typed in is a blank cell; a formula's is a cell. + s.typed_ne = COUNTIF(d.note, "<>").set_display_name("typed_ne") + s.typed_blank = COUNTIF(d.note, "").set_display_name("typed_blank") + s.typed_counta = COUNTA(d.note).set_display_name("typed_counta") + s.formula_ne = COUNTIF(d.gap, "<>").set_display_name("formula_ne") + s.formula_blank = COUNTIF(d.gap, "").set_display_name("formula_blank") + s.formula_counta = COUNTA(d.gap).set_display_name("formula_counta") + # ``~`` takes the next character literally. + s.tilde = COUNTIF(d.note, "a~b").set_display_name("tilde") + s.tilde_lead = COUNTIF(d.region, "~north").set_display_name("tilde_lead") + # None, a blank cell and an empty-text cell as the criteria are 0. + s.none = COUNTIF(d.mix, None).set_display_name("none") + s.blank_cell = SUMIF(d.mix, i.blank, d.amount).set_display_name("blank_cell") + s.empty_cell = COUNTIF(d.mix, i.empty).set_display_name("empty_cell") + # Dates are numbers; the largest, the smallest and the mean come back as dates. + s.dt_serial = COUNTIF(d.closed, ">45000").set_display_name("dt_serial") + s.dt_ge = COUNTIF(d.closed, (">=", dt.date(2026, 1, 1))).set_display_name("dt_ge") + s.dt_maxifs = MAXIFS(d.closed, d.region, "N*").set_display_name("dt_maxifs") + s.dt_minifs = MINIFS(d.closed, d.region, "N*").set_display_name("dt_minifs") + s.dt_averageif = AVERAGEIF(d.region, "<>South", d.closed).set_display_name("dt_averageif") + s.dt_sumif = SUMIF(d.region, "N*", d.closed).set_display_name("dt_sumif") + s.v_maxifs_date = MAXIFS(closes, close_regions, "North").set_display_name("v_maxifs_date") + s.v_averageifs_date = AVERAGEIFS(closes, close_regions, "North").set_display_name("v_averageifs_date") + # Numeric text meets an equality, never an ordering. + s.nt_gt = COUNTIF(d.numtxt, ">5").set_display_name("nt_gt") + s.nt_ge0 = COUNTIF(d.numtxt, ">=0").set_display_name("nt_ge0") + s.nt_lt0 = COUNTIF(d.numtxt, "<0").set_display_name("nt_lt0") + s.nt_sumif = SUMIF(d.numtxt, ">5", d.amount).set_display_name("nt_sumif") + s.nt_text = COUNTIF(d.numtxt, "10").set_display_name("nt_text") + # '=' is a blank cell only; an ordering alone compares text with ''. + s.eq_formula = COUNTIF(d.gap, "=").set_display_name("eq_formula") + s.eq_typed = COUNTIF(d.note, "=").set_display_name("eq_typed") + s.ge_alone = COUNTIF(d.gap, ">=").set_display_name("ge_alone") + s.gt_alone = COUNTIF(d.gap, ">").set_display_name("gt_alone") + s.le_alone = COUNTIF(d.gap, "<=").set_display_name("le_alone") + s.ge_typed = COUNTIF(d.note, ">=").set_display_name("ge_typed") + s.eq_none = COUNTIF(d.mix, ("=", None)).set_display_name("eq_none") + s.ne_none = COUNTIF(d.mix, ("<>", None)).set_display_name("ne_none") + s.ge_blank = COUNTIF(d.region, (">=", i.blank)).set_display_name("ge_blank") + # Numbers are equal as the spreadsheet compares them: 0.1 + 0.2 is 0.3. + s.fl_eq = COUNTIF(d.share, 0.3).set_display_name("fl_eq") + s.fl_cell = COUNTIF(d.share, i.thr).set_display_name("fl_cell") + s.fl_ge = COUNTIF(d.share, ">=0.3").set_display_name("fl_ge") + s.fl_gt = COUNTIF(d.share, ">0.3").set_display_name("fl_gt") + s.fl_sumif = SUMIF(d.share, 0.3, d.amount).set_display_name("fl_sumif") + # A criteria cell holding an error is the result. + s.err_count = COUNTIF(d.region, i.err).set_display_name("err_count") + s.err_sumif = SUMIF(d.region, i.err, d.amount).set_display_name("err_sumif") + s.err_joined = COUNTIF(d.amount, (">", i.err)).set_display_name("err_joined") + # A date written 2026-01-31 is that date. + s.iso_ge = COUNTIF(d.closed, ">=2026-01-01").set_display_name("iso_ge") + s.iso_eq = COUNTIF(d.closed, "2026-01-31").set_display_name("iso_eq") + s.iso_sumif = SUMIF(d.closed, "<2026-02-01", d.amount).set_display_name("iso_sumif") + # The text TRUE meets the criteria written "TRUE". + s.true_text = COUNTIF(d.tf, "TRUE").set_display_name("true_text") + s.not_true = COUNTIF(d.tf, "<>TRUE").set_display_name("not_true") + # A time of day is written as the serial, never as DATE(...). + s.noon = COUNTIF(d.when, dt.datetime(2026, 1, 31, 12)).set_display_name("noon") + s.from_noon = COUNTIF(d.when, (">=", dt.datetime(2026, 1, 31, 12))).set_display_name("from_noon") + # None typed into COUNT is the 0 the formula holds; COUNTA lists its cells. + s.count_none = COUNT(1, None).set_display_name("count_none") + s.counta_list = COUNTA([t.a.amount, t.c.amount]).set_display_name("counta_list") + calc = _recalculated(m, tmp_path) + lines = [(name, line) for name, line in m.s.walk() if isinstance(line, mo.Variable)] + assert len(lines) == 75 + # Empty text typed into Note is a blank; Gap's, a formula's, is a cell. + assert m.s.count_pp.value == [2, 1, 2] and m.s.counta_pp.value == [4, 4, 3] + assert (m.s.nt_gt.value, m.s.nt_lt0.value, m.s.eq_formula.value, m.s.fl_eq.value) == (1, 0, 0, 3) + assert (m.s.err_count.value, m.s.iso_ge.value, m.s.true_text.value) == ("#DIV/0!", 2, 2) + assert (m.s.noon.value, m.s.from_noon.value) == (1, 2) + assert m.s.dt_maxifs.value == dt.date(2026, 1, 31) + assert m.s.dt_averageif.value == dt.date(2025, 2, 14) + assert m.s.v_averageifs_date.value == dt.datetime(2025, 3, 15, 12) + for name, line in lines: + if isinstance(line.value, list): + # One call per period, in the period columns after the scalars'. + assert calc[name][1:4] == pytest.approx(line.value), (name, calc[name], line.value) + continue + assert _serial(calc[name][0]) == pytest.approx(_serial(line.value)), ( + name, calc[name][0], line.value) + + +@pytest.mark.slow +@pytest.mark.xfail(strict=True, reason=( + "LibreOffice reads a boolean cell as a number, does not read numeric text in a " + "criteria range as a number, and matches a wildcard against a number's digits; " + "Excel does none of these, and the engine keeps Excel's rule")) +def test_where_libreoffice_departs_from_excel(tmp_path): + m = mo.Model("m", display_name="M", default_grain="month", default_start="2026-01", + default_periods=6) + with m: + m.data = mo.MultiVariable("Data", excel_props={"tab": True}) + m.data.bools = mo.Variable([True, False, True, None, 1, 0], display_name="Bools") + m.data.num = mo.Variable([100.0, -5.0, 0.0, None, "10", True], display_name="Num") + m.s = mo.MultiVariable("Summary", excel_props={"tab": True}) + d = m.data + with m.s as s: + # A boolean cell: Excel 1, LibreOffice 3 (TRUE is 1 there); COUNT 2 against 5. + s.boolean = COUNTIF(d.bools, ">0").set_display_name("boolean") + s.boolean_count = COUNT(d.bools).set_display_name("boolean_count") + # Numeric text in the criteria range: Excel 1, LibreOffice 0. + s.numeric_text = COUNTIF(d.num, 10).set_display_name("numeric_text") + # A wildcard over the number 100: Excel 1 (the text "10"), LibreOffice 2. + s.wildcard = COUNTIF(d.num, "1*").set_display_name("wildcard") + # "TRUE" over TRUE, FALSE, TRUE, 1, 0: Excel 2, LibreOffice 3 (1 is TRUE there). + s.true_one = COUNTIF(d.bools, "TRUE").set_display_name("true_one") + assert (m.s.boolean.value, m.s.boolean_count.value, m.s.true_one.value) == (1, 2, 2) + assert (m.s.numeric_text.value, m.s.wildcard.value) == (1, 1) + calc = _recalculated(m, tmp_path) + departing = [(name, calc[name][0], line.value) for name, line in m.s.walk() + if isinstance(line, mo.Variable) and calc[name][0] != line.value] + assert departing == [], departing diff --git a/tests/test_excel_dates.py b/tests/test_excel_dates.py new file mode 100644 index 0000000..eaeeb82 --- /dev/null +++ b/tests/test_excel_dates.py @@ -0,0 +1,323 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The workbook writes ``DATEDIF``, ``WEEKDAY``, ``WEEKNUM``, +``NETWORKDAYS`` and ``WORKDAY`` as the live calls they are, and +recalculates to the engine's values. + +A scalar call reads its cells (``=DATEDIF(B2, B3, "M")``); a call over +a series is one call per period, each reading this period's cell +(``=WEEKDAY(C2, 2)``); the holidays of ``NETWORKDAYS`` / ``WORKDAY`` are +the one operand written as a range, the same in every period +(``=NETWORKDAYS(I!B2, C2, I!C4:E4)``); a ``WORKDAY`` cell shows as a +date. All five names are classic, stored bare (no ``_xlfn.`` prefix). +The slow test lets LibreOffice recalculate a workbook that holds every +function under test and compares; the three documented differences +between Excel and LibreOffice — ``DATEDIF``'s ``"YD"`` across a 29 +February, ``WEEKNUM`` in the last days of a year, and the error a +``DATEDIF`` with the start after the end shows — are each pinned as an +expected failure. +""" + +from __future__ import annotations + +import shutil +import subprocess +import warnings +from datetime import date, datetime +from pathlib import Path + +import pytest +from openpyxl import load_workbook + +import modeleon as mo +from modeleon.compile.excel.renderer import CrossScopeReferenceWarning +from modeleon.functions.dates import DATEDIF, NETWORKDAYS, WEEKDAY, WEEKNUM, WORKDAY + + +def _cells(ws): + """``{label: [cells right of the label]}`` of one sheet.""" + out = {} + for row in ws.iter_rows(): + label = row[0].value + if label: + out[str(label)] = [c for c in row[1:] if c.value is not None] + return out + + +def _values(ws): + return {label: [c.value for c in cells] for label, cells in _cells(ws).items()} + + +def _model(): + m = mo.Model("m", default_grain="month", default_start="2024-01", default_periods=3) + with m: + m.i = mo.MultiVariable("I") + m.i.start = mo.Variable(date(2024, 1, 15), display_name="start") + m.i.end = mo.Variable(date(2024, 3, 20), display_name="end") + m.i.hol = mo.Variable( + [date(2024, 1, 1), date(2024, 2, 5), date(2024, 3, 15)], display_name="hol", + ) + m.x = mo.MultiVariable("X") + m.x.ends = mo.Variable( + [date(2024, 1, 31), date(2024, 2, 29), date(2024, 3, 31)], display_name="ends", + ) + return m + + +def _book(model, tmp_path): + path = tmp_path / "book.xlsx" + model.to_excel(str(path)) + return load_workbook(str(path)) + + +class TestTheWrittenFormulas: + def test_scalar_calls_read_their_cells(self, tmp_path): + m = _model() + with m: + m.i.months = DATEDIF(m.i.start, m.i.end, "M").set_display_name("months") + m.i.wd = WEEKDAY(m.i.start).set_display_name("wd") + m.i.wd2 = WEEKDAY(m.i.start, 2).set_display_name("wd2") + m.i.wk = WEEKNUM(m.i.end, 21).set_display_name("wk") + m.i.n = NETWORKDAYS(m.i.start, m.i.end, m.i.hol).set_display_name("n") + m.i.n0 = NETWORKDAYS(m.i.start, m.i.end).set_display_name("n0") + m.i.n1 = NETWORKDAYS(m.i.start, m.i.end, m.i.start).set_display_name("n1") + m.i.w = WORKDAY(m.i.start, 10, m.i.hol).set_display_name("w") + m.i.w1 = WORKDAY(m.i.start, 10, date(2024, 1, 16)).set_display_name("w1") + i = _values(_book(m, tmp_path)["I"]) + assert i["start"] == [datetime(2024, 1, 15)] and i["end"] == [datetime(2024, 3, 20)] + assert i["hol"] == [datetime(2024, 1, 1), datetime(2024, 2, 5), datetime(2024, 3, 15)] + assert i["months"] == ['=DATEDIF(B2, B3, "M")'] + assert i["wd"] == ["=WEEKDAY(B2)"] + assert i["wd2"] == ["=WEEKDAY(B2, 2)"] + assert i["wk"] == ["=WEEKNUM(B3, 21)"] + assert i["n"] == ["=NETWORKDAYS(B2, B3, C4:E4)"] + assert i["n0"] == ["=NETWORKDAYS(B2, B3)"] + assert i["n1"] == ["=NETWORKDAYS(B2, B3, B2)"] + assert i["w"] == ["=WORKDAY(B2, 10, C4:E4)"] + assert i["w1"] == ["=WORKDAY(B2, 10, DATE(2024, 1, 16))"] + + def test_a_call_over_a_series_is_one_call_per_period(self, tmp_path): + m = _model() + with m: + m.x.d = DATEDIF(m.i.start, m.x.ends, "D").set_display_name("d") + m.x.wd = WEEKDAY(m.x.ends, 2).set_display_name("wd") + m.x.wk = WEEKNUM(m.x.ends).set_display_name("wk") + m.x.n = NETWORKDAYS(m.i.start, m.x.ends, m.i.hol).set_display_name("n") + m.x.w = WORKDAY(m.x.ends, -5, m.i.hol).set_display_name("w") + x = _values(_book(m, tmp_path)["X"]) + assert x["d"] == ['=DATEDIF(I!B2, B2, "D")', '=DATEDIF(I!B2, C2, "D")', + '=DATEDIF(I!B2, D2, "D")'] + assert x["wd"] == ["=WEEKDAY(B2, 2)", "=WEEKDAY(C2, 2)", "=WEEKDAY(D2, 2)"] + assert x["wk"] == ["=WEEKNUM(B2)", "=WEEKNUM(C2)", "=WEEKNUM(D2)"] + assert x["n"] == ["=NETWORKDAYS(I!B2, B2, I!C4:E4)", "=NETWORKDAYS(I!B2, C2, I!C4:E4)", + "=NETWORKDAYS(I!B2, D2, I!C4:E4)"] + assert x["w"] == ["=WORKDAY(B2, -5, I!C4:E4)", "=WORKDAY(C2, -5, I!C4:E4)", + "=WORKDAY(D2, -5, I!C4:E4)"] + + def test_a_workday_cell_shows_as_a_date_and_a_count_as_a_number(self, tmp_path): + m = _model() + with m: + m.i.w = WORKDAY(m.i.start, 10, m.i.hol).set_display_name("w") + m.i.n = NETWORKDAYS(m.i.start, m.i.end, m.i.hol).set_display_name("n") + m.x.w = WORKDAY(m.x.ends, -5).set_display_name("w") + m.x.d = DATEDIF(m.i.start, m.x.ends, "D").set_display_name("d") + book = _book(m, tmp_path) + i, x = _cells(book["I"]), _cells(book["X"]) + assert [c.number_format for c in i["w"]] == ["yyyy-mm-dd"] + assert [c.number_format for c in x["w"]] == ["yyyy-mm-dd"] * 3 + assert "yy" not in i["n"][0].number_format + assert all("yy" not in c.number_format for c in x["d"]) + + def test_the_names_are_stored_bare(self, tmp_path): + m = _model() + with m: + m.i.a = DATEDIF(m.i.start, m.i.end, "Y").set_display_name("a") + m.i.b = WEEKDAY(m.i.start).set_display_name("b") + m.i.c = WEEKNUM(m.i.start).set_display_name("c") + m.i.d = NETWORKDAYS(m.i.start, m.i.end).set_display_name("d") + m.i.e = WORKDAY(m.i.start, 1).set_display_name("e") + i = _values(_book(m, tmp_path)["I"]) + for label in "abcde": + (formula,) = i[label] + assert formula.startswith("=") and "_xlfn." not in formula, formula + + def test_a_shorter_holidays_line_is_written_as_its_own_cells(self, tmp_path): + m = _model() + with m: + m.i.hol2 = mo.Variable([date(2024, 1, 1), date(2024, 2, 5)], extend=mo.none(), + display_name="hol2") + m.i.n2 = NETWORKDAYS(m.i.start, m.i.end, m.i.hol2).set_display_name("n2") + m.x.n2 = NETWORKDAYS(m.i.start, m.x.ends, m.i.hol2).set_display_name("n2") + book = _book(m, tmp_path) + i, x = _values(book["I"]), _values(book["X"]) + assert i["hol2"] == [datetime(2024, 1, 1), datetime(2024, 2, 5)] + assert i["n2"] == ["=NETWORKDAYS(B2, B3, C5:D5)"] + assert x["n2"] == ["=NETWORKDAYS(I!B2, B2, I!C5:D5)", "=NETWORKDAYS(I!B2, C2, I!C5:D5)", + "=NETWORKDAYS(I!B2, D2, I!C5:D5)"] + + def test_a_plain_list_of_holidays_has_no_cells_so_the_value_is_written(self, tmp_path): + m = _model() + with m: + m.x.n = NETWORKDAYS(m.i.start, m.x.ends, [date(2024, 1, 16)]).set_display_name("n") + assert m.x.n.value == [12, 33, 54] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + x = _values(_book(m, tmp_path)["X"]) + assert x["n"] == [12, 33, 54] + assert any(issubclass(w.category, CrossScopeReferenceWarning) + and "NETWORKDAYS" in str(w.message) for w in caught) + + +# ─── LibreOffice recalculates the workbook ──────────────────────────── + + +def _soffice(): + found = shutil.which("soffice") or shutil.which("libreoffice") + if found: + return found + mac = Path("/Applications/LibreOffice.app/Contents/MacOS/soffice") + return str(mac) if mac.exists() else None + + +def _recalculated(model, tmp_path): + """Write ``model``, let LibreOffice calculate every formula, and + return the recalculated workbook's stored values.""" + soffice = _soffice() + if soffice is None: + pytest.skip("LibreOffice is not installed") + src = tmp_path / "book.xlsx" + model.to_excel(str(src)) + outdir = tmp_path / "calc" + profile = (tmp_path / "lo-profile").as_uri() # private: parallel runs don't clash + subprocess.run( + [soffice, f"-env:UserInstallation={profile}", "--headless", + "--convert-to", "xlsx", "--outdir", str(outdir), str(src)], + check=True, capture_output=True, timeout=180, + ) + return load_workbook(str(outdir / "book.xlsx"), data_only=True) + + +def _as_date(value): + return value.date() if isinstance(value, datetime) else value + + +def _parity_model(): + m = _model() + with m: + m.i.s = mo.Variable(date(2001, 6, 1), display_name="s") + m.i.e = mo.Variable(date(2002, 8, 15), display_name="e") + for unit in ("Y", "M", "D", "MD", "YM", "YD"): + setattr(m.i, f"dif_{unit.lower()}", + DATEDIF(m.i.s, m.i.e, unit).set_display_name(f"dif_{unit}")) + m.i.jan31 = mo.Variable(date(2016, 1, 31), display_name="jan31") + m.i.mar1 = mo.Variable(date(2016, 3, 1), display_name="mar1") + m.i.md_neg = DATEDIF(m.i.jan31, m.i.mar1, "MD").set_display_name("md_neg") + m.i.months = DATEDIF(m.i.start, m.i.end, "M").set_display_name("months") + m.i.wd = WEEKDAY(m.i.start).set_display_name("wd") + m.i.wd3 = WEEKDAY(m.i.start, 3).set_display_name("wd3") + m.i.wd16 = WEEKDAY(m.i.end, 16).set_display_name("wd16") + m.i.wk = WEEKNUM(m.i.end, 21).set_display_name("wk") + m.i.wk2 = WEEKNUM(m.i.end, 2).set_display_name("wk2") + m.i.n = NETWORKDAYS(m.i.start, m.i.end, m.i.hol).set_display_name("n") + m.i.n0 = NETWORKDAYS(m.i.start, m.i.end).set_display_name("n0") + m.i.nrev = NETWORKDAYS(m.i.end, m.i.start, m.i.hol).set_display_name("nrev") + m.i.hol2 = mo.Variable([date(2024, 1, 1), date(2024, 2, 5)], extend=mo.none(), + display_name="hol2") + m.i.n2 = NETWORKDAYS(m.i.start, m.i.end, m.i.hol2).set_display_name("n2") + m.i.w = WORKDAY(m.i.start, 10, m.i.hol).set_display_name("w") + m.i.w0 = WORKDAY(m.i.start, 0).set_display_name("w0") + m.i.wback = WORKDAY(m.i.end, -12, m.i.hol).set_display_name("wback") + m.x.d = DATEDIF(m.i.start, m.x.ends, "D").set_display_name("d") + # No whole year from s2 to the ends: Excel's and LibreOffice's "YD" + # count the same window; the one that straddles a 29 February is + # the expected failure below. + m.i.s2 = mo.Variable(date(2023, 6, 1), display_name="s2") + m.x.yd = DATEDIF(m.i.s2, m.x.ends, "YD").set_display_name("yd") + m.x.wd = WEEKDAY(m.x.ends, 2).set_display_name("wd") + m.x.wd14 = WEEKDAY(m.x.ends, 14).set_display_name("wd14") + m.x.wk = WEEKNUM(m.x.ends).set_display_name("wk") + m.x.wk11 = WEEKNUM(m.x.ends, 11).set_display_name("wk11") + m.x.wk17 = WEEKNUM(m.x.ends, 17).set_display_name("wk17") + m.x.n = NETWORKDAYS(m.i.start, m.x.ends, m.i.hol).set_display_name("n") + m.x.w = WORKDAY(m.x.ends, -5, m.i.hol).set_display_name("w") + return m + + +@pytest.mark.slow +def test_the_workbook_recalculates_to_the_python_values(tmp_path): + m = _parity_model() + # The engine's own numbers first, so a parity failure is LibreOffice's. + assert [getattr(m.i, f"dif_{u}").value for u in ("y", "m", "d", "md", "ym", "yd")] == \ + [1, 14, 440, 14, 2, 75] + assert m.i.md_neg.value == -1 + assert (m.i.n.value, m.i.n0.value, m.i.nrev.value, m.i.n2.value) == (46, 48, -46, 47) + assert (m.i.w.value, m.i.w0.value, m.i.wback.value) == \ + (date(2024, 1, 29), date(2024, 1, 15), date(2024, 3, 1)) + assert m.x.n.value == [13, 33, 53] + assert m.x.yd.value == [244, 273, 304] + assert (m.x.wd14.value, m.x.wk11.value, m.x.wk17.value) == ([7, 1, 4], [5, 9, 13], [5, 9, 14]) + book = _recalculated(m, tmp_path) + i, x = _values(book["I"]), _values(book["X"]) + for label in ("dif_Y", "dif_M", "dif_D", "dif_MD", "dif_YM", "dif_YD", "md_neg", "months", + "wd", "wd3", "wd16", "wk", "wk2", "n", "n0", "nrev", "n2"): + assert i[label] == [getattr(m.i, label.lower()).value], label + for label in ("w", "w0", "wback"): + assert [_as_date(v) for v in i[label]] == [getattr(m.i, label).value], label + for label in ("d", "yd", "wd", "wd14", "wk", "wk11", "wk17", "n"): + assert x[label] == getattr(m.x, label).value, label + assert [_as_date(v) for v in x["w"]] == m.x.w.value + + +@pytest.mark.slow +@pytest.mark.xfail( + strict=True, + reason="LibreOffice's DATEDIF \"YD\" places the start's month and day in the end's " + "year and counts 324 days here; Excel places the end's month and day in the " + "start's year and returns 323, which the engine computes (the difference is " + "noted in LibreOffice's own source, sc/source/core/tool/interpr2.cxx).", +) +def test_datedif_yd_across_a_leap_day_is_excel_s_not_libreoffice_s(tmp_path): + m = mo.Model("m") + m.s = mo.Variable(date(1988, 6, 22), display_name="s") + m.e = mo.Variable(date(2012, 5, 11), display_name="e") + m.yd = DATEDIF(m.s, m.e, "YD").set_display_name("yd") + assert m.yd.value == 323 + ws = _recalculated(m, tmp_path).active + assert _values(ws)["yd"] == [323] + + +@pytest.mark.slow +@pytest.mark.xfail( + strict=True, + reason="LibreOffice's WEEKNUM in every mode but 21 (tools::Date::GetWeekOfYear with one " + "minimum day in a week) numbers a date in the last partial week of a year, the one " + "that runs into the next year, week 1 of that next year, and turns a week 54 into 1; " + "Excel numbers the week holding 1 January 1 and lets the last days of a year reach 53 " + "or 54 (31 December 2020 is 53, 31 December 2000 is 54), which the engine computes.", +) +def test_weeknum_s_last_days_of_a_year_are_excel_s_not_libreoffice_s(tmp_path): + m = mo.Model("m") + m.d = mo.Variable(date(2020, 12, 31), display_name="d") + m.e = mo.Variable(date(2000, 12, 31), display_name="e") + m.wk = WEEKNUM(m.d).set_display_name("wk") + m.wk2 = WEEKNUM(m.d, 2).set_display_name("wk2") + m.wk54 = WEEKNUM(m.e).set_display_name("wk54") + assert (m.wk.value, m.wk2.value, m.wk54.value) == (53, 53, 54) + ws = _recalculated(m, tmp_path).active + values = _values(ws) + assert (values["wk"], values["wk2"], values["wk54"]) == ([53], [53], [54]) + + +@pytest.mark.slow +@pytest.mark.xfail( + strict=True, + reason="A DATEDIF with the start after the end is #NUM! in Excel, which the engine " + "computes; LibreOffice recalculates it to Err:502 and saves the cell as #VALUE!.", +) +def test_datedif_with_the_start_after_the_end_is_num_in_excel_not_value(tmp_path): + m = mo.Model("m") + m.s = mo.Variable(date(2002, 8, 15), display_name="s") + m.e = mo.Variable(date(2001, 6, 1), display_name="e") + m.d = DATEDIF(m.s, m.e, "D").set_display_name("d") + assert m.d.value == "#NUM!" + ws = _recalculated(m, tmp_path).active + assert _values(ws)["d"] == ["#NUM!"] diff --git a/tests/test_excel_financial.py b/tests/test_excel_financial.py new file mode 100644 index 0000000..6ef7f53 --- /dev/null +++ b/tests/test_excel_financial.py @@ -0,0 +1,331 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The workbook writes the loan schedule and depreciation functions as +live formulas — ``=IPMT(I!B1, 2, I!B3, I!B2)`` in every column of a +schedule row, an optional argument only when given — and LibreOffice +recalculates them to the engine's numbers. +""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +import pytest +from openpyxl import load_workbook + +import modeleon as mo +from modeleon.functions.financial import ( + CUMIPMT, + CUMPRINC, + DB, + DDB, + IPMT, + NPER, + PPMT, + PV, + RATE, + SLN, + SYD, +) + + +def _rows(ws): + out = {} + for row in ws.iter_rows(): + label = row[0].value + if label: + out[str(label)] = [c.value for c in row[1:]] + return out + + +def _book(model, tmp_path): + path = tmp_path / "book.xlsx" + model.to_excel(path) + book = load_workbook(path) + return {name: _rows(book[name]) for name in book.sheetnames} + + +def _model(periods=4): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=periods) + with m: + m.i = mo.MultiVariable("I") + m.i.rate = mo.Variable(0.1, display_name="rate") + m.i.loan = mo.Variable(8000.0, display_name="loan") + m.i.term = mo.Variable(periods, display_name="term") + m.i.pmt = mo.Variable(-200.0, display_name="pmt") + m.i.cost = mo.Variable(2400.0, display_name="cost") + m.i.salvage = mo.Variable(300.0, display_name="salvage") + m.s = mo.MultiVariable("S") + return m + + +class TestTheFormulasWritten: + def test_a_scalar_call_is_one_cell_with_the_optional_arguments_given(self, tmp_path): + m = _model() + with m: + m.i.n = NPER(m.i.rate, m.i.pmt, m.i.loan).set_display_name("n") + m.i.n1 = NPER(m.i.rate, m.i.pmt, m.i.loan, when=1).set_display_name("n1") + m.i.r = RATE(m.i.term, m.i.pmt, m.i.loan).set_display_name("r") + m.i.rg = RATE(m.i.term, m.i.pmt, m.i.loan, guess=0.05).set_display_name("rg") + m.i.ci = CUMIPMT(m.i.rate, m.i.term, m.i.loan, 1, 2).set_display_name("ci") + m.i.cp = CUMPRINC(m.i.rate, m.i.term, m.i.loan, 1, 2, 1).set_display_name("cp") + m.i.sl = SLN(m.i.cost, m.i.salvage, m.i.term).set_display_name("sl") + m.i.i2 = IPMT(m.i.rate, 2, m.i.term, m.i.loan, 0, 1).set_display_name("i2") + m.i.p2 = PPMT(m.i.rate, 2, m.i.term, m.i.loan, 1000.0).set_display_name("p2") + i = _book(m, tmp_path)["I"] + assert i["n"] == ["=NPER(B1, B4, B2)"], i["n"] + assert i["n1"] == ["=NPER(B1, B4, B2, 0, 1)"], i["n1"] + assert i["r"] == ["=RATE(B3, B4, B2)"], i["r"] + assert i["rg"] == ["=RATE(B3, B4, B2, 0, 0, 0.05)"], i["rg"] + assert i["ci"] == ["=CUMIPMT(B1, B3, B2, 1, 2, 0)"], i["ci"] + assert i["cp"] == ["=CUMPRINC(B1, B3, B2, 1, 2, 1)"], i["cp"] + assert i["sl"] == ["=SLN(B5, B6, B3)"], i["sl"] + assert i["i2"] == ["=IPMT(B1, 2, B3, B2, 0, 1)"], i["i2"] + assert i["p2"] == ["=PPMT(B1, 2, B3, B2, 1000.0)"], i["p2"] + + def test_a_schedule_row_is_one_call_per_column_with_its_period(self, tmp_path): + m = _model() + with m: + m.s.interest = IPMT(m.i.rate, mo.time.index, m.i.term, m.i.loan).set_display_name("i") + m.s.principal = PPMT(m.i.rate, mo.time.index, m.i.term, m.i.loan).set_display_name("p") + m.s.db = DB(m.i.cost, m.i.salvage, m.i.term, mo.time.index, 7).set_display_name("db") + m.s.ddb = DDB(m.i.cost, m.i.salvage, m.i.term, mo.time.index).set_display_name("ddb") + m.s.ddb15 = DDB(m.i.cost, m.i.salvage, m.i.term, mo.time.index, + 1.5).set_display_name("ddb15") + m.s.syd = SYD(m.i.cost, m.i.salvage, m.i.term, mo.time.index).set_display_name("syd") + m.s.lit = IPMT(0.1, [1, 2, 3, 4], 4, 8000.0).set_display_name("lit") + s = _book(m, tmp_path)["S"] + assert s["i"] == [f"=IPMT(I!B1, {p}, I!B3, I!B2)" for p in (1, 2, 3, 4)], s["i"] + assert s["p"] == [f"=PPMT(I!B1, {p}, I!B3, I!B2)" for p in (1, 2, 3, 4)], s["p"] + assert s["db"] == [f"=DB(I!B5, I!B6, I!B3, {p}, 7)" for p in (1, 2, 3, 4)], s["db"] + assert s["ddb"] == [f"=DDB(I!B5, I!B6, I!B3, {p})" for p in (1, 2, 3, 4)], s["ddb"] + assert s["ddb15"] == [f"=DDB(I!B5, I!B6, I!B3, {p}, 1.5)" for p in (1, 2, 3, 4)], s["ddb15"] + assert s["syd"] == [f"=SYD(I!B5, I!B6, I!B3, {p})" for p in (1, 2, 3, 4)], s["syd"] + assert s["lit"] == [f"=IPMT(0.1, {p}, 4, 8000.0)" for p in (1, 2, 3, 4)], s["lit"] + + def test_the_cached_values_are_the_engine_s(self, tmp_path): + m = _model() + with m: + m.s.interest = IPMT(m.i.rate, mo.time.index, m.i.term, m.i.loan).set_display_name("i") + m.i.r = RATE(m.i.term, m.i.pmt, m.i.loan).set_display_name("r") + path = tmp_path / "book.xlsx" + m.to_excel(path) + book = load_workbook(path, data_only=True) + assert _rows(book["S"])["i"][:4] == [pytest.approx(v) for v in m.s.interest.value] + assert _rows(book["I"])["r"][0] == pytest.approx(m.i.r.value) + + +# ══════════════════════════════════════════════════════════════════════ +# The spreadsheet computes the same numbers +# ══════════════════════════════════════════════════════════════════════ + + +def _soffice(): + for candidate in ("soffice", "libreoffice"): + found = shutil.which(candidate) + if found: + return found + mac = Path("/Applications/LibreOffice.app/Contents/MacOS/soffice") + return str(mac) if mac.exists() else None + + +def _parity_model(): + """Every loan schedule and depreciation function, several argument sets each: the + documented examples, a payment at the start of the period (type 1), a + zero rate and a rate of -100%, a future value, a guess, fractional + periods, the rates LibreOffice finds only from its further starts, + lives under two periods, a rate and periods read as a cell shows them, + numeric text, schedule rows over the model's periods, the cells that + are errors on both sides, and the cells LibreOffice reads its own way.""" + m = _model(periods=5) + with m: + i, s = m.i, m.s + i.zero = mo.Variable(0.0, display_name="zero") + i.r01 = mo.Variable(0.01, display_name="r01") + i.pmt100 = mo.Variable(-100.0, display_name="pmt100") + i.pv1000 = mo.Variable(-1000.0, display_name="pv1000") + i.fv10000 = mo.Variable(10000.0, display_name="fv10000") + i.monthly = mo.Variable(0.09 / 12, display_name="monthly") + i.n360 = mo.Variable(360, display_name="n360") + i.big = mo.Variable(125000.0, display_name="big") + i.months = mo.Variable(7, display_name="months") + i.life4 = mo.Variable(4, display_name="life4") + i.loan_text = mo.Variable("8000", display_name="loan_text") + # NPER + i.n_t1 = NPER(i.r01, i.pmt100, i.pv1000, i.fv10000, 1).set_display_name("n_t1") + i.n_t0 = NPER(i.r01, i.pmt100, i.pv1000, i.fv10000).set_display_name("n_t0") + i.n_nofv = NPER(i.r01, i.pmt100, i.pv1000).set_display_name("n_nofv") + i.n_zero = NPER(i.zero, i.pmt100, -i.pv1000).set_display_name("n_zero") + # IPMT / PPMT, one cell each + i.i_t1 = IPMT(i.rate, 2, 3, i.loan, 0, 1).set_display_name("i_t1") + i.i_fv = IPMT(i.rate, 2, 3, i.loan, 1000.0).set_display_name("i_fv") + i.i_zero = IPMT(i.zero, 2, 10, i.loan).set_display_name("i_zero") + i.i_frac = IPMT(i.rate, 2.5, 3, i.loan).set_display_name("i_frac") + i.p_t1 = PPMT(i.rate, 2, 3, i.loan, 0, 1).set_display_name("p_t1") + i.p_zero = PPMT(i.zero, 2, 10, i.loan).set_display_name("p_zero") + # A rate of -100%, and a first period on a term too long for the + # payment: computed as the spreadsheet computes them. + i.i_m1 = IPMT(-1, 1, 3, i.loan).set_display_name("i_m1") + i.p_m1 = PPMT(-1, 1, 3, i.loan).set_display_name("p_m1") + i.p_m1b = PPMT(-1, 2, 3, i.loan).set_display_name("p_m1b") + i.i_long = IPMT(i.rate, 1, 10000, i.loan).set_display_name("i_long") + # A small rate: (1 + rate) ** nper - 1 loses no digits. + i.p_small = PPMT(1e-12, 2, 10, 1000.0).set_display_name("p_small") + i.cp_small = CUMPRINC(1e-9, 10, 1000.0, 1, 10, 0).set_display_name("cp_small") + # RATE + i.r_a = RATE(48, i.pmt, i.loan).set_display_name("r_a") + i.r_t1 = RATE(48, i.pmt, i.loan, 0, 1).set_display_name("r_t1") + i.r_fv = RATE(10, i.pmt100, 0, 1500.0).set_display_name("r_fv") + i.r_g = RATE(48, i.pmt, i.loan, 0, 0, 0.05).set_display_name("r_g") + i.r_frac = RATE(10.5, i.pmt100, 800.0).set_display_name("r_frac") + i.r_neg = RATE(5, 50.0, i.pv1000).set_display_name("r_neg") + # Newton from 10% runs below -100% on a loan repaid in a few + # payments its own size, and the spreadsheet starts again at 20%, + # 5%, 30%, ...; a long term settles inside the 150 steps. + i.r_high = RATE(5, -1000.0, 1000.0).set_display_name("r_high") + i.r_twelve = RATE(12, -1000.0, 1000.0).set_display_name("r_twelve") + i.r_long0 = RATE(1000, -1.0, 1000.0).set_display_name("r_long0") + i.r_long = RATE(1200, -1.0, 1000.0).set_display_name("r_long") + i.r_due = RATE(3, -600.0, 1000.0, 0, 1).set_display_name("r_due") + i.r_frac_high = RATE(5.5, -1000.0, 1000.0).set_display_name("r_frac_high") + i.r_hardly = RATE(2, -1.0, 1000.0).set_display_name("r_hardly") + # No flow at all: every rate balances, the guess is the answer. + i.r_nothing = RATE(10, i.zero, i.zero).set_display_name("r_nothing") + # Numeric text, written as a literal and held in a cell. + i.r_text = RATE(48, i.pmt, "8000").set_display_name("r_text") + i.r_text_cell = RATE(48, i.pmt, i.loan_text).set_display_name("r_text_cell") + # CUMIPMT / CUMPRINC + i.ci = CUMIPMT(i.monthly, i.n360, i.big, 13, 24, 0).set_display_name("ci") + i.ci_t1 = CUMIPMT(i.monthly, i.n360, i.big, 1, 12, 1).set_display_name("ci_t1") + i.ci_frac = CUMIPMT(i.monthly, i.n360, i.big, 13.7, 24.2, 0).set_display_name("ci_frac") + i.cp = CUMPRINC(i.monthly, i.n360, i.big, 13, 24, 0).set_display_name("cp") + i.cp_t1 = CUMPRINC(i.monthly, i.n360, i.big, 1, 12, 1).set_display_name("cp_t1") + # 12.999999999999998 shows as 13: the count starts at period 13. + i.ci_shown = CUMIPMT(i.monthly, i.n360, i.big, 12.999999999999998, 24, + 0).set_display_name("ci_shown") + i.cp_shown = CUMPRINC(i.monthly, i.n360, i.big, 12.999999999999998, 24, + 0).set_display_name("cp_shown") + # SLN + i.sl = SLN(i.cost, i.salvage, i.term).set_display_name("sl") + # DB: a period that reads as 1 is the first period's charge, even + # past a life under two periods; the rate, a period and a month + # are read as a cell shows them. + i.db_short = DB(1000.0, 100.0, 0.5, 1).set_display_name("db_short") + i.db_short15 = DB(1000.0, 100.0, 0.5, 1.5, 6).set_display_name("db_short15") + i.db_life09 = DB(1000.0, 100.0, 0.9, 1, 6).set_display_name("db_life09") + i.db_life1 = DB(1000.0, 100.0, 1, 1.5, 6).set_display_name("db_life1") + i.db_life12 = DB(1000.0, 100.0, 1.2, 1.5, 6).set_display_name("db_life12") + i.db_stub12 = DB(1000.0, 100.0, 1.2, 2, 6).set_display_name("db_stub12") + i.db_round = DB(i.cost, 318.0, 1, 1).set_display_name("db_round") + i.db_round7 = DB(i.cost, 318.0, 1, 2, i.months).set_display_name("db_round7") + i.db_shown = DB(1000.0, 100.0, 5, 2.9999999999999996).set_display_name("db_shown") + i.db_month_shown = DB(1000.0, 100.0, 5, 1, + 2.9999999999999996).set_display_name("db_month_shown") + # Schedule rows over the five periods: a five-year loan, a + # four-year asset whose fifth period is DB's stub, and one of + # four and a half years. + s.interest = IPMT(i.rate, mo.time.index, i.term, i.loan).set_display_name("interest") + s.principal = PPMT(i.rate, mo.time.index, i.term, i.loan).set_display_name("principal") + s.interest_t1 = IPMT(i.rate, mo.time.index, i.term, i.loan, 0, + 1).set_display_name("interest_t1") + s.principal_t1 = PPMT(i.rate, mo.time.index, i.term, i.loan, 0, + 1).set_display_name("principal_t1") + s.interest_zero = IPMT(i.zero, mo.time.index, i.term, + i.loan).set_display_name("interest_zero") + s.principal_zero = PPMT(i.zero, mo.time.index, i.term, + i.loan).set_display_name("principal_zero") + s.nper_row = NPER(i.rate, i.pmt100, mo.time.index * 100.0).set_display_name("nper_row") + s.rate_row = RATE(mo.time.index + 5, i.pmt100, 800.0).set_display_name("rate_row") + s.db = DB(i.cost, i.salvage, i.life4, mo.time.index, i.months).set_display_name("db") + s.db12 = DB(i.cost, i.salvage, i.life4, mo.time.index).set_display_name("db12") + s.db_frac = DB(i.cost, i.salvage, 4.5, mo.time.index, i.months).set_display_name("db_frac") + s.ddb = DDB(i.cost, i.salvage, i.term, mo.time.index).set_display_name("ddb") + s.ddb15 = DDB(i.cost, i.salvage, i.term, mo.time.index, 1.5).set_display_name("ddb15") + s.syd = SYD(i.cost, i.salvage, i.term, mo.time.index).set_display_name("syd") + # Errors on both sides: a period past the term, a rate with no + # root, an iteration that runs off from the guess given or from + # every start. + i.i_past = IPMT(i.rate, 6, i.term, i.loan).set_display_name("i_past") + i.r_none = RATE(10, 100.0, 1000.0).set_display_name("r_none") + i.r_poor = RATE(10, i.pmt100, 800.0, 0, 0, -0.5).set_display_name("r_poor") + i.r_runaway = RATE(2, -5000.0, 1000.0).set_display_name("r_runaway") + # The cells LibreOffice reads its own way (DIFFERS below). + i.ci_t09 = CUMIPMT(i.monthly, i.n360, i.big, 1, 12, 0.9).set_display_name("ci_t09") + i.syd_past = SYD(i.cost, i.salvage, i.life4, 5).set_display_name("syd_past") + i.db_half = DB(i.cost, i.salvage, i.life4, 0.5).set_display_name("db_half") + i.r_trap = RATE(120, i.pmt100, 1000.0, 0, 1).set_display_name("r_trap") + i.r_flat = RATE(10, i.zero, 1000.0).set_display_name("r_flat") + return m + + +SCALARS = ("n_t1", "n_t0", "n_nofv", "n_zero", "i_t1", "i_fv", "i_zero", "i_frac", "p_t1", + "p_zero", "i_m1", "p_m1", "p_m1b", "i_long", "p_small", "cp_small", "r_a", "r_t1", + "r_fv", "r_g", "r_frac", "r_neg", "r_high", "r_twelve", "r_long0", "r_long", + "r_due", "r_frac_high", "r_hardly", "r_nothing", "r_text", "r_text_cell", "ci", + "ci_t1", "ci_frac", "cp", "cp_t1", "ci_shown", "cp_shown", "sl", "db_short", + "db_short15", "db_life09", "db_life1", "db_life12", "db_stub12", "db_round", + "db_round7", "db_shown", "db_month_shown") +#: Errors in the engine and in LibreOffice, which spells them its own way: +#: Err:502 is written as #VALUE! for a period past the term, and a RATE +#: that does not converge as #N/A, where Excel shows #NUM!, the engine's +#: marker. +ERRORS = ("i_past", "r_none", "r_poor", "r_runaway") +#: Where LibreOffice departs from Excel's documentation, which the engine +#: follows, and what its cell holds instead. A fractional ``type`` in +#: CUMIPMT is truncated by Excel and refused by LibreOffice (#VALUE!). +#: SYD past the life and DB before period 1 are outside Excel's domain +#: (#NUM!), where LibreOffice computes the formula as written (0). RATE's +#: iteration ends, in LibreOffice, at -100% on an annuity due (the engine +#: goes on to the rate the payments carry) and at -98.2% on a loan with no +#: payment, which only -100% balances (the engine finds no rate). Each +#: pins the engine's reading and lets the spreadsheet refuse the cell, +#: agree, or hold its own value. +DIFFERS = {"ci_t09": None, "syd_past": 0, "db_half": 0, "r_trap": -1.0, + "r_flat": -0.981934476404913} +ROWS = ("interest", "principal", "interest_t1", "principal_t1", "interest_zero", + "principal_zero", "nper_row", "rate_row", "db", "db12", "db_frac", "ddb", "ddb15", "syd") + + +@pytest.mark.slow +def test_the_workbook_recalculates_to_the_python_values(tmp_path): + soffice = _soffice() + if soffice is None: + pytest.skip("LibreOffice not installed") + m = _parity_model() + path = tmp_path / "financial.xlsx" + m.to_excel(path) + subprocess.run( + [soffice, f"-env:UserInstallation={(tmp_path / 'prof').as_uri()}", + "--headless", "--convert-to", "xlsx", "--outdir", + str(tmp_path / "calc"), str(path)], + check=True, capture_output=True, + ) + book = load_workbook(tmp_path / "calc" / "financial.xlsx", data_only=True) + i = _rows(book["I"]) + s = _rows(book["S"]) + # The engine's numbers are not the spreadsheet's defaults: a wrong + # one would be caught here. + assert m.i.n_t1.value == pytest.approx(59.6738657, abs=5e-8) + assert m.s.db.value[4] == pytest.approx(DB(2400, 300, 4, 5, 7).value) and m.s.db.value[4] > 0 + assert m.i.db_short.value == 990.0 and m.i.db_round.value == pytest.approx(2083.2) + for name in SCALARS: + assert i[name][0] == pytest.approx(getattr(m.i, name).value, rel=1e-9, abs=1e-9), name + for name in ROWS: + got = s[name][:5] + assert got == pytest.approx(getattr(m.s, name).value, rel=1e-9, abs=1e-9), name + for name in ERRORS: + assert getattr(m.i, name).value == '#NUM!' and str(i[name][0]).startswith("#"), name + # The cells were recalculated, not handed back as the engine cached + # them: LibreOffice spells some of these errors its own way. + assert {str(i[name][0]) for name in ERRORS} != {'#NUM!'} + # The engine's own readings, exactly. + assert m.i.ci_t09.value == pytest.approx(CUMIPMT(0.09 / 12, 360, 125000.0, 1, 12, 0).value) + assert m.i.syd_past.value == m.i.db_half.value == m.i.r_flat.value == '#NUM!' + assert m.i.r_trap.value == pytest.approx(0.1111107124217, rel=1e-9) + assert PV(m.i.r_trap.value, 120, -100.0, 0, 1).value == pytest.approx(1000.0) + for name, own in DIFFERS.items(): + got, ours = i[name][0], getattr(m.i, name).value + allowed = [v for v in (ours, own) if isinstance(v, int | float)] + assert str(got).startswith("#") or any(got == pytest.approx(v) for v in allowed), \ + (name, got) diff --git a/tests/test_excel_house_look.py b/tests/test_excel_house_look.py index 3ddbf58..8f1e94a 100644 --- a/tests/test_excel_house_look.py +++ b/tests/test_excel_house_look.py @@ -540,3 +540,59 @@ def test_a_bar_rows_total_stays_a_number(tmp_path): assert totals["Active"].value.startswith("=SUM(") assert totals["Active"].number_format == "General" assert totals["Cash"].number_format == "#,##0" + + +# ══════════════════════════════════════════════════════════════════════ +# A named view, applied +# ══════════════════════════════════════════════════════════════════════ + + +def test_an_applied_named_view_dresses_the_book_and_ends_with_the_block(tmp_path): + m = deal(mo.ExcelView(font={"name": "Arial", "size": 10}, sheet={"gridlines": False})) + m.board = mo.ExcelView("Board pack", font={"name": "Georgia", "size": 11}) + # A named view on the model changes nothing until applied: the book is the model's own. + assert book(m, tmp_path)["Series"]["A1"].font.name == "Arial" + with mo.applied_view(m.board): + view = resolve_excel_view(m.td.volume) + assert view.base_font == {"name": "Georgia", "size": 11} # the applied view + assert view.sheet_gridlines is False # the model's, kept + wb = book(m, tmp_path) + assert wb["Series"]["A1"].font.name == "Georgia" + assert wb["Series"].sheet_view.showGridLines is False + # Outside the block the model's own look is back. + assert resolve_excel_view(m.td.volume).base_font == {"name": "Arial", "size": 10} + assert book(m, tmp_path)["Series"]["A1"].font.name == "Arial" + + +def test_an_applied_view_lies_over_the_house_and_the_model(tmp_path): + house = mo.ExcelView(font={"name": "Arial", "size": 10}, meta=LEAD) + m = deal(mo.ExcelView(font={"name": "Georgia", "size": 12})) + m.print = mo.ExcelView("Print", font={"name": "Verdana", "size": 9}) + with mo.house_view(house), mo.applied_view(m.print): + view = resolve_excel_view(m.ti.rate) + assert view.base_font == {"name": "Verdana", "size": 9} # the applied view + assert view.meta_fields == (("row_sum", "Total"),) # the house, kept + + +def test_an_applied_view_s_sheet_order_wins(tmp_path): + m = deal() + m.series_first = mo.ExcelView("Series first", sheet={"order": ["td", "ti", "timing"]}) + assert book(m, tmp_path).sheetnames == ["Inputs", "Series", "Timing"] + with mo.applied_view(m.series_first): + assert book(m, tmp_path).sheetnames == ["Series", "Inputs", "Timing"] + + +def test_an_applied_view_s_orient_is_not_a_look(): + m = deal() + m.side = mo.ExcelView("Side", orient="records", font={"name": "Georgia"}) + own = resolve_excel_view(m.td.volume).orient + with mo.applied_view(m.side): + view = resolve_excel_view(m.td.volume) + assert view.base_font == {"name": "Georgia"} + assert view.orient == own and view.orient != "records" + + +def test_applied_view_refuses_what_is_not_a_view(): + with pytest.raises(TypeError, match="ExcelView"): + with mo.applied_view({"font": {"name": "Georgia"}}): + pass diff --git a/tests/test_excel_logical.py b/tests/test_excel_logical.py new file mode 100644 index 0000000..c105c00 --- /dev/null +++ b/tests/test_excel_logical.py @@ -0,0 +1,246 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The workbook writes ``IFERROR``, ``ISERROR``, ``ISNUMBER``, ``XOR``, +``IFS`` and ``SWITCH`` as the spreadsheet's own functions — one call per +period over that period's cells, the names Excel added after 2007 stored +with the ``_xlfn.`` prefix — and, recalculated by LibreOffice, computes +the engine's values. +""" + +from __future__ import annotations + +import datetime as dt +import shutil +import subprocess +from pathlib import Path + +import pytest +from openpyxl import load_workbook + +import modeleon as mo +from modeleon.functions.logical import IFERROR, IFS, ISERROR, ISNUMBER, SWITCH, XOR + + +def _rows(ws): + out = {} + for row in ws.iter_rows(): + label = row[0].value + if label: + out[str(label)] = [c.value for c in row[1:]] + return out + + +def _model(): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with m: + m.i = mo.MultiVariable("I") + m.i.k = mo.Variable(7.0, display_name="k") + m.i.d0 = mo.Variable(dt.date(2026, 1, 1), display_name="d0") + m.x = mo.MultiVariable("X") + m.x.a = mo.Variable([3.0, 8.0, 5.0], display_name="a") + m.x.z = mo.Variable([1.0, 0.0, 2.0], display_name="z") + m.x.q = (m.x.a / m.x.z).set_display_name("q") + m.x.blank = mo.Variable([None, 5.0, None], display_name="blank") + m.x.words = mo.Variable(["abc", "", "zz"], display_name="words") + m.x.names = mo.Variable(["Base", "bull", "x"], display_name="names") + m.x.flags = mo.Variable([True, False, True], display_name="flags") + m.x.nums = mo.Variable([0, 2, -1], display_name="nums") + return m + + +def _add(container, **rows): + for name, var in rows.items(): + setattr(container, name, var.set_display_name(name)) + + +class TestTheWorkbook: + def test_each_function_is_written_as_the_spreadsheet_s_own(self, tmp_path): + m = _model() + with m: + _add( + m.x, + iferr=IFERROR(m.x.q, 0.0), + iferr_txt=IFERROR(m.x.q, "n/a"), + iferr_row=IFERROR(m.x.q, [9.0, 8.0, 7.0]), + iserr=ISERROR(m.x.q), + isnum=ISNUMBER(m.x.a), + xor=XOR(m.x.a > 4.0, True), + ifs=IFS(m.x.a > 7.0, "big", m.x.a > 4.0, "mid", True, "small"), + sw=SWITCH(m.x.names, "base", 0.05, "bull", 0.08, 0.03), + nested=IFERROR(IFS(m.x.a > 100.0, 1.0), -1.0), + ) + _add( + m.i, + ifs_k=IFS(m.i.k > 5.0, "hi", True, "lo"), + sw_date=SWITCH(m.i.d0, 46023, "yes", "no"), + isnum_date=ISNUMBER(m.i.d0), + ) + path = tmp_path / "logical.xlsx" + m.to_excel(path) + book = load_workbook(path) + x = _rows(book["X"]) + i = _rows(book["I"]) + assert x["iferr"] == ["=IFERROR(B4, 0.0)", "=IFERROR(C4, 0.0)", "=IFERROR(D4, 0.0)"] + assert x["iferr_txt"][0] == '=IFERROR(B4, "n/a")' + # A per-period list literal is this period's number. + assert x["iferr_row"] == ["=IFERROR(B4, 9.0)", "=IFERROR(C4, 8.0)", "=IFERROR(D4, 7.0)"] + assert x["iserr"] == ["=ISERROR(B4)", "=ISERROR(C4)", "=ISERROR(D4)"] + assert x["isnum"][0] == "=ISNUMBER(B2)" + assert x["xor"] == ["=_xlfn.XOR(B2 > 4.0, TRUE)", "=_xlfn.XOR(C2 > 4.0, TRUE)", + "=_xlfn.XOR(D2 > 4.0, TRUE)"] + assert x["ifs"][0] == '=_xlfn.IFS(B2 > 7.0, "big", B2 > 4.0, "mid", TRUE, "small")' + assert x["ifs"][2] == '=_xlfn.IFS(D2 > 7.0, "big", D2 > 4.0, "mid", TRUE, "small")' + assert x["sw"][0] == '=_xlfn.SWITCH(B7, "base", 0.05, "bull", 0.08, 0.03)' + assert x["sw"][1] == '=_xlfn.SWITCH(C7, "base", 0.05, "bull", 0.08, 0.03)' + assert x["nested"][0] == "=IFERROR(_xlfn.IFS(B2 > 100.0, 1.0), -1.0)" + assert i["ifs_k"][0] == '=_xlfn.IFS(B1 > 5.0, "hi", TRUE, "lo")' + assert i["sw_date"][0] == '=_xlfn.SWITCH(B2, 46023, "yes", "no")' + assert i["isnum_date"][0] == "=ISNUMBER(B2)" + + def test_the_cells_carry_the_engine_s_values(self, tmp_path): + m = _model() + with m: + _add( + m.x, + iferr=IFERROR(m.x.q, 0.0), + iserr=ISERROR(m.x.q), + xor=XOR(m.x.a > 4.0, True), + ifs=IFS(m.x.a > 7.0, "big", m.x.a > 4.0, "mid", True, "small"), + ifs_na=IFS(m.x.a > 100.0, 1.0), + sw=SWITCH(m.x.names, "base", 0.05, "bull", 0.08, 0.03), + ) + path = tmp_path / "logical.xlsx" + m.to_excel(path) + x = _rows(load_workbook(path, data_only=True)["X"]) + assert x["iferr"] == [3.0, 0.0, 2.5] == m.x.iferr.value + assert x["iserr"] == [False, True, False] == m.x.iserr.value + assert x["xor"] == [True, False, False] == m.x.xor.value + assert x["ifs"] == ["small", "big", "mid"] == m.x.ifs.value + assert x["ifs_na"] == ["#N/A"] * 3 == m.x.ifs_na.value + assert x["sw"] == [0.05, 0.08, 0.03] == m.x.sw.value + + +def _soffice(): + for candidate in ("soffice", "libreoffice"): + found = shutil.which(candidate) + if found: + return found + mac = Path("/Applications/LibreOffice.app/Contents/MacOS/soffice") + return str(mac) if mac.exists() else None + + +def _agree(got, want) -> bool: + if got is None and want == "": + return True # a formula's empty text reads back as an empty cell + if isinstance(want, bool) or isinstance(got, bool): + return got is want + if isinstance(want, (int, float)) and isinstance(got, (int, float)): + return round(float(got), 9) == round(float(want), 9) + return got == want + + +@pytest.mark.slow +def test_the_workbook_recalculates_to_the_python_values(tmp_path): + """Every function of the module, in the corners Excel and LibreOffice + read alike: errors caught and named, blanks and text in conditions + (the text TRUE and FALSE among them), a lazy branch, the ``#N/A`` of + nothing matching, computed numbers matched to 15 significant digits, + text matched regardless of case, a date by its serial number, an + empty text typed into a line (an empty cell) and a formula's (text), + a per-period and a scalar call.""" + soffice = _soffice() + if soffice is None: + pytest.skip("LibreOffice not installed") + m = _model() + with m: + m.x.f = mo.Variable([0.1, 0.2, 0.3], display_name="f") + m.x.tf = mo.Variable(["TRUE", "FALSE", "true"], display_name="tf") + m.x.gaps = mo.Variable([True, "", False], display_name="gaps") + m.x.fempty = IFS(m.x.a > 100.0, "x", True, "").set_display_name("fempty") + m.i.big = mo.Variable(4503599627370496.0, display_name="big") + _add( + m.x, + iferr=IFERROR(m.x.q, 0.0), + iferr_txt=IFERROR(m.x.q, "n/a"), + iferr_blank=IFERROR(m.x.blank, "x"), + iferr_typed=IFERROR(m.x.words, "x"), + iferr_fempty=IFERROR(m.x.fempty, "x"), + iferr_fallback_blank=IFERROR(m.x.q, m.x.blank), + iferr_nested=IFERROR(IFS(m.x.a > 100.0, 1.0), -1.0), + iserr=ISERROR(m.x.q), + iserr_blank=ISERROR(m.x.blank), + isnum_q=ISNUMBER(m.x.q), + isnum_words=ISNUMBER(m.x.words), + isnum_blank=ISNUMBER(m.x.blank), + xor_tf=XOR(m.x.a > 4.0, True), + xor_three=XOR(m.x.a > 4.0, m.x.z > 0.0, True), + xor_blank=XOR(m.x.blank, True), + xor_gaps=XOR(m.x.gaps, True), + xor_fempty=XOR(m.x.fempty, True), + xor_num=XOR(m.x.nums), + xor_err=XOR(m.x.q > 1.0, m.x.flags), + xor_all_blank=XOR(m.x.blank), + ifs_band=IFS(m.x.a > 7.0, "big", m.x.a > 4.0, "mid", True, "small"), + ifs_na=IFS(m.x.a > 100.0, 1.0), + ifs_lazy=IFS(m.x.z == 0, 0.0, True, m.x.q), + ifs_err=IFS(m.x.q > 1.0, 1.0, True, 2.0), + ifs_cond_after=IFS(True, 1.0, m.x.q > 1.0, 2.0), + ifs_num=IFS(m.x.nums, "yes", True, "no"), + ifs_words=IFS(m.x.words, 1.0, True, 2.0), + ifs_tf=IFS(m.x.tf, 1.0, True, 2.0), + ifs_fempty=IFS(m.x.fempty, 1.0, True, 2.0), + ifs_blank=IFS(m.x.blank, 1.0, True, 2.0), + ifs_pick_blank=IFS(True, m.x.blank), + ifs_pick_typed=IFS(True, m.x.words), + ifs_flags=IFS(m.x.flags, m.x.a, True, 0.0), + sw_txt=SWITCH(m.x.names, "base", 0.05, "BULL", 0.08, 0.03), + sw_na=SWITCH(m.x.a, 3, "three"), + sw_num=SWITCH(m.x.a, 3, "three", 8, "eight", "other"), + sw_blank_txt=SWITCH(m.x.blank, "", "empty", "other"), + sw_typed=SWITCH(m.x.words, "", "empty", "other"), + sw_fempty=SWITCH(m.x.fempty, "", "empty", "other"), + sw_fempty0=SWITCH(m.x.fempty, 0, "zero", "other"), + sw_money=SWITCH(m.x.f * 3, 0.3, "p3", 0.6, "p6", 0.9, "p9", "other"), + sw_rate=SWITCH(m.x.f + 0.2, 0.3, "a", 0.4, "b", 0.5, "c", "other"), + sw_apart=SWITCH(m.x.f + 1e-12, 0.1, "eq", 0.2, "eq", 0.3, "eq", "apart"), + sw_err=SWITCH(m.x.q, 3, "three", "other"), + sw_pick_blank=SWITCH(m.x.a, 3, m.x.blank, 8, m.x.blank, "other"), + sw_series=SWITCH(2, 1, m.x.a, 2, m.x.z), + ) + _add( + m.i, + ifs_k=IFS(m.i.k > 5.0, "hi", True, "lo"), + sw_k=SWITCH(m.i.k, 7, "seven", 8, "eight"), + sw_date=SWITCH(m.i.d0, 46023, "yes", "no"), + sw_whole=SWITCH(m.i.big, 4503599627370497, "eq", "ne"), + isnum_date=ISNUMBER(m.i.d0), + ifs_true_lit=IFS("TRUE", 1.0, True, 2.0), + ifs_false_lit=IFS("False", 1.0, True, 2.0), + ) + assert m.x.ifs_words.value == ["#VALUE!", 2.0, "#VALUE!"] + assert m.x.sw_err.value == ["three", "#DIV/0!", "other"] + assert m.x.ifs_tf.value == [1.0, 2.0, 1.0] and m.x.ifs_fempty.value == ["#VALUE!"] * 3 + assert m.x.iferr_typed.value == ["abc", 0, "zz"] == m.x.ifs_pick_typed.value + assert m.x.iferr_fempty.value == ["", "", ""] and m.x.xor_gaps.value == [False, True, True] + assert m.x.sw_money.value == ["p3", "p6", "p9"] and m.x.sw_rate.value == ["a", "b", "c"] + assert m.x.sw_apart.value == ["apart"] * 3 and m.i.sw_whole.value == "ne" + assert m.x.sw_typed.value == ["other", "empty", "other"] + assert m.x.sw_fempty0.value == ["other"] * 3 + path = tmp_path / "logical.xlsx" + m.to_excel(path) + subprocess.run( + [soffice, f"-env:UserInstallation={(tmp_path / 'prof').as_uri()}", + "--headless", "--convert-to", "xlsx", "--outdir", + str(tmp_path / "calc"), str(path)], + check=True, capture_output=True, + ) + book = load_workbook(tmp_path / "calc" / "logical.xlsx", data_only=True) + for sheet, container in (("X", m.x), ("I", m.i)): + rows = _rows(book[sheet]) + for name, var in container._components.items(): + if not isinstance(var, mo.Variable) or var._expr is None: + continue + want = var.value if isinstance(var.value, list) else [var.value] + got = rows[var.display_name][:len(want)] + assert len(got) == len(want) and all( + _agree(g, w) for g, w in zip(got, want) + ), f"{sheet}.{name}: LibreOffice {got!r}, the engine {want!r}" diff --git a/tests/test_excel_lookup.py b/tests/test_excel_lookup.py new file mode 100644 index 0000000..7f353b5 --- /dev/null +++ b/tests/test_excel_lookup.py @@ -0,0 +1,472 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Lookup and reference in the workbook. + +A row is its cells' range (``Inputs!C2:E2``); a list of scalar lines +whose cells stand one under another is a vertical range (``Deals!B3:B5``); +a table of records is one block, the records' labels in its first +column and their scalar fields after (``Deals!A3:C5``); a list of rows +laid out one under another is one block too (``Inputs!C2:E3``, the +period block ``Deals!D8:F10``). A series key writes one call per +period over that period's key cell and the whole range. Cells that form +no rectangle — the keys typed into a lookup over a keyed line, records +laid out as sections, a unit column between a record's label and its +fields, rows on two sheets — cannot be one range, so the cell carries +its computed value and ``check_compat`` lists it. ``XLOOKUP`` is stored +as ``_xlfn.XLOOKUP``; a mode without ``if_not_found`` is written after +``NA()``. Every function recalculates in LibreOffice to the engine's +numbers, except where LibreOffice departs from Excel (a boolean read as +a number, text that reads as a number matching the number, a column or +a row number past the block), pinned as an expected failure. +""" + +import datetime as dt +import shutil +import subprocess +from pathlib import Path + +import pytest +from openpyxl import load_workbook + +import modeleon as mo +from modeleon.compile import check_compat +from modeleon.functions.lookup import HLOOKUP, INDEX, MATCH, VLOOKUP, XLOOKUP + + +def _rows(ws): + out = {} + for row in ws.iter_rows(): + label = row[0].value + if label: + out[str(label).split(" (")[0]] = [c.value for c in row[1:]] + return out + + +def _book(model, tmp_path): + path = tmp_path / "book.xlsx" + model.to_excel(path) + return load_workbook(path) + + +class Deal(mo.MultiVariableClass): + def compute(self, region, amount): + self.region = mo.Variable(region, display_name="Region") + self.amount = mo.Variable(amount, display_name="Amount", unit="USD") + self.flow = mo.Variable([amount * 0.1, amount * 0.2, amount * 0.3], display_name="Flow") + + +def _model(): + m = mo.Model("m", display_name="M", default_grain="year", default_start="2026", + default_periods=3) + with m: + m.inputs = mo.MultiVariable("Inputs", excel_props={"tab": True}) + with m.inputs as i: + i.years = mo.Variable([2026, 2027, 2028], display_name="Years") + i.rates = mo.Variable([0.05, 0.04, 0.03], display_name="Rates") + i.names = mo.Variable(["Alpha", "Beta", "Gamma"], display_name="Names") + i.desc = mo.Variable([30, 20, 10], display_name="Desc") + i.dup = mo.Variable([20, 10, 20], display_name="Dup") + i.vals = mo.Variable(["a", "b", "c"], display_name="Vals") + i.mixed = mo.Variable([1.0, None, 3.0], display_name="Mixed") + i.dates = mo.Variable([dt.date(2026, 1, 1), dt.date(2027, 1, 1), dt.date(2028, 1, 1)], + display_name="Dates") + i.infl = mo.Variable({2026: 0.05, 2027: 0.04, 2028: 0.03}, display_name="Inflation") + i.pick = mo.Variable(2027, display_name="Pick") + i.word = mo.Variable("beta", display_name="Word") + i.tpos = mo.Variable("2", display_name="TPos") + i.blank = mo.Variable(None, display_name="Blank") + i.zeros = mo.Variable([0, 1, 2], display_name="Zeros") + i.err = (i.pick / 0).set_display_name("Err") + m.deals = mo.MultiVariable("Deals", excel_props={"tab": True, "orient": "records"}) + with m.deals as d: + d.a = Deal(region="North", amount=100.0, display_name="Alpha") + d.b = Deal(region="South", amount=200.0, display_name="Beta") + d.c = Deal(region="North", amount=300.0, display_name="Gamma") + m.t = mo.MultiVariable("Time", excel_props={"tab": True}) + m.t.year = mo.YEAR(mo.time.start).set_display_name("Year") + m.t.label = mo.Variable(["Alpha", "Gamma", "Delta"], display_name="Label") + m.s = mo.MultiVariable("Summary", excel_props={"tab": True}) + return m + + +# ══════════════════════════════════════════════════════════════════════ +# Rows are ranges, records and rows are blocks +# ══════════════════════════════════════════════════════════════════════ + + +class TestRangesAndBlocks: + def test_every_function_over_rows(self, tmp_path): + m = _model() + i = m.inputs + with m.s as s: + s.match = MATCH(i.pick, i.years, 0).set_display_name("match") + s.match_default = MATCH(2027.5, i.years).set_display_name("match_default") + s.index = INDEX(i.rates, 2).set_display_name("index") + s.index_match = INDEX(i.rates, MATCH(i.pick, i.years, 0)).set_display_name("index_match") + s.xl = XLOOKUP(i.pick, i.years, i.rates).set_display_name("xl") + s.xl_text = XLOOKUP(i.word, i.names, i.rates).set_display_name("xl_text") + s.xl_nf = XLOOKUP(2030, i.years, i.rates, 0).set_display_name("xl_nf") + s.xl_modes = XLOOKUP(2026.5, i.years, i.rates, 0, 1).set_display_name("xl_modes") + s.xl_na = XLOOKUP(2027.5, i.years, i.rates, match_mode=-1).set_display_name("xl_na") + s.xl_last = XLOOKUP(20, i.dup, i.vals, search_mode=-1).set_display_name("xl_last") + s.xl_date = XLOOKUP(dt.date(2027, 1, 1), i.dates, i.rates).set_display_name("xl_date") + s.hl = HLOOKUP(i.pick, [i.years, i.rates], 2, False).set_display_name("hl") + s.hl_default = HLOOKUP(2027.5, [i.years, i.rates], 2).set_display_name("hl_default") + s.index_rows = INDEX([i.years, i.rates], 2, 3).set_display_name("index_rows") + rows = _rows(_book(m, tmp_path)["Summary"]) + assert rows["match"][0] == "=MATCH(Inputs!B12, Inputs!C2:E2, 0)" + assert rows["match_default"][0] == "=MATCH(2027.5, Inputs!C2:E2)" + assert rows["index"][0] == "=INDEX(Inputs!C3:E3, 2)" + assert rows["index_match"][0] == "=INDEX(Inputs!C3:E3, MATCH(Inputs!B12, Inputs!C2:E2, 0))" + assert rows["xl"][0] == "=_xlfn.XLOOKUP(Inputs!B12, Inputs!C2:E2, Inputs!C3:E3)" + assert rows["xl_text"][0] == "=_xlfn.XLOOKUP(Inputs!B13, Inputs!C4:E4, Inputs!C3:E3)" + assert rows["xl_nf"][0] == "=_xlfn.XLOOKUP(2030, Inputs!C2:E2, Inputs!C3:E3, 0)" + assert rows["xl_modes"][0] == "=_xlfn.XLOOKUP(2026.5, Inputs!C2:E2, Inputs!C3:E3, 0, 1)" + assert rows["xl_na"][0] == "=_xlfn.XLOOKUP(2027.5, Inputs!C2:E2, Inputs!C3:E3, NA(), -1)" + assert rows["xl_last"][0] == "=_xlfn.XLOOKUP(20, Inputs!C6:E6, Inputs!C7:E7, NA(), 0, -1)" + assert rows["xl_date"][0] == "=_xlfn.XLOOKUP(DATE(2027, 1, 1), Inputs!C9:E9, Inputs!C3:E3)" + assert rows["hl"][0] == "=HLOOKUP(Inputs!B12, Inputs!C2:E3, 2, FALSE)" + assert rows["hl_default"][0] == "=HLOOKUP(2027.5, Inputs!C2:E3, 2)" + assert rows["index_rows"][0] == "=INDEX(Inputs!C2:E3, 2, 3)" + # The values the cells cache are the engine's. + assert (m.s.match.value, m.s.index_match.value, m.s.xl.value) == (2, 0.04, 0.04) + assert (m.s.xl_na.value, m.s.xl_last.value, m.s.hl.value) == (0.04, "c", 0.04) + + def test_a_table_of_records_is_one_block_the_label_column_first(self, tmp_path): + m = _model() + d = m.deals + with m.s as s: + s.vl = VLOOKUP("Beta", m.deals, 3, False).set_display_name("vl") + s.vl_default = VLOOKUP("Bob", m.deals, 3).set_display_name("vl_default") + s.vl_label = VLOOKUP("gamma", m.deals, 1, False).set_display_name("vl_label") + s.index_block = INDEX(m.deals, 2, 3).set_display_name("index_block") + s.index_label = INDEX(m.deals, 2, 1).set_display_name("index_label") + s.hl_block = HLOOKUP("North", m.deals, 2, False).set_display_name("hl_block") + s.match_vert = MATCH("South", [d.a.region, d.b.region, d.c.region], 0).set_display_name("match_vert") + s.index_vert = INDEX([d.a.amount, d.b.amount, d.c.amount], 3).set_display_name("index_vert") + s.xl_vert = XLOOKUP("South", [d.a.region, d.b.region, d.c.region], + [d.a.amount, d.b.amount, d.c.amount]).set_display_name("xl_vert") + book = _book(m, tmp_path) + rows = _rows(book["Summary"]) + assert rows["vl"][0] == '=VLOOKUP("Beta", Deals!A3:C5, 3, FALSE)' + assert rows["vl_default"][0] == '=VLOOKUP("Bob", Deals!A3:C5, 3)' + assert rows["vl_label"][0] == '=VLOOKUP("gamma", Deals!A3:C5, 1, FALSE)' + assert rows["index_block"][0] == "=INDEX(Deals!A3:C5, 2, 3)" + assert rows["index_label"][0] == "=INDEX(Deals!A3:C5, 2, 1)" + assert rows["hl_block"][0] == '=HLOOKUP("North", Deals!A3:C5, 2, FALSE)' + assert rows["match_vert"][0] == '=MATCH("South", Deals!B3:B5, 0)' + assert rows["index_vert"][0] == "=INDEX(Deals!C3:C5, 3)" + assert rows["xl_vert"][0] == '=_xlfn.XLOOKUP("South", Deals!B3:B5, Deals!C3:C5)' + # The block the formula names holds what the engine looked through: + # the labels in column A, the fields after them. + ws = book["Deals"] + grid = [[ws.cell(r, c).value for c in range(1, 4)] for r in range(3, 6)] + assert grid == [["Alpha", "North", 100], ["Beta", "South", 200], ["Gamma", "North", 300]] + assert (m.s.vl.value, m.s.vl_default.value, m.s.vl_label.value) == (200.0, 200.0, "Gamma") + assert (m.s.index_block.value, m.s.index_label.value, m.s.hl_block.value) == ( + 200.0, "Beta", "South") + + def test_the_rows_of_the_records_period_blocks_are_one_block(self, tmp_path): + m = _model() + d = m.deals + with m.s as s: + s.flows = INDEX([d.a.flow, d.b.flow, d.c.flow], 2, 3).set_display_name("flows") + s.hl_flows = HLOOKUP(20, [d.a.flow, d.b.flow, d.c.flow], 3, False).set_display_name("hl_flows") + rows = _rows(_book(m, tmp_path)["Summary"]) + assert rows["flows"][0] == "=INDEX(Deals!D8:F10, 2, 3)" + assert rows["hl_flows"][0] == "=HLOOKUP(20, Deals!D8:F10, 3, FALSE)" + assert m.s.flows.value == 60.0 and m.s.hl_flows.value == 60.0 + + def test_a_series_key_is_one_call_per_period_over_the_whole_range(self, tmp_path): + m = _model() + i, t = m.inputs, m.t + with m.s as s: + s.series = XLOOKUP(t.year, i.years, i.rates, 0).set_display_name("series") + s.series_match = MATCH(t.year, i.years, 0).set_display_name("series_match") + s.series_index = INDEX(i.rates, MATCH(t.year, i.years, 0)).set_display_name("series_index") + s.series_vl = VLOOKUP(t.label, m.deals, 3, False).set_display_name("series_vl") + s.series_pos = INDEX(i.rates, t.year - 2025).set_display_name("series_pos") + rows = _rows(_book(m, tmp_path)["Summary"]) + # A sheet of series only: the periods start at B. + assert rows["series"] == [ + "=_xlfn.XLOOKUP(Time!B2, Inputs!C2:E2, Inputs!C3:E3, 0)", + "=_xlfn.XLOOKUP(Time!C2, Inputs!C2:E2, Inputs!C3:E3, 0)", + "=_xlfn.XLOOKUP(Time!D2, Inputs!C2:E2, Inputs!C3:E3, 0)", + ] + assert rows["series_match"][0] == "=MATCH(Time!B2, Inputs!C2:E2, 0)" + assert rows["series_index"][2] == "=INDEX(Inputs!C3:E3, MATCH(Time!D2, Inputs!C2:E2, 0))" + assert rows["series_vl"] == [ + '=VLOOKUP(Time!B3, Deals!A3:C5, 3, FALSE)', + '=VLOOKUP(Time!C3, Deals!A3:C5, 3, FALSE)', + '=VLOOKUP(Time!D3, Deals!A3:C5, 3, FALSE)', + ] + assert rows["series_pos"][1] == "=INDEX(Inputs!C3:E3, Time!C2 - 2025)" + assert m.s.series.value == [0.05, 0.04, 0.03] + assert m.s.series_vl.value == [100.0, 300.0, "#N/A"] + assert m.s.series_pos.value == [0.05, 0.04, 0.03] + + def test_on_the_rows_own_sheet_and_inside_a_larger_formula(self, tmp_path): + m = _model() + with m.inputs as i: + i.rate = XLOOKUP(i.pick, i.years, i.rates).set_display_name("rate") + i.scaled = (INDEX(i.rates, MATCH(i.pick, i.years, 0)) * 100).set_display_name("scaled") + rows = _rows(_book(m, tmp_path)["Inputs"]) + assert rows["rate"][0] == "=_xlfn.XLOOKUP(B12, C2:E2, C3:E3)" + assert rows["scaled"][0] == "=INDEX(C3:E3, MATCH(B12, C2:E2, 0)) * 100" + + +# ══════════════════════════════════════════════════════════════════════ +# No rectangle: the cell carries its value and check_compat lists it +# ══════════════════════════════════════════════════════════════════════ + + +class TestNoRectangleShipsTheValue: + def test_the_keys_of_a_keyed_line_have_no_cells(self, tmp_path): + m = _model() + i = m.inputs + with m.s as s: + s.keyed = XLOOKUP(i.pick, i.infl).set_display_name("keyed") + s.keyed_series = XLOOKUP(m.t.year, i.infl, if_not_found=0).set_display_name("keyed_series") + with pytest.warns(mo.CrossScopeReferenceWarning, match="XLOOKUP"): + rows = _rows(_book(m, tmp_path)["Summary"]) + assert rows["keyed"][0] == 0.04 + assert rows["keyed_series"][1:] == [0.05, 0.04, 0.03] + with pytest.warns(mo.CrossScopeReferenceWarning): + issues = check_compat(m, "excel") + assert [(x.kind, x.qpath, x.period) for x in issues] == [ + ("value", m.s.keyed.id, 0), + ("value", m.s.keyed_series.id, 0), + ("value", m.s.keyed_series.id, 1), + ("value", m.s.keyed_series.id, 2), + ] + assert issues[0].inlined == "XLOOKUP(pick, [2026, 2027, 2028], infl)" + + def test_records_laid_out_as_sections(self, tmp_path): + m = _model() + with m: + m.stack = mo.MultiVariable("Stack", excel_props={"tab": True}) + with m.stack as t: + t.a = Deal(region="North", amount=100.0, display_name="Alpha") + t.b = Deal(region="South", amount=200.0, display_name="Beta") + with m.s as s: + s.vl = VLOOKUP("Beta", m.stack, 3, False).set_display_name("vl") + rows = _rows(_book(m, tmp_path)["Summary"]) + assert rows["vl"][0] == 200 + issues = check_compat(m, "excel") + assert [(x.kind, x.qpath) for x in issues] == [("value", m.s.vl.id)] + assert issues[0].inlined.startswith("[['Alpha', ") + + def test_a_unit_column_between_a_record_s_label_and_its_fields(self, tmp_path): + m = _model() + m.default_excel_view = mo.ExcelView(meta={"unit": "Unit"}) + with m.s as s: + s.vl = VLOOKUP("Beta", m.deals, 3, False).set_display_name("vl") + s.index = INDEX(m.deals, 2, 3).set_display_name("index") + book = _book(m, tmp_path) + ws = book["Deals"] + # The fields start at C (the unit row under the captions pushes the + # records to row 4): nothing names the label cell at A. + assert [ws.cell(4, c).value for c in range(1, 5)] == ["Alpha", None, "North", 100] + rows = _rows(book["Summary"]) + # (The Summary sheet has the unit column too: the values stand at C.) + assert rows["vl"][1] == 200 and rows["index"][1] == 200 + assert [(x.kind, x.qpath) for x in check_compat(m, "excel")] == [ + ("value", m.s.vl.id), ("value", m.s.index.id)] + + def test_rows_on_two_sheets_a_typed_row_or_a_row_the_book_has_no_cells_for(self, tmp_path): + m = _model() + i = m.inputs + with m.s as s: + s.sheets = HLOOKUP(2027, [i.years, m.t.year], 2, False).set_display_name("sheets") + s.typed = HLOOKUP(2027, [[2026, 2027, 2028], i.rates], 2, False).set_display_name("typed") + s.unlaid = HLOOKUP(2027, [i.years, i.rates * 2], 2, False).set_display_name("unlaid") + rows = _rows(_book(m, tmp_path)["Summary"]) + assert rows["sheets"][0] == 2027 and rows["typed"][0] == 0.04 and rows["unlaid"][0] == 0.08 + assert [(x.kind, x.qpath) for x in check_compat(m, "excel")] == [ + ("value", m.s.sheets.id), ("value", m.s.typed.id), ("value", m.s.unlaid.id)] + + def test_a_record_that_lacks_a_field(self, tmp_path): + class Custom(mo.MultiVariableClass): + def compute(self, salary, bonus=None): + self.salary = mo.Variable(salary, display_name="Salary") + if bonus is not None: + self.bonus = mo.Variable(bonus, display_name="Bonus") + + m = mo.Model("m", display_name="M") + with m: + m.staff = mo.MultiVariable("Staff", excel_props={"tab": True, "orient": "records"}) + with m.staff as t: + t.a = Custom(salary=1, display_name="A") + t.b = Custom(salary=2, bonus=9, display_name="B") + m.s = mo.MultiVariable("Summary", excel_props={"tab": True}) + m.s.bonus = VLOOKUP("B", m.staff, 3, False).set_display_name("bonus") + rows = _rows(_book(m, tmp_path)["Summary"]) + assert rows["bonus"][0] == 9 + assert [(x.kind, x.qpath) for x in check_compat(m, "excel")] == [("value", m.s.bonus.id)] + + def test_a_rectangle_is_listed_by_nobody(self, tmp_path): + m = _model() + i = m.inputs + with m.s as s: + s.vl = VLOOKUP("Beta", m.deals, 3, False).set_display_name("vl") + s.hl = HLOOKUP(i.pick, [i.years, i.rates], 2, False).set_display_name("hl") + s.xl = XLOOKUP(m.t.year, i.years, i.rates).set_display_name("xl") + _book(m, tmp_path) + assert check_compat(m, "excel") == [] + + +# ══════════════════════════════════════════════════════════════════════ +# LibreOffice recalculates these functions to the engine's numbers +# ══════════════════════════════════════════════════════════════════════ + + +def _soffice(): + for candidate in ("soffice", "libreoffice"): + found = shutil.which(candidate) + if found: + return found + mac = Path("/Applications/LibreOffice.app/Contents/MacOS/soffice") + return str(mac) if mac.exists() else None + + +def _recalculated(model, tmp_path): + """The Summary sheet after LibreOffice recalculates the written + workbook, by line label.""" + soffice = _soffice() + if soffice is None: + pytest.skip("LibreOffice not installed") + path = tmp_path / "lookup.xlsx" + model.to_excel(path) + subprocess.run( + [soffice, f"-env:UserInstallation={(tmp_path / 'prof').as_uri()}", + "--headless", "--convert-to", "xlsx", "--outdir", + str(tmp_path / "calc"), str(path)], + check=True, capture_output=True, + ) + return _rows(load_workbook(tmp_path / "calc" / "lookup.xlsx", data_only=True)["Summary"]) + + +def _cells(calc_row, value): + """What the line's cells hold after the recalculation: the one cell + of a scalar line, the period cells of a series (past the sheet's + constants column).""" + if not isinstance(value, list): + return calc_row[0] + start = next(i for i, c in enumerate(calc_row) if c is not None) + return calc_row[start:start + len(value)] + + +@pytest.mark.slow +def test_the_workbook_recalculates_to_the_python_values(tmp_path): + m = _model() + i, d, t = m.inputs, m.deals, m.t + regions = [d.a.region, d.b.region, d.c.region] + amounts = [d.a.amount, d.b.amount, d.c.amount] + flows = [d.a.flow, d.b.flow, d.c.flow] + with m.s as s: + s.match_exact = MATCH(i.pick, i.years, 0).set_display_name("match_exact") + s.match_approx = MATCH(2027.5, i.years).set_display_name("match_approx") + s.match_desc = MATCH(25, i.desc, -1).set_display_name("match_desc") + s.match_text = MATCH("b*", i.names, 0).set_display_name("match_text") + s.match_na = MATCH(2030, i.years, 0).set_display_name("match_na") + s.match_vert = MATCH("South", regions, 0).set_display_name("match_vert") + s.index_row = INDEX(i.rates, 2).set_display_name("index_row") + s.index_match = INDEX(i.rates, MATCH(i.pick, i.years, 0)).set_display_name("index_match") + s.index_vert = INDEX(amounts, 3).set_display_name("index_vert") + s.index_block = INDEX(m.deals, 2, 3).set_display_name("index_block") + s.index_label = INDEX(m.deals, 2, 1).set_display_name("index_label") + s.index_rows = INDEX([i.years, i.rates], 2, 3).set_display_name("index_rows") + s.index_flows = INDEX(flows, 2, 3).set_display_name("index_flows") + s.index_blank = INDEX(i.mixed, 2).set_display_name("index_blank") + s.index_ref = INDEX(i.rates, 4).set_display_name("index_ref") + s.xl = XLOOKUP(i.pick, i.years, i.rates).set_display_name("xl") + s.xl_text = XLOOKUP(i.word, i.names, i.rates).set_display_name("xl_text") + s.xl_nf = XLOOKUP(2030, i.years, i.rates, 0).set_display_name("xl_nf") + s.xl_na = XLOOKUP(2030, i.years, i.rates).set_display_name("xl_na") + s.xl_below = XLOOKUP(2027.5, i.years, i.rates, match_mode=-1).set_display_name("xl_below") + s.xl_above = XLOOKUP(2026.5, i.years, i.rates, 0, 1).set_display_name("xl_above") + s.xl_last = XLOOKUP(20, i.dup, i.vals, search_mode=-1).set_display_name("xl_last") + s.xl_first = XLOOKUP(20, i.dup, i.vals).set_display_name("xl_first") + s.xl_wild = XLOOKUP("g*", i.names, i.rates, 0, 2).set_display_name("xl_wild") + s.xl_vert = XLOOKUP("South", regions, amounts).set_display_name("xl_vert") + s.xl_date = XLOOKUP(dt.date(2027, 1, 1), i.dates, i.rates).set_display_name("xl_date") + s.xl_blank = XLOOKUP(2027, i.years, i.mixed).set_display_name("xl_blank") + s.xl_keyed = XLOOKUP(i.pick, i.infl).set_display_name("xl_keyed") + s.vl = VLOOKUP("Beta", m.deals, 3, False).set_display_name("vl") + s.vl_approx = VLOOKUP("Bob", m.deals, 3).set_display_name("vl_approx") + s.vl_label = VLOOKUP("gamma", m.deals, 1, False).set_display_name("vl_label") + s.vl_na = VLOOKUP("Delta", m.deals, 2, False).set_display_name("vl_na") + s.vl_wild = VLOOKUP("G*", m.deals, 2, False).set_display_name("vl_wild") + s.hl = HLOOKUP(i.pick, [i.years, i.rates], 2, False).set_display_name("hl") + s.hl_approx = HLOOKUP(2027.5, [i.years, i.rates], 2).set_display_name("hl_approx") + s.hl_block = HLOOKUP("North", m.deals, 2, False).set_display_name("hl_block") + s.series = XLOOKUP(t.year, i.years, i.rates, 0).set_display_name("series") + s.series_match = MATCH(t.year, i.years, 0).set_display_name("series_match") + s.series_vl = VLOOKUP(t.label, m.deals, 3, False).set_display_name("series_vl") + s.series_index = INDEX(i.rates, MATCH(t.year, i.years, 0)).set_display_name("series_index") + # A position is read as a number: text that reads as one, a boolean. + s.index_text = INDEX(i.rates, "2").set_display_name("index_text") + s.index_true = INDEX(i.rates, True).set_display_name("index_true") + s.index_text_cell = INDEX(i.rates, i.tpos).set_display_name("index_text_cell") + s.index_block_text = INDEX(m.deals, "2", "3").set_display_name("index_block_text") + s.vl_text_col = VLOOKUP("Beta", m.deals, "3", False).set_display_name("vl_text_col") + s.vl_text_cell = VLOOKUP("Beta", m.deals, i.tpos, False).set_display_name("vl_text_cell") + s.hl_text_row = HLOOKUP(i.pick, [i.years, i.rates], "2", False).set_display_name( + "hl_text_row") + s.index_date = INDEX(i.rates, dt.date(2026, 1, 1)).set_display_name("index_date") + # A key read from a blank cell finds nothing, not the 0. + s.match_blank = MATCH(i.blank, i.zeros, 0).set_display_name("match_blank") + s.xl_blank_key = XLOOKUP(i.blank, i.zeros, i.rates, -1).set_display_name("xl_blank_key") + # The column or row number is checked before the key: its error wins. + s.vl_err_zero = VLOOKUP(i.err, m.deals, 0, False).set_display_name("vl_err_zero") + s.hl_err_zero = HLOOKUP(i.err, [i.years, i.rates], 0, False).set_display_name( + "hl_err_zero") + s.vl_err_key = VLOOKUP(i.err, m.deals, 3, False).set_display_name("vl_err_key") + with pytest.warns(mo.CrossScopeReferenceWarning, match="XLOOKUP"): + calc = _recalculated(m, tmp_path) + lines = [(name, line) for name, line in m.s.walk() if isinstance(line, mo.Variable)] + assert len(lines) == 53 + assert (m.s.index_text.value, m.s.index_true.value, m.s.vl_text_cell.value) == ( + 0.04, 0.05, "South") + assert (m.s.match_blank.value, m.s.xl_blank_key.value) == ("#N/A", -1) + assert (m.s.vl_err_zero.value, m.s.hl_err_zero.value, m.s.vl_err_key.value) == ( + "#VALUE!", "#VALUE!", "#DIV/0!") + assert m.s.series_vl.value == [100.0, 300.0, "#N/A"] + assert m.s.match_na.value == "#N/A" and m.s.index_ref.value == "#REF!" + for name, line in lines: + assert _cells(calc[name], line.value) == pytest.approx(line.value), ( + name, calc[name], line.value) + + +@pytest.mark.slow +@pytest.mark.xfail(strict=True, reason=( + "LibreOffice reads a boolean as a number in a lookup, matches text that reads as a " + "number against the number, and gives #VALUE! for a VLOOKUP column or HLOOKUP row " + "number past the block; Excel does none of these (#REF! for the last), and the " + "engine keeps Excel's rule")) +def test_where_libreoffice_departs_from_excel(tmp_path): + m = _model() + with m.inputs as i: + i.bools = mo.Variable([1, True, "TRUE"], display_name="Bools") + i.tyear = mo.Variable("2027", display_name="TYear") + with m.s as s: + # A boolean matches a boolean only: Excel 2, LibreOffice 1 (TRUE is 1 there). + s.match_bool = MATCH(True, i.bools, 0).set_display_name("match_bool") + s.xl_bool = XLOOKUP(True, i.bools, i.years).set_display_name("xl_bool") + # Text is not a number: Excel #N/A, LibreOffice finds 2027. + s.match_text = MATCH("2027", i.years, 0).set_display_name("match_text") + s.match_text_cell = MATCH(i.tyear, i.years, 0).set_display_name("match_text_cell") + s.xl_text_cell = XLOOKUP(i.tyear, i.years, i.vals).set_display_name("xl_text_cell") + # Past the block: Excel #REF!, found or not; LibreOffice #VALUE!. + s.vl_ref = VLOOKUP("Beta", m.deals, 4, False).set_display_name("vl_ref") + s.hl_ref = HLOOKUP(i.pick, [i.years, i.rates], 3, False).set_display_name("hl_ref") + s.vl_na_ref = VLOOKUP("Delta", m.deals, 4, False).set_display_name("vl_na_ref") + s.hl_na_ref = HLOOKUP(2025, [i.years, i.rates], 3, False).set_display_name("hl_na_ref") + s.vl_err_ref = VLOOKUP(i.err, m.deals, 4, False).set_display_name("vl_err_ref") + assert [m.s.match_bool.value, m.s.xl_bool.value, m.s.match_text.value, + m.s.match_text_cell.value, m.s.xl_text_cell.value] == [2, 2027, "#N/A", "#N/A", "#N/A"] + assert {m.s.vl_ref.value, m.s.hl_ref.value, m.s.vl_na_ref.value, m.s.hl_na_ref.value, + m.s.vl_err_ref.value} == {"#REF!"} + calc = _recalculated(m, tmp_path) + departing = [(name, calc[name][0], line.value) for name, line in m.s.walk() + if isinstance(line, mo.Variable) and calc[name][0] != line.value] + assert departing == [], departing diff --git a/tests/test_excel_math.py b/tests/test_excel_math.py new file mode 100644 index 0000000..7241a04 --- /dev/null +++ b/tests/test_excel_math.py @@ -0,0 +1,381 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The rounding family, the powers and logarithms, ``SIGN`` and +``PRODUCT`` in the workbook. + +An element-wise function writes one call per period cell — ``=ROUNDUP(B2, +1)``, ``=CEILING(B2, B3)``, ``=LOG(B4)`` with the base left out when none +was given — a scalar one cell, and ``PRODUCT`` writes a row as a range, +``=PRODUCT(X!B4:G4)``. Every name is the classic one, stored without a +prefix, and every call renders natively. The slow test recalculates a +workbook with every function in LibreOffice and reads the engine's +numbers back, a whole number to the unit; the cells where LibreOffice's +answer is not the engine's are pinned, and the parity case over them is +an expected failure. +""" + +import datetime as dt +import shutil +import subprocess +from pathlib import Path + +import pytest +from openpyxl import load_workbook + +import modeleon as mo +from modeleon.functions._helpers import is_error +from modeleon.functions.mathfn import ( + CEILING, + EXP, + FLOOR, + LN, + LOG, + LOG10, + POWER, + PRODUCT, + ROUNDDOWN, + ROUNDUP, + SIGN, + SQRT, + TRUNC, +) + +NAMES = ( + "ROUNDUP", "ROUNDDOWN", "TRUNC", "CEILING", "FLOOR", "POWER", "SQRT", "EXP", + "LN", "LOG", "LOG10", "SIGN", "PRODUCT", +) + + +def _rows(ws): + out = {} + for row in ws.iter_rows(): + label = row[0].value + if label: + out[str(label)] = [c.value for c in row[1:]] + return out + + +def _book(model, tmp_path, sheet): + path = tmp_path / "book.xlsx" + model.to_excel(path) + return _rows(load_workbook(path)[sheet]) + + +def _model(): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=6) + with m: + m.i = mo.MultiVariable("I") + m.i.p = mo.Variable(2.5, display_name="p") + m.i.q = mo.Variable(-2.0, display_name="q") + m.i.z = mo.Variable(0.0, display_name="z") + m.i.eight = mo.Variable(8.0, display_name="eight") + m.i.m8 = mo.Variable(-8.0, display_name="m8") + m.i.t = mo.Variable(True, display_name="t") + m.i.four = mo.Variable("4", display_name="four") + m.i.word = mo.Variable("abc", display_name="word") + m.x = mo.MultiVariable("X") + m.x.a = mo.Variable([2.5, -2.5, 0.0, 3.14159, 1234.5678, -0.125], display_name="a") + m.x.b = mo.Variable([0.1, 2.0, 0.5, 100.0, 3.0, 0.25], display_name="b") + m.x.c = mo.Variable([16.0, 2.0, 1.0, 100000.0, 0.5, 1000.0], display_name="c") + m.x.d = mo.Variable([0.0, 1.0, -1.0, 2.5, 10.0, -20.0], display_name="d") + return m + + +class TestEachFunctionWritesOneCallPerPeriod: + @pytest.mark.parametrize("build, first, second", [ + (lambda x, i: ROUNDUP(x.a, 1), "=ROUNDUP(B2, 1)", "=ROUNDUP(C2, 1)"), + (lambda x, i: ROUNDDOWN(x.a, 1), "=ROUNDDOWN(B2, 1)", "=ROUNDDOWN(C2, 1)"), + (lambda x, i: TRUNC(x.a), "=TRUNC(B2)", "=TRUNC(C2)"), + (lambda x, i: TRUNC(x.a, 2), "=TRUNC(B2, 2)", "=TRUNC(C2, 2)"), + (lambda x, i: CEILING(x.a, 0.5), "=CEILING(B2, 0.5)", "=CEILING(C2, 0.5)"), + (lambda x, i: CEILING(x.a, x.b), "=CEILING(B2, B3)", "=CEILING(C2, C3)"), + (lambda x, i: FLOOR(x.a, 0.5), "=FLOOR(B2, 0.5)", "=FLOOR(C2, 0.5)"), + (lambda x, i: FLOOR(x.a, i.p), "=FLOOR(B2, I!B1)", "=FLOOR(C2, I!B1)"), + (lambda x, i: POWER(x.a, 2), "=POWER(B2, 2)", "=POWER(C2, 2)"), + (lambda x, i: POWER(x.c, x.d), "=POWER(B4, B5)", "=POWER(C4, C5)"), + (lambda x, i: SQRT(x.c), "=SQRT(B4)", "=SQRT(C4)"), + (lambda x, i: EXP(x.d), "=EXP(B5)", "=EXP(C5)"), + (lambda x, i: LN(x.c), "=LN(B4)", "=LN(C4)"), + (lambda x, i: LOG(x.c), "=LOG(B4)", "=LOG(C4)"), + (lambda x, i: LOG(x.c, 2), "=LOG(B4, 2)", "=LOG(C4, 2)"), + (lambda x, i: LOG(x.c, i.eight), "=LOG(B4, I!B4)", "=LOG(C4, I!B4)"), + (lambda x, i: LOG10(x.c), "=LOG10(B4)", "=LOG10(C4)"), + (lambda x, i: SIGN(x.a), "=SIGN(B2)", "=SIGN(C2)"), + ]) + def test_the_call_over_this_period_s_cells(self, tmp_path, build, first, second): + m = _model() + with m: + m.x.r = build(m.x, m.i).set_display_name("r") + r = _book(m, tmp_path, "X")["r"][:6] + assert r[:2] == [first, second], r + assert all(isinstance(cell, str) and cell.startswith("=") for cell in r), r + + def test_a_scalar_is_one_cell(self, tmp_path): + m = _model() + with m: + m.i.ru = ROUNDUP(m.i.p, 0).set_display_name("ru") + m.i.pw = POWER(m.i.q, 0.5).set_display_name("pw") + m.i.lg = LOG(m.i.eight, 2).set_display_name("lg") + m.i.ce = CEILING(m.i.p, m.i.q).set_display_name("ce") + m.i.sq = SQRT(m.i.eight * 2).set_display_name("sq") + i = _book(m, tmp_path, "I") + assert i["ru"][0] == "=ROUNDUP(B1, 0)", i["ru"] + assert i["pw"][0] == "=POWER(B2, 0.5)", i["pw"] + assert i["lg"][0] == "=LOG(B4, 2)", i["lg"] + assert i["ce"][0] == "=CEILING(B1, B2)", i["ce"] + assert i["sq"][0] == "=SQRT(B4 * 2)", i["sq"] + + +class TestProductWritesARange: + def test_a_row_is_a_range_and_a_scalar_a_cell(self, tmp_path): + m = _model() + with m: + m.i.row = PRODUCT(m.x.c).set_display_name("row") + m.i.cells = PRODUCT(m.i.p, m.i.q, 3).set_display_name("cells") + m.i.rows = (PRODUCT(m.x.b) * PRODUCT(m.x.d)).set_display_name("rows") + i = _book(m, tmp_path, "I") + assert i["row"][0] == "=PRODUCT(X!B4:G4)", i["row"] + assert i["cells"][0] == "=PRODUCT(B1, B2, 3)", i["cells"] + assert i["rows"][0] == "=PRODUCT(X!B3:G3) * PRODUCT(X!B5:G5)", i["rows"] + + def test_several_series_are_one_call_per_period(self, tmp_path): + # One call per period, as ``b * d`` and ``c * 2`` are. + m = _model() + with m: + m.x.rows = PRODUCT(m.x.b, m.x.d).set_display_name("rows") + m.x.mixed = PRODUCT(m.x.c, 2).set_display_name("mixed") + x = _book(m, tmp_path, "X") + assert x["rows"][:2] == ["=PRODUCT(B3, B5)", "=PRODUCT(C3, C5)"], x["rows"] + assert x["mixed"][:2] == ["=PRODUCT(B4, 2)", "=PRODUCT(C4, 2)"], x["mixed"] + assert m.x.rows.value == [0.0, 2.0, -0.5, 250.0, 30.0, -5.0] + + +def _every_function(m): + """Every function above in ``m``, label -> line: the series on + X, the scalars and the reductions on I.""" + with m: + m.x.tenth = mo.Variable([0.1, 4.35, 1.1, 2.675, 0.7, 0.3], display_name="tenth") + m.x.mult = mo.Variable([3, 100, 1, 100, 3, 1], display_name="mult") + m.x.blank = mo.Variable([None] * 6, display_name="blank") + # Held as 0.30000000000000004, 434.99999999999994, …; shown as 0.3, 435, …. + m.x.held = (m.x.tenth * m.x.mult).set_display_name("held") + m.x.e = (m.x.b / m.x.d).set_display_name("e") # #DIV/0! in the first period + m.x.when = mo.EOMONTH(mo.time.start, 0).set_display_name("when") # a date row + m.i.day = mo.Variable(dt.date(2024, 2, 29), display_name="day") + m.i.noon = mo.Variable(dt.datetime(2024, 2, 29, 12, 0), display_name="noon") + m.i.big = mo.Variable(1e300, display_name="big") + m.i.tiny = mo.Variable(1e-300, display_name="tiny") + m.i.whole16 = mo.Variable(1000000000000001, display_name="whole16") + # A cell is written to 15 digits; the half is computed in the sheet. + m.i.half16 = (m.i.whole16 - 0.5).set_display_name("half16") + lines = { + "roundup": ROUNDUP(m.x.a, 1), + "rounddown": ROUNDDOWN(m.x.a, 1), + "trunc": TRUNC(m.x.a), + "trunc2": TRUNC(m.x.a, 2), + "ceil": CEILING(m.x.a, 0.5), + "floor": FLOOR(m.x.a, 0.5), + "ceil_b": CEILING(m.x.a, m.x.b), + "floor_b": FLOOR(m.x.a, m.x.b), + "power": POWER(m.x.a, 2), + "power_d": POWER(m.x.c, m.x.d), + "sqrt": SQRT(m.x.c), + "exp": EXP(m.x.d), + "ln": LN(m.x.c), + "log": LOG(m.x.c), + "log2": LOG(m.x.c, 2), + "log10": LOG10(m.x.c), + "sign": SIGN(m.x.a), + "ru_shown": ROUNDUP(m.x.held, 1), + "rd_shown": ROUNDDOWN(m.x.held, 0), + "tr_shown": TRUNC(m.x.held, 1), + "ce_shown": CEILING(m.x.held, 0.1), + "fl_shown": FLOOR(m.x.held, 0.1), + "sqrt_a": SQRT(m.x.a), + "ln_d": LN(m.x.d), + "exp_big": EXP(m.x.c), + "ru_err": ROUNDUP(m.x.e, 1), + "sq_err": SQRT(m.x.e), + "pw_err": POWER(m.x.e, 2), + "c0": CEILING(m.i.p, m.i.z), + "f0": FLOOR(m.i.p, m.i.z), + "p00": POWER(m.i.z, m.i.z), + "p0m1": POWER(m.i.z, -1), + "pneg": POWER(m.i.q, 0.5), + "l1": LOG(m.i.eight, 1), + "lneg": LOG(m.i.eight, m.i.q), + "c_nn": CEILING(m.i.q * 1.25, m.i.q), + "f_nn": FLOOR(m.i.q * 1.25, m.i.q), + "c_pn": CEILING(m.i.p, m.i.q), + "f_pn": FLOOR(m.i.p, m.i.q), + "c_np": CEILING(m.i.q * 1.25, 2), + "f_np": FLOOR(m.i.q * 1.25, 2), + "ru_s": ROUNDUP(m.i.p, 0), + "sq_s": SQRT(m.i.eight), + "lg_s": LOG(m.i.eight, 2), + "odd_root": POWER(m.i.m8, 1 / 3), + "odd_fifth": POWER(m.i.m8 * 4, 0.2), + "odd_neg": POWER(m.i.m8, -1 / 3), + "even_root": POWER(m.i.m8, 0.4), + "underflow": POWER(m.i.eight, -400), + "ru_big": ROUNDUP(m.i.p, -320), + "rd_big": ROUNDDOWN(m.i.p, -320), + "t_ru": ROUNDUP(m.i.t, 0), + "t_pw": POWER(m.i.t, 2), + "four_ru": ROUNDUP(m.i.four, 0), + "four_sq": SQRT(m.i.four), + "four_pw": POWER(2, m.i.four), + "word_ru": ROUNDUP(m.i.word, 0), + "word_sg": SIGN(m.i.word), + "prod_tb": PRODUCT(m.i.t, m.i.p), + "prod_blank": PRODUCT(m.x.blank), + "prod_c": PRODUCT(m.x.c), + "prod_pq": PRODUCT(m.i.p, m.i.q, 3), + "prod_rows": PRODUCT(m.x.b, m.x.d), + "fl_date": FLOOR(m.x.when, 7), + "ce_date": CEILING(m.x.when, 7), + "ru_date": ROUNDUP(m.x.when, -1), + "sg_date": SIGN(m.x.when), + "tr_noon": TRUNC(m.i.noon), + "sq_day": SQRT(m.i.day), + "prod_day": PRODUCT(m.i.day, 2), + "f00": FLOOR(m.i.z, m.i.z), + "c_past": CEILING(m.i.big, m.i.tiny), + "f_past": FLOOR(m.i.big, m.i.tiny), + "c_below": CEILING(m.i.tiny, m.i.big), + "c_whole16": CEILING(m.i.whole16, 1), + "c_half16": CEILING(m.i.half16, 1), + "f_half16": FLOOR(m.i.half16, 1), + } + for label, line in lines.items(): + sheet = m.x if line.var_type == 'list' else m.i + setattr(sheet, label, line.set_display_name(label)) + return lines + + +def test_every_name_is_classic_and_renders_natively(tmp_path): + m = _model() + lines = _every_function(m) + assert mo.check_compat(m, "excel") == [] + path = tmp_path / "book.xlsx" + m.to_excel(path) + formulas = [ + cell.value + for ws in load_workbook(path).worksheets + for row in ws.iter_rows() + for cell in row + if isinstance(cell.value, str) and cell.value.startswith("=") + ] + assert len(formulas) >= len(lines) + assert not [f for f in formulas if "_xlfn." in f] + for name in NAMES: + assert any(f.startswith(f"={name}(") for f in formulas), name + + +# ─── LibreOffice recalculates the workbook ────────────────────────── + + +def _soffice(): + for candidate in ("soffice", "libreoffice"): + found = shutil.which(candidate) + if found: + return found + mac = Path("/Applications/LibreOffice.app/Contents/MacOS/soffice") + return str(mac) if mac.exists() else None + + +def _recalculated(m, tmp_path): + """The workbook of ``m`` recalculated by LibreOffice: ``(X rows, I rows)`` + of values.""" + soffice = _soffice() + if soffice is None: + pytest.skip("LibreOffice not installed") + path = tmp_path / "math.xlsx" + m.to_excel(path) + subprocess.run( + [soffice, f"-env:UserInstallation={(tmp_path / 'prof').as_uri()}", + "--headless", "--convert-to", "xlsx", "--outdir", + str(tmp_path / "calc"), str(path)], + check=True, capture_output=True, timeout=300, + ) + book = load_workbook(tmp_path / "calc" / "math.xlsx", data_only=True) + return _rows(book["X"]), _rows(book["I"]) + + +def _number(cell): + """A cell's number: one LibreOffice formats as a date (it gives a + product of a date and a number the date's format) is read back as the + serial it holds.""" + if isinstance(cell, dt.datetime): + days = (cell - dt.datetime(1899, 12, 30)) / dt.timedelta(days=1) + return int(days) if days.is_integer() else days + return cell + + +def _cells(lines, x, i): + """Every function cell as ``((label, period), engine value, LibreOffice + value)``.""" + for label, line in lines.items(): + if line.var_type == 'list': + for period, (ours, theirs) in enumerate(zip(line.value, x[label][:6])): + yield (label, period), ours, _number(theirs) + else: + yield (label, 0), line.value, _number(i[label][0]) + + +#: Where LibreOffice's answer is not the engine's: what LibreOffice 25.2 +#: writes instead, pinned by cell. Most are where it departs from Excel, +#: whose answers (Microsoft's documentation, its known behaviour) the +#: engine computes: LibreOffice's "invalid argument" error is written as +#: #VALUE! where Excel says #NUM! (a root or a logarithm outside its +#: domain, a positive number with a negative significance) or #DIV/0! +#: (FLOOR of a number by 0, a logarithm to base 1); 0 ^ -1 is its #NUM! +#: where Excel divides by zero; it defines 0 ^ 0 as 1 where Excel says +#: #NUM!. Two are the engine's own choice where Excel's answer is neither +#: documented nor checked: a power too small for a cell is 0 (as EXP(-1000) +#: is 0 in both programs) where LibreOffice says #NUM!, and ROUNDUP to a +#: multiple no cell can hold (1e320) is #NUM! where LibreOffice says 0. +_LIBREOFFICE_INSTEAD = { + ("sqrt_a", 1): "#VALUE!", ("sqrt_a", 5): "#VALUE!", + ("ln_d", 0): "#VALUE!", ("ln_d", 2): "#VALUE!", ("ln_d", 5): "#VALUE!", + ("sq_err", 2): "#VALUE!", ("sq_err", 5): "#VALUE!", + ("c_pn", 0): "#VALUE!", ("f_pn", 0): "#VALUE!", ("lneg", 0): "#VALUE!", + ("f0", 0): "#VALUE!", ("l1", 0): "#VALUE!", + ("p0m1", 0): "#NUM!", + ("p00", 0): 1, + ("underflow", 0): "#NUM!", + ("ru_big", 0): 0, +} + + +@pytest.mark.slow +def test_the_workbook_recalculates_to_the_python_values(tmp_path): + m = _model() + lines = _every_function(m) + x, i = _recalculated(m, tmp_path) + for cell, ours, theirs in _cells(lines, x, i): + expected = _LIBREOFFICE_INSTEAD.get(cell, ours) + if is_error(expected): + assert theirs == expected, (cell, ours, theirs) + elif float(expected).is_integer() and abs(expected) < 2 ** 53: + assert theirs == expected, (cell, ours, theirs) # a whole number to the unit + else: + assert theirs == pytest.approx(expected, rel=1e-9, abs=1e-12), (cell, ours, theirs) + # PRODUCT over two rows is one call per period: 33 rows. + assert sum(1 for _ in _cells(lines, x, i)) == 33 * 6 + 44 + + +@pytest.mark.slow +@pytest.mark.xfail(strict=True, reason=( + "LibreOffice names a domain error differently from Excel (#VALUE! for its " + "'invalid argument' where Excel says #NUM! or #DIV/0!) and defines 0 ^ 0 as 1, " + "where the engine computes Excel's answers; where Excel's answer is unchecked, the " + "engine calls a power too small for a cell 0 and a ROUNDUP past the largest number " + "#NUM!, LibreOffice #NUM! and 0 — see _LIBREOFFICE_INSTEAD." +)) +def test_the_errors_libreoffice_names_differently(tmp_path): + m = _model() + lines = _every_function(m) + x, i = _recalculated(m, tmp_path) + for cell, ours, theirs in _cells(lines, x, i): + if cell in _LIBREOFFICE_INSTEAD: + assert theirs == ours, (cell, ours, theirs) diff --git a/tests/test_excel_statistics.py b/tests/test_excel_statistics.py new file mode 100644 index 0000000..a128934 --- /dev/null +++ b/tests/test_excel_statistics.py @@ -0,0 +1,711 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Statistics in the workbook. + +A row is its cells' range (``Data!B3:F3``); a list of scalar lines whose +cells stand one under another is a vertical range — one field down the +records of a table, ``Comps!C2:C6`` — and ``RANK`` of each record among +its peers is one call per record against that range. ``MEDIAN`` and the +spreads take any number of arguments, two rows and more one call per +period over each period's cells; the others one range — a list of yearly +lines one range per period, the run of their cells in its column — and +cells that form no run carry the computed value (``check_compat`` lists +them). +The names Excel 2010 added are stored with the ``_xlfn.`` prefix +(``=_xlfn.PERCENTILE.INC(...)``). Every function recalculates in +LibreOffice to the engine's numbers, except where LibreOffice departs +from Excel, pinned by cell as an expected failure. +""" + +import datetime as dt +import shutil +import subprocess +from pathlib import Path + +import pytest +from openpyxl import load_workbook + +import modeleon as mo +from modeleon.compile import check_compat +from modeleon.functions.statistics import ( + LARGE, + MEDIAN, + PERCENTILE, + PERCENTILE_EXC, + PERCENTILE_INC, + QUARTILE, + QUARTILE_EXC, + QUARTILE_INC, + RANK, + RANK_AVG, + RANK_EQ, + SMALL, + STDEV, + STDEV_P, + STDEV_S, + STDEVP, + VAR, + VAR_P, + VAR_S, + VARP, +) + + +def _rows(ws): + out = {} + for row in ws.iter_rows(): + label = row[0].value + if label: + out[str(label).split(" (")[0]] = [c.value for c in row[1:]] + return out + + +def _book(model, tmp_path): + path = tmp_path / "book.xlsx" + model.to_excel(path) + return load_workbook(path) + + +class Comp(mo.MultiVariableClass): + def compute(self, multiple): + self.multiple = mo.Variable(multiple, display_name="Multiple", unit="x") + + +def _model(): + m = mo.Model("m", display_name="M", default_grain="month", default_start="2026-01", + default_periods=5) + with m: + m.inputs = mo.MultiVariable("Inputs", excel_props={"tab": True}) + m.inputs.k = mo.Variable(0.3, display_name="K") + m.inputs.quart = mo.Variable(3, display_name="Quart") + m.data = mo.MultiVariable("Data", excel_props={"tab": True}) + m.data.v = mo.Variable([10.0, 20.0, 30.0, 40.0, 50.0], display_name="V", unit="USD") + m.data.u = mo.Variable([40.0, 10.0, 30.0, 20.0, 50.0], display_name="U", unit="USD") + m.data.ties = mo.Variable([5, 3, 5, 1, 3], display_name="Ties") + m.comps = mo.MultiVariable("Comps", excel_props={"tab": True, "orient": "records"}) + with m.comps as c: + for key, x in (("a", 8.0), ("b", 12.0), ("c", 10.0), ("d", 9.0), ("e", 10.0)): + setattr(c, key, Comp(multiple=x, display_name=key.upper())) + m.s = mo.MultiVariable("Summary", excel_props={"tab": True}) + return m + + +def _peers(m): + c = m.comps + return [c.a.multiple, c.b.multiple, c.c.multiple, c.d.multiple, c.e.multiple] + + +# ══════════════════════════════════════════════════════════════════════ +# Rows are ranges, the names as the file stores them +# ══════════════════════════════════════════════════════════════════════ + + +class TestRowsAreRanges: + def test_every_function_over_rows(self, tmp_path): + m = _model() + d, i = m.data, m.inputs + with m.s as s: + s.median = MEDIAN(d.v).set_display_name("median") + s.stdev_s = STDEV_S(d.v).set_display_name("stdev_s") + s.stdev_p = STDEV_P(d.v).set_display_name("stdev_p") + s.var_s = VAR_S(d.v).set_display_name("var_s") + s.var_p = VAR_P(d.v).set_display_name("var_p") + s.stdev = STDEV(d.v).set_display_name("stdev") + s.stdevp = STDEVP(d.v).set_display_name("stdevp") + s.var = VAR(d.v).set_display_name("var") + s.varp = VARP(d.v).set_display_name("varp") + s.p_inc = PERCENTILE_INC(d.v, i.k).set_display_name("p_inc") + s.p_exc = PERCENTILE_EXC(d.v, 0.4).set_display_name("p_exc") + s.q_inc = QUARTILE_INC(d.v, i.quart).set_display_name("q_inc") + s.q_exc = QUARTILE_EXC(d.v, 1).set_display_name("q_exc") + s.perc = PERCENTILE(d.v, 0.3).set_display_name("perc") + s.quart = QUARTILE(d.v, 1).set_display_name("quart") + s.large = LARGE(d.v, 2).set_display_name("large") + s.small = SMALL(d.v, 2).set_display_name("small") + s.rank_eq = RANK_EQ(3, d.ties).set_display_name("rank_eq") + s.rank_avg = RANK_AVG(3, d.ties, 1).set_display_name("rank_avg") + s.rank = RANK(i.quart, d.ties).set_display_name("rank") + rows = _rows(_book(m, tmp_path)["Summary"]) + assert rows["median"][0] == "=MEDIAN(Data!B2:F2)" + assert rows["stdev_s"][0] == "=_xlfn.STDEV.S(Data!B2:F2)" + assert rows["stdev_p"][0] == "=_xlfn.STDEV.P(Data!B2:F2)" + assert rows["var_s"][0] == "=_xlfn.VAR.S(Data!B2:F2)" + assert rows["var_p"][0] == "=_xlfn.VAR.P(Data!B2:F2)" + assert rows["stdev"][0] == "=STDEV(Data!B2:F2)" + assert rows["stdevp"][0] == "=STDEVP(Data!B2:F2)" + assert rows["var"][0] == "=VAR(Data!B2:F2)" + assert rows["varp"][0] == "=VARP(Data!B2:F2)" + assert rows["p_inc"][0] == "=_xlfn.PERCENTILE.INC(Data!B2:F2, Inputs!B1)" + assert rows["p_exc"][0] == "=_xlfn.PERCENTILE.EXC(Data!B2:F2, 0.4)" + assert rows["q_inc"][0] == "=_xlfn.QUARTILE.INC(Data!B2:F2, Inputs!B2)" + assert rows["q_exc"][0] == "=_xlfn.QUARTILE.EXC(Data!B2:F2, 1)" + assert rows["perc"][0] == "=PERCENTILE(Data!B2:F2, 0.3)" + assert rows["quart"][0] == "=QUARTILE(Data!B2:F2, 1)" + assert rows["large"][0] == "=LARGE(Data!B2:F2, 2)" + assert rows["small"][0] == "=SMALL(Data!B2:F2, 2)" + assert rows["rank_eq"][0] == "=_xlfn.RANK.EQ(3, Data!B4:F4)" + assert rows["rank_avg"][0] == "=_xlfn.RANK.AVG(3, Data!B4:F4, 1)" + assert rows["rank"][0] == "=RANK(Inputs!B2, Data!B4:F4)" + # The values the cells cache are the engine's. + assert (m.s.median.value, m.s.var_s.value, m.s.p_inc.value) == (30.0, 250.0, 22.0) + assert (m.s.q_inc.value, m.s.large.value, m.s.rank_avg.value) == (40.0, 40.0, 2.5) + + def test_rows_are_one_call_per_period_cells_and_typed_values_arguments(self, tmp_path): + m = _model() + d, c = m.data, m.comps + with m.s as s: + s.two = MEDIAN(d.v, d.u).set_display_name("two") + s.typed = STDEV_S(d.v, 5, True, "3").set_display_name("typed") + s.cells = MEDIAN(c.a.multiple, c.c.multiple, 7.5).set_display_name("cells") + s.half = (MEDIAN(d.v) / 2).set_display_name("half") + rows = _rows(_book(m, tmp_path)["Summary"]) + # The scalar lines take column B; the periods move to C. + assert rows["two"][1:] == [f"=MEDIAN(Data!{col}2, Data!{col}3)" for col in "BCDEF"] + assert rows["typed"][1:] == [ + f'=_xlfn.STDEV.S(Data!{col}2, 5, TRUE, "3")' for col in "BCDEF"] + assert rows["cells"][0] == "=MEDIAN(Comps!B2, Comps!B4, 7.5)" + assert rows["half"][0] == "=MEDIAN(Data!B2:F2) / 2" + assert m.s.two.value == [25.0, 15.0, 30.0, 30.0, 50.0] + + def test_on_the_rows_own_sheet(self, tmp_path): + m = _model() + with m.data as d: + d.mid = MEDIAN(d.v).set_display_name("mid") + d.top = LARGE(d.v, 1).set_display_name("top") + rows = _rows(_book(m, tmp_path)["Data"]) + # The scalar lines take column B; the periods move to C. + assert rows["mid"][0] == "=MEDIAN(C2:G2)" + assert rows["top"][0] == "=LARGE(C2:G2, 1)" + + +# ══════════════════════════════════════════════════════════════════════ +# One field down the records is a vertical range +# ══════════════════════════════════════════════════════════════════════ + + +class TestRecordsAreVerticalRanges: + def test_a_field_down_the_records(self, tmp_path): + m = _model() + peers = _peers(m) + with m.s as s: + s.median = MEDIAN(peers).set_display_name("median") + s.q1 = PERCENTILE_INC(peers, m.inputs.k).set_display_name("q1") + s.q3 = QUARTILE_EXC(peers, 3).set_display_name("q3") + s.sd = STDEV_S(peers).set_display_name("sd") + s.top = LARGE(peers, 1).set_display_name("top") + s.rank_a = RANK_AVG(m.comps.c.multiple, peers).set_display_name("rank_a") + rows = _rows(_book(m, tmp_path)["Summary"]) + assert rows["median"][0] == "=MEDIAN(Comps!B2:B6)" + assert rows["q1"][0] == "=_xlfn.PERCENTILE.INC(Comps!B2:B6, Inputs!B1)" + assert rows["q3"][0] == "=_xlfn.QUARTILE.EXC(Comps!B2:B6, 3)" + assert rows["sd"][0] == "=_xlfn.STDEV.S(Comps!B2:B6)" + assert rows["top"][0] == "=LARGE(Comps!B2:B6, 1)" + assert rows["rank_a"][0] == "=_xlfn.RANK.AVG(Comps!B4, Comps!B2:B6)" + assert (m.s.median.value, m.s.top.value, m.s.rank_a.value) == (10.0, 12.0, 2.5) + assert str(m.s.median._unit) == "x" and m.s.rank_a._unit is None + assert check_compat(m, "excel") == [] + + def test_each_record_ranked_among_its_peers(self, tmp_path): + m = _model() + peers = _peers(m) + with m.comps as c: + for rec in (c.a, c.b, c.c, c.d, c.e): + rec.rank = RANK_EQ(rec.multiple, peers).set_display_name("Rank") + ws = _book(m, tmp_path)["Comps"] + header = [cell.value for cell in ws[1]] + at = header.index("Rank") + ranks = {row[0].value: row[at].value for row in ws.iter_rows(min_row=2)} + assert ranks == { + "A": "=_xlfn.RANK.EQ(B2, B2:B6)", "B": "=_xlfn.RANK.EQ(B3, B2:B6)", + "C": "=_xlfn.RANK.EQ(B4, B2:B6)", "D": "=_xlfn.RANK.EQ(B5, B2:B6)", + "E": "=_xlfn.RANK.EQ(B6, B2:B6)", + } + assert [r.rank.value for r in (c.a, c.b, c.c, c.d, c.e)] == [5, 1, 2, 4, 2] + assert check_compat(m, "excel") == [] + + +# ══════════════════════════════════════════════════════════════════════ +# A series parameter is one call per period over the same range +# ══════════════════════════════════════════════════════════════════════ + + +class TestOneCallPerPeriod: + def test_a_series_k_and_each_period_ranked_among_its_row(self, tmp_path): + m = _model() + with m.data as d: + d.ks = mo.Variable([0.0, 0.25, 0.5, 0.75, 1.0], display_name="Ks") + d.pct = PERCENTILE_INC(d.v, d.ks).set_display_name("Pct") + d.rank = RANK_EQ(d.u, d.u).set_display_name("Rank") + rows = _rows(_book(m, tmp_path)["Data"]) + assert rows["Pct"][:5] == [f"=_xlfn.PERCENTILE.INC(B2:F2, {col}5)" for col in "BCDEF"] + assert rows["Rank"][:5] == [f"=_xlfn.RANK.EQ({col}3, B3:F3)" for col in "BCDEF"] + assert d.pct.value == [10.0, 20.0, 30.0, 40.0, 50.0] + assert d.rank.value == [2, 5, 3, 4, 1] + + +# ══════════════════════════════════════════════════════════════════════ +# Cells that form no run carry the value +# ══════════════════════════════════════════════════════════════════════ + + +class TestNoRunShipsTheValue: + def test_one_range_over_a_gap_two_sheets_or_a_literal(self, tmp_path): + m = _model() + c = m.comps + with m.s as s: + s.gap = PERCENTILE_INC([c.a.multiple, c.c.multiple], 0.5).set_display_name("gap") + s.sheets = LARGE([c.a.multiple, m.inputs.k], 1).set_display_name("sheets") + s.literal = RANK_EQ(8.0, [c.a.multiple, 7.0]).set_display_name("literal") + s.backwards = SMALL([c.c.multiple, c.b.multiple, c.a.multiple], + 1).set_display_name("backwards") + rows = _rows(_book(m, tmp_path)["Summary"]) + assert rows["gap"][0] == 9 and rows["sheets"][0] == 8 + assert rows["literal"][0] == 1 and rows["backwards"][0] == 8 + issues = check_compat(m, "excel") + assert [(i.kind, i.qpath) for i in issues] == [ + ("value", m.s.gap.id), ("value", m.s.sheets.id), + ("value", m.s.literal.id), ("value", m.s.backwards.id)] + + def test_a_reduction_over_a_gap_carries_its_value_over_cells_it_lists_them( + self, tmp_path): + # A list of lines is ONE range: cells that form no run carry the + # value; the lines as separate arguments are a live comma list. + m = _model() + c = m.comps + with m.s as s: + s.gap = MEDIAN([c.a.multiple, c.c.multiple]).set_display_name("gap") + s.args = MEDIAN(c.a.multiple, c.c.multiple).set_display_name("args") + rows = _rows(_book(m, tmp_path)["Summary"]) + assert rows["gap"][0] == 9 + assert rows["args"][0] == "=MEDIAN(Comps!B2, Comps!B4)" + assert [(i.kind, i.qpath) for i in check_compat(m, "excel")] == [ + ("value", m.s.gap.id)] + + def test_a_row_split_by_a_totals_column(self, tmp_path): + m = mo.Model("m", display_name="M", default_grain="month", default_start="2026-01", + default_periods=6, + default_excel_view=mo.ExcelView(timeline={"totals": ["quarter"]})) + with m: + m.data = mo.MultiVariable("Data", excel_props={"tab": True}) + m.data.v = mo.Variable([10.0, 20.0, 30.0, 40.0, 50.0, 60.0], display_name="V", + regrain=mo.up("sum")) + m.s = mo.MultiVariable("Summary", excel_props={"tab": True}) + m.s.mid = MEDIAN(m.data.v).set_display_name("mid") + m.s.p = PERCENTILE_INC(m.data.v, 0.5).set_display_name("p") + rows = _rows(_book(m, tmp_path)["Summary"]) + # Never =MEDIAN(Data!B3:H3), which would read the Q1 total too. + assert rows["mid"][0] == 35 and rows["p"][0] == 35 + assert [(i.kind, i.qpath) for i in check_compat(m, "excel")] == [ + ("value", m.s.mid.id), ("value", m.s.p.id)] + + def test_a_row_the_book_has_no_cells_for(self, tmp_path): + m = _model() + with m.s as s: + s.scaled = MEDIAN(m.data.v * 2.0).set_display_name("scaled") + with pytest.warns(mo.CrossScopeReferenceWarning, match="MEDIAN"): + rows = _rows(_book(m, tmp_path)["Summary"]) + assert rows["scaled"][0] == 60 + + +# ══════════════════════════════════════════════════════════════════════ +# LibreOffice recalculates these functions to the engine's numbers +# ══════════════════════════════════════════════════════════════════════ + + +def _soffice(): + for candidate in ("soffice", "libreoffice"): + found = shutil.which(candidate) + if found: + return found + mac = Path("/Applications/LibreOffice.app/Contents/MacOS/soffice") + return str(mac) if mac.exists() else None + + +def _recalculated(model, tmp_path): + """Every sheet after LibreOffice recalculates the written workbook + (conftest seeds the profile to recalculate on load).""" + soffice = _soffice() + if soffice is None: + pytest.skip("LibreOffice not installed") + path = tmp_path / "statistics.xlsx" + model.to_excel(path) + subprocess.run( + [soffice, f"-env:UserInstallation={(tmp_path / 'prof').as_uri()}", + "--headless", "--convert-to", "xlsx", "--outdir", + str(tmp_path / "calc"), str(path)], + check=True, capture_output=True, + ) + return load_workbook(tmp_path / "calc" / "statistics.xlsx", data_only=True) + + +def _serial(value): + """A date as the cell holds it, so a recalculated date (read back as + a datetime) compares with the engine's date.""" + if isinstance(value, dt.datetime): + return (value - dt.datetime(1899, 12, 30)) / dt.timedelta(days=1) + if isinstance(value, dt.date): + return (value - dt.date(1899, 12, 30)).days + return value + + +class Peer(mo.MultiVariableClass): + def compute(self, rev): + self.rev = mo.Variable(rev, display_name="Revenue", unit="USD") + + +#: The per-period statistics written on Data, read back period by period. +_DATA_PER_PERIOD = ( + "pct", "rank_row", "rank_ties", "kth", + "med_pp", "med3_pp", "med_list_pp", "med_typed_pp", "med_mixed_pp", "med_err_pp", + "med_dates_pp", "sd_pp", "stdevp_pp", "var_pp", "var_list_pp", + "p_pp", "p_k_pp", "p_exc_pp", "q_pp", "large_pp", "small_pp", "rank_pp", + "rank_avg_pp", "large_err_pp", "p_none_pp", +) + + +def _parity_model(): + """Every function and its edges: rows on Data (with the per-period + calls — over the same range, and over sibling rows one period at a + time), the peers on Comps (with a rank per record), a records table of + yearly lines on Peers (with each year's rank per record), one cell per + call on Summary — a row of them where the call is one per period.""" + m = _model() + i, d, c = m.inputs, m.data, m.comps + peers = _peers(m) + m.peers = mo.MultiVariable("Peers", excel_props={"tab": True, "orient": "records"}) + with m.peers as pp: + pp.a = Peer(rev=[10.0, 30.0, 25.0, None, 5.0], display_name="PA") + pp.b = Peer(rev=[20.0, 10.0, 45.0, 7.0, 5.0], display_name="PB") + pp.c = Peer(rev=[35.0, 40.0, 10.0, 8.0, 5.0], display_name="PC") + pp.d = Peer(rev=[15.0, 40.0, 30.0, 9.0, 5.0], display_name="PD") + revs = [r.rev for r in (pp.a, pp.b, pp.c, pp.d)] + three = peers[:3] # 8, 12, 10 + i.blank = mo.Variable(None, display_name="Blank") + i.text_k = mo.Variable("0.5", display_name="TextK") + i.text_two = mo.Variable("2", display_name="TextTwo") + i.err = mo.Variable("#DIV/0!", display_name="Err") + i.na = mo.Variable("#N/A", display_name="Na") + i.flag = mo.Variable(True, display_name="Flag") + d.zero = mo.Variable([1.0, 0.0, 1.0, 1.0, 1.0], display_name="Zero") + d.err = (d.v / d.zero).set_display_name("Err") # 10, #DIV/0!, 30, 40, 50 + d.none = mo.Variable(["a", None, "b", None, "c"], display_name="None") + d.mixed = mo.Variable([3.0, None, "x", 1.0, 4.0], display_name="Mixed") + d.numtext = mo.Variable(["10", "20", 5.0, None, "x"], display_name="Numtext") + d.bools = mo.Variable([3.0, True, 1.0, 4.0, False], display_name="Bools") + d.dates = mo.Variable([dt.date(2026, 1, 31), dt.date(2026, 3, 31), dt.date(2026, 2, 28), + dt.date(2026, 2, 27), dt.date(2025, 12, 31)], display_name="Dates") + d.ks = mo.Variable([0.0, 0.25, 0.5, 0.75, 1.0], display_name="Ks") + # One call per period over the same range. + d.pct = PERCENTILE_INC(d.v, d.ks).set_display_name("Pct") + d.rank_row = RANK_EQ(d.u, d.u).set_display_name("RankRow") + d.rank_ties = RANK_AVG(d.ties, d.ties, 1).set_display_name("RankTies") + d.kth = LARGE(d.u, d.ties).set_display_name("Kth") + # Several rows, one call per period over each period's cells. + rows = [d.v, d.u, d.ties] # rows 2-4: one run per column + d.med_pp = MEDIAN(d.v, d.u).set_display_name("MedPP") + d.med3_pp = MEDIAN(d.v, d.u, d.ties).set_display_name("Med3PP") + d.med_list_pp = MEDIAN(rows).set_display_name("MedListPP") + d.med_typed_pp = MEDIAN(d.v, [1.0, 50.0, 25.0, 0.0, 100.0]).set_display_name("MedTypedPP") + d.med_mixed_pp = MEDIAN(d.mixed, d.v).set_display_name("MedMixedPP") + d.med_err_pp = MEDIAN(d.err, d.u).set_display_name("MedErrPP") + d.med_dates_pp = MEDIAN(d.dates, dt.date(2026, 2, 1)).set_display_name("MedDatesPP") + d.sd_pp = STDEV_S(d.v, d.u, 5.0).set_display_name("SdPP") + d.stdevp_pp = STDEVP(d.v, d.u).set_display_name("StdevpPP") + d.var_pp = VAR(d.v, d.ties).set_display_name("VarPP") + d.var_list_pp = VAR_P(rows).set_display_name("VarListPP") + d.p_pp = PERCENTILE_INC(rows, 0.25).set_display_name("PPP") + d.p_k_pp = PERCENTILE_INC(rows, d.ks).set_display_name("PKPP") + d.p_exc_pp = PERCENTILE_EXC(rows, 0.5).set_display_name("PExcPP") + d.q_pp = QUARTILE_INC(rows, 3).set_display_name("QPP") + d.large_pp = LARGE(rows, 1).set_display_name("LargePP") + d.small_pp = SMALL(rows, 2).set_display_name("SmallPP") + d.rank_pp = RANK_EQ(d.u, rows).set_display_name("RankPP") + d.rank_avg_pp = RANK_AVG(d.ties, rows, 1).set_display_name("RankAvgPP") + d.large_err_pp = LARGE([d.zero, d.err], 1).set_display_name("LargeErrPP") # rows 5-6 + d.p_none_pp = PERCENTILE_INC([d.none, d.mixed], 0.5).set_display_name("PNonePP") # 7-8 + # Each record among its peers. + with m.comps: + for rec in (c.a, c.b, c.c, c.d, c.e): + rec.rank = RANK_EQ(rec.multiple, peers).set_display_name("Rank") + rec.rank_avg = RANK_AVG(rec.multiple, peers, 1).set_display_name("RankAvg") + # Each record's yearly line among its peers', year by year. + with m.peers: + for rec in (pp.a, pp.b, pp.c, pp.d): + rec.rank = RANK_EQ(rec.rev, revs).set_display_name("Rank") + rec.rank_avg = RANK_AVG(rec.rev, revs, 1).set_display_name("RankAvg") + with m.s as s: + # MEDIAN + s.med = MEDIAN(d.v) + s.med_even = MEDIAN(d.u, 60.0) + s.med_two = MEDIAN(d.v, d.ties) + s.med_mixed = MEDIAN(d.mixed) + s.med_numtext = MEDIAN(d.numtext) + s.med_typed = MEDIAN(1, True, 5) + s.med_none = MEDIAN(1, None, 5) + s.med_text = MEDIAN(1, "abc") + s.med_err = MEDIAN(d.err) + s.med_err_text = MEDIAN(i.err, "x") + s.med_text_err = MEDIAN("x", i.err) + s.med_two_errors = MEDIAN(i.err, i.na) + s.med_dates = MEDIAN(d.dates) + s.med_peers = MEDIAN(peers) + s.med_cells = MEDIAN(c.a.multiple, c.c.multiple) + s.med_empty = MEDIAN(d.none) + s.med_numtext_typed = MEDIAN(1, "3") + s.med_bools = MEDIAN(d.bools) + s.med_flag = MEDIAN(i.flag, 4) + # The spreads + s.sd_s = STDEV_S(d.v) + s.sd_p = STDEV_P(d.v) + s.var_s = VAR_S(d.v) + s.var_p = VAR_P(d.v) + s.stdev = STDEV(d.u) + s.stdevp = STDEVP(d.u) + s.var = VAR(d.u) + s.varp = VARP(d.u) + s.var_two = VAR_S(d.v, d.ties) + s.sd_mixed = STDEV_S(d.mixed) + s.sd_typed = STDEV_S(1, True, 5) + s.sd_big = STDEV_S(1e15 + 1, 1e15 + 2, 1e15 + 3) + s.var_small = VAR_S(0.1, 0.2, 0.3) + s.sd_s_one = STDEV_S(c.a.multiple) + s.sd_p_one = STDEV_P(c.a.multiple) + s.var_s_empty = VAR_S(d.none) + s.var_p_empty = VAR_P(d.none) + s.sd_err = STDEV_S(d.err) + s.sd_dates = STDEV_S(d.dates) + s.var_peers = VAR_P(peers) + s.sd_numtext_typed = STDEV_S(1, "3", 5) + s.sd_text_typed = STDEV_S(1, "abc", 5) + s.sd_bools = STDEV_S(d.bools) + s.sd_two_errors = STDEV_S(i.err, i.na) + # Percentiles + s.p_inc = PERCENTILE_INC(d.v, i.k) + s.p_inc_0 = PERCENTILE_INC(d.v, 0) + s.p_inc_1 = PERCENTILE_INC(d.v, 1) + s.p_inc_unsorted = PERCENTILE_INC(d.u, 0.3) + s.p_inc_text_k = PERCENTILE_INC(d.v, "0.5") + s.p_inc_cell_text_k = PERCENTILE_INC(d.v, i.text_k) + s.p_inc_true = PERCENTILE_INC(d.v, True) + s.p_inc_blank = PERCENTILE_INC(d.v, i.blank) + s.p_inc_err = PERCENTILE_INC(d.err, 0.5) + s.p_inc_kerr = PERCENTILE_INC(d.v, i.err) + s.p_inc_kerr_err = PERCENTILE_INC(d.err, i.na) + s.p_inc_dates = PERCENTILE_INC(d.dates, 0.25) + s.p_inc_peers = PERCENTILE_INC(peers, i.k) + s.p_inc_one = PERCENTILE_INC(c.a.multiple, 0.3) + s.p_inc_mixed = PERCENTILE_INC(d.mixed, 0.25) + s.p_exc = PERCENTILE_EXC(d.v, 0.4) + s.p_exc_lo = PERCENTILE_EXC(three, 0.25) + s.p_exc_hi = PERCENTILE_EXC(three, 0.75) + s.p_exc_near = PERCENTILE_EXC(d.v, 0.167) + s.perc = PERCENTILE(d.u, 0.3) + s.p_inc_bad = PERCENTILE_INC(d.v, 1.1) + s.p_inc_empty = PERCENTILE_INC(d.none, 0.5) + s.p_inc_err_badk = PERCENTILE_INC(d.err, 2) + s.p_exc_below = PERCENTILE_EXC(d.v, 0.16) + s.p_exc_0 = PERCENTILE_EXC(d.v, 0) + s.p_exc_empty = PERCENTILE_EXC(d.none, 0.5) + # Quartiles + s.q_inc_1 = QUARTILE_INC(d.v, 1) + s.q_inc_2 = QUARTILE_INC(d.u, 2) + s.q_inc_cell = QUARTILE_INC(d.v, i.quart) + s.q_inc_15 = QUARTILE_INC(d.v, 1.5) + s.q_inc_45 = QUARTILE_INC(d.v, 4.5) + s.q_exc_1 = QUARTILE_EXC(d.v, 1) + s.q_exc_3 = QUARTILE_EXC(d.u, 3) + s.q_exc_35 = QUARTILE_EXC(d.v, 3.5) + s.q_exc_two = QUARTILE_EXC([c.a.multiple, c.b.multiple], 2) + s.q_inc_err = QUARTILE_INC(d.err, 1) + s.q_inc_kerr = QUARTILE_INC(d.err, i.na) + s.quart = QUARTILE(d.u, 1) + s.q_inc_neg = QUARTILE_INC(d.v, -0.5) + s.q_inc_5 = QUARTILE_INC(d.v, 5) + s.q_exc_0 = QUARTILE_EXC(d.v, 0) + s.q_exc_one = QUARTILE_EXC(c.a.multiple, 1) + s.q_inc_empty = QUARTILE_INC(d.none, 1) + # The k-th largest and smallest + s.large_1 = LARGE(d.u, 1) + s.large_25 = LARGE(d.u, 2.5) + s.large_05 = LARGE(d.u, 0.5) + s.small_25 = SMALL(d.u, 2.5) + s.small_55 = SMALL(d.u, 5.5) + s.large_ties = LARGE(d.ties, 2) + s.large_true = LARGE(d.v, True) + s.large_err = LARGE(d.err, 1) + s.large_kerr = LARGE(d.err, i.na) + s.large_err_badk = LARGE(d.err, 0) + s.large_dates = LARGE(d.dates, 2) + s.small_dates = SMALL(d.dates, 1) + s.large_peers = LARGE(peers, 2) + s.small_peers = SMALL(peers, 1) + s.small_mixed = SMALL(d.mixed, 2) + s.large_0 = LARGE(d.v, 0) + s.large_55 = LARGE(d.v, 5.5) + s.small_6 = SMALL(d.v, 6) + s.small_05 = SMALL(d.v, 0.5) + s.large_empty = LARGE(d.none, 1) + s.large_text_k = LARGE(d.v, "2") + s.large_cell_text_k = LARGE(d.v, i.text_two) + s.small_cell_text_k = SMALL(d.v, i.text_two) + s.large_bools = LARGE(d.bools, 4) + # Ranks + s.rank_desc = RANK_EQ(3, d.ties) + s.rank_asc = RANK_EQ(3, d.ties, 1) + s.rank_avg = RANK_AVG(3, d.ties) + s.rank_avg_asc = RANK_AVG(5, d.ties, 1) + s.rank_old = RANK(5, d.ties) + s.rank_absent = RANK_EQ(4, d.ties) + s.rank_err = RANK_EQ(1, d.err) + s.rank_text_num = RANK_EQ("3", d.ties) + s.rank_true = RANK_EQ(True, d.ties) + s.rank_blank = RANK_EQ(i.blank, d.ties) + s.rank_order_true = RANK_EQ(3, d.ties, True) + s.rank_order_text = RANK_EQ(3, d.ties, "1") + s.rank_order_cell = RANK_EQ(3, d.ties, i.flag) + s.rank_num_err = RANK_EQ(i.err, d.v) + s.rank_order_err = RANK_EQ(10, d.v, i.err) + s.rank_num_err_ref_err = RANK_EQ(i.na, d.err) + s.rank_dates = RANK_EQ(dt.date(2026, 2, 28), d.dates) + s.rank_mixed = RANK_EQ(1, d.mixed) + s.rank_peer = RANK_AVG(c.c.multiple, peers) + s.rank_none = RANK_EQ(1, d.none) + s.rank_bools = RANK_EQ(0, d.bools) + # A records table of yearly lines: each year's statistic. + s.peer_med = MEDIAN(revs) + s.peer_q1 = PERCENTILE_INC(revs, i.k) + s.peer_sd = STDEV_S(revs) + s.peer_top = LARGE(revs, 2) + for name, line in m.s.walk(): + if isinstance(line, mo.Variable): + line.set_display_name(name) + return m + + +#: Where LibreOffice's answer is not the engine's: what LibreOffice 25.2 +#: writes instead, pinned by line. Each is where it departs from Excel, +#: whose rules the engine computes: a ``k`` or a quart out of its range +#: and a range without numbers are LibreOffice's "invalid argument" / +#: "no value", written as #VALUE! where Excel says #NUM! (a rank among no +#: numbers: #N/A); numeric text typed into MEDIAN or a spread is #VALUE! +#: there, where Excel counts it, and so is numeric text as LARGE's or +#: SMALL's k, typed or in a cell — LibreOffice reads that k from a number +#: only, the engine as Excel reads a value argument (Microsoft documents +#: no rule for this k; unmeasured); a boolean in a range is a number +#: there (TRUE 1, FALSE 0), where Excel skips it; a negative fractional +#: quart is rounded down there (-0.5 is -1, out of range) where Excel +#: truncates it (to 0, the smallest); and LibreOffice's spreads read their +#: arguments from the last, so of two errors the later one shows. +_LIBREOFFICE_INSTEAD = { + "med_empty": "#VALUE!", "p_inc_bad": "#VALUE!", "p_inc_empty": "#VALUE!", + "p_inc_err_badk": "#VALUE!", "p_exc_below": "#VALUE!", "p_exc_0": "#VALUE!", + "p_exc_empty": "#VALUE!", "q_inc_5": "#VALUE!", "q_exc_0": "#VALUE!", + "q_exc_one": "#VALUE!", "q_inc_empty": "#VALUE!", "large_0": "#VALUE!", + "large_55": "#VALUE!", "small_6": "#VALUE!", "small_05": "#VALUE!", + "large_empty": "#VALUE!", "rank_none": "#VALUE!", + "med_numtext_typed": "#VALUE!", "sd_numtext_typed": "#VALUE!", + "large_text_k": "#VALUE!", "large_cell_text_k": "#VALUE!", + "small_cell_text_k": "#VALUE!", + "med_bools": 1, "med_flag": 2.5, "sd_bools": 1.6431676725155, "large_bools": 1, + "rank_bools": 5, + "q_inc_neg": "#VALUE!", + "sd_two_errors": "#N/A", + "p_none_pp[1]": "#VALUE!", "p_none_pp[2]": "#VALUE!", +} + + +def _blocks(ws): + """A records sheet of yearly lines: ``{(field, record): its cells}`` — + each field a block of rows under its own label, a row per record.""" + out, field = {}, None + for row in ws.iter_rows(min_row=2): + label = row[0].value + if label is None: + continue + cells = [c.value for c in row[1:]] + if all(v is None for v in cells): + field = str(label).split(" (")[0] + else: + out[(field, str(label))] = cells + return out + + +def _cells(m, book): + """Every function cell as ``(where, engine value, LibreOffice value)``.""" + # The scalar lines take Summary's column B, the periods C onwards. + summary = _rows(book["Summary"]) + for name, line in m.s.walk(): + if isinstance(line, mo.Variable): + if isinstance(line.value, list): + for period, (ours, theirs) in enumerate(zip(line.value, summary[name][1:6])): + yield f"{name}[{period}]", ours, theirs + else: + yield name, line.value, summary[name][0] + data = _rows(book["Data"]) + for name in _DATA_PER_PERIOD: + line = getattr(m.data, name) + for period, (ours, theirs) in enumerate(zip(line.value, data[line.display_name][:5])): + yield f"{name}[{period}]", ours, theirs + ws = book["Comps"] + header = [cell.value for cell in ws[1]] + for row in ws.iter_rows(min_row=2): + rec = getattr(m.comps, str(row[0].value).lower()) + for field, label in (("rank", "Rank"), ("rank_avg", "RankAvg")): + yield (f"{row[0].value}.{field}", getattr(rec, field).value, + row[header.index(label)].value) + blocks = _blocks(book["Peers"]) + for key in ("a", "b", "c", "d"): + rec = getattr(m.peers, key) + for field, label in (("rank", "Rank"), ("rank_avg", "RankAvg")): + cells = blocks[(label, rec.display_name)] + for period, (ours, theirs) in enumerate(zip(getattr(rec, field).value, cells[:5])): + yield f"{rec.display_name}.{field}[{period}]", ours, theirs + + +def _same(ours, theirs): + if isinstance(ours, str) or isinstance(theirs, str): + return ours == theirs + return _serial(theirs) == pytest.approx(_serial(ours), rel=1e-9, abs=1e-12) + + +@pytest.mark.slow +def test_the_workbook_recalculates_to_the_python_values(tmp_path): + m = _parity_model() + book = _recalculated(m, tmp_path) + cells = list(_cells(m, book)) + assert len(cells) == 338 + names = {name for name, _ours, _theirs in cells} + assert set(_LIBREOFFICE_INSTEAD) <= names + for name, ours, theirs in cells: + expected = _LIBREOFFICE_INSTEAD.get(name, ours) + assert _same(expected, theirs), (name, ours, theirs) + # A few the comparison would pass for the wrong reason. + s = m.s + assert (s.med_dates.value, s.large_dates.value) == (dt.date(2026, 2, 27), dt.date(2026, 2, 28)) + assert (s.large_25.value, s.small_25.value, s.large_05.value) == (30.0, 20.0, 50.0) + assert (s.rank_num_err_ref_err.value, s.p_inc_kerr_err.value) == ("#DIV/0!", "#N/A") + assert (s.large_kerr.value, s.large_err_badk.value) == ("#N/A", "#DIV/0!") + assert (s.med_two_errors.value, s.sd_two_errors.value) == ("#DIV/0!", "#DIV/0!") + assert (s.large_cell_text_k.value, s.small_cell_text_k.value) == (40.0, 20.0) + # One call per period, never one number over every cell. + d = m.data + assert d.med_pp.value == [25.0, 15.0, 30.0, 30.0, 50.0] + assert d.p_pp.value == [7.5, 6.5, 17.5, 10.5, 26.5] + assert d.rank_pp.value == [1, 2, 1, 2, 1] + assert s.peer_med.value == [17.5, 35.0, 27.5, 8.0, 5.0] + assert m.peers.a.rank.value == [4, 3, 3, "#N/A", 1] + + +@pytest.mark.slow +@pytest.mark.xfail(strict=True, reason=( + "LibreOffice writes a k or a quart out of range and a range without numbers as #VALUE! " + "(Excel: #NUM!, a rank #N/A), refuses numeric text typed into MEDIAN or a spread " + "and as LARGE's or SMALL's k (typed or in a cell), counts a boolean in a range, " + "rounds a negative quart down and reads a spread's arguments from the last; the " + "engine keeps Excel's rules — see _LIBREOFFICE_INSTEAD." +)) +def test_where_libreoffice_departs_from_excel(tmp_path): + m = _parity_model() + book = _recalculated(m, tmp_path) + departing = [(name, ours, theirs) for name, ours, theirs in _cells(m, book) + if name in _LIBREOFFICE_INSTEAD and not _same(ours, theirs)] + assert departing == [], departing diff --git a/tests/test_excel_text.py b/tests/test_excel_text.py new file mode 100644 index 0000000..45fdee5 --- /dev/null +++ b/tests/test_excel_text.py @@ -0,0 +1,339 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The text helpers write the spreadsheet's own calls — one per period +(``=LEFT(I!C2, 2)``), a scalar in one cell, ``TEXTJOIN`` over the row as +a range (``=_xlfn.TEXTJOIN(", ", TRUE, I!C2:E2)``) — and the cells, once +LibreOffice recalculates them, agree with the Python values. +""" + +from __future__ import annotations + +import logging +import shutil +import subprocess +import sys +from datetime import date, timedelta +from pathlib import Path + +import pytest +from openpyxl import load_workbook + +import modeleon as mo +from modeleon.core.errors import CrossScopeReferenceWarning +from modeleon.functions.text import ( + FIND, + LEFT, + MID, + REPT, + RIGHT, + SEARCH, + SUBSTITUTE, + TEXT, + TEXTJOIN, + TRIM, + VALUE, +) + +PERIODS = 3 + +#: Every line the model writes on sheet X. +LINES = ( + "left", "left1", "leftnum", "leftdate", "leftnums", "leftbig", "leftbig2", "leftspan", + "leftbool", "right", "mid", "midpast", "trim", "find", "find2", "findempty", "findempty2", + "search", "search2", "searchempty", "searchempty2", "tilde", "sub", "sub2", "value", "value2", + "valuebool", "valuebig", "valuetime", "valuetime2", "valuetime3", "text0", "text2", "textp", + "textg", "textgbig", "textz", "textpad", "textgroup", "textd", "textd2", "textd3", "textm", + "textb", "textlong", "textwide", "textempty", "rept", "reptbig", "lit", "join", "join2", + "joinx", "quote", "chain", "err", "blank", "blanklist", "blankfind", "blanksub", "blankrept", + "blankfmt", "blankjoin", "blanklen", "blankupper", "blankcat", +) + +#: Where the engine and LibreOffice part — where Excel and LibreOffice +#: part, the engine follows Excel — the engine's answer, then +#: LibreOffice's. Both are pinned so a change on either side is seen. +DIVERGENT = { + # a negative that rounds to zero keeps its sign + "text0": (["1235", "-0", "3"], ["1235", "0", "3"]), + # TRUE is the word, not 1 + "textb": (["TRUE", "FALSE", "TRUE"], ["1", "0", "1"]), + "leftbool": (["TR", "FA", "TR"], ["1", "0", "1"]), + # General text keeps 15 significant digits, writes a whole number out + # in full below 1E20, and a fraction while it fits 20 characters; + # LibreOffice keeps a 16th digit and writes three-digit exponents + "leftbig": (["1234567890123460", "123456789012346000", "1.23456789012346E-05"], + ["1234567890123456", "1.23456789012346E+017", "0.0000123456789012346"]), + "textgbig": (["1234567890123460", "123456789012346000", "1.23456789012346E-05"], + ["1.23456789012346E+15", "1.23456789012346E+17", "0.0000123456789012346"]), + # past those bounds E notation has two exponent digits at least; + # LibreOffice writes three, and turns to E notation below about 1E-14 + "leftbig2": (["1.23456789012346E+20", "0.000000000000001", "-1E-19"], + ["1.23456789012346E+020", "1E-015", "-1E-019"]), + # a number format shows 15 significant digits too; LibreOffice a 16th + "textgroup": (["1,234,567,890,123,460", "123,456,789,012,346,000", "0"], + ["1,234,567,890,123,456", "123,456,789,012,346,000", "0"]), + # empty find_text is found at start_num + "findempty": ([1, 1, "#VALUE!"], ["#VALUE!", "#VALUE!", "#VALUE!"]), + "searchempty2": ([2, 2, 2], ["#VALUE!", "#VALUE!", "#VALUE!"]), + # VALUE reads no locale: a currency sign is not a number, nor is TRUE + "value2": ([1234567, "#VALUE!", 46023], [1234567, 1234, 46023]), + "valuebool": ("#VALUE!", 1), + # nor a T between a date and its time, nor a negative time + "valuetime2": (["#VALUE!", "#VALUE!", 0], [46053.5, -0.0625, 0]), + # a time is hours, then minutes and seconds below 60; LibreOffice also + # reads a bare "12:" and "0:90" (90 minutes) — Excel not measured + "valuetime3": ([0.0625, "#VALUE!", "#VALUE!"], [0.0625, 0.5, 0.0625]), + # a number too large for a cell; LibreOffice clamps to the largest double + "valuebig": ("#NUM!", sys.float_info.max), + # a text longer than a cell holds (32,767 characters); LibreOffice has no limit + "reptbig": ("#VALUE!", "a" * 32768), +} + + +def _model(): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=PERIODS) + with m: + m.i = mo.MultiVariable("I") + m.i.names = mo.Variable(["Alpha Fund", "beta co", None], display_name="names") + m.i.codes = mo.Variable(["AB-123", "CD-45", "EF-6"], display_name="codes") + m.i.amounts = mo.Variable([1234.5678, -0.4, 2.5], display_name="amounts") + m.i.texts = mo.Variable(["1,234", "12%", "(1,234)"], display_name="texts") + m.i.dates = mo.Variable([date(2026, 1, 31), date(2026, 2, 28), date(2026, 12, 25)], + display_name="dates") + m.i.k = mo.Variable(3, display_name="k") + m.i.sep = mo.Variable(" | ", display_name="sep") + m.i.padded = mo.Variable(" a b ", display_name="padded") + m.i.zero = mo.Variable(0.0, display_name="zero") + m.i.bad = (m.i.amounts / m.i.zero).set_display_name("bad") + m.i.big = (m.i.amounts > 1).set_display_name("big") + m.i.nums = mo.Variable([1e15, 0.00001, 1 / 3], display_name="nums") + m.i.texts2 = mo.Variable(["1234,567", "$1,234", "2026-01-01"], display_name="texts2") + m.i.flag = mo.Variable(True, display_name="flag") + m.i.bigs = mo.Variable([1234567890123456, 1.2345678901234568e17, 1.2345678901234568e-5], + display_name="bigs") + m.i.times = mo.Variable(["1:30", "3:00 PM", "2026-01-31 18:00"], display_name="times") + m.i.times2 = mo.Variable(["2026-01-31T12:00", "-1:30", "12:00 AM"], display_name="times2") + m.i.prec = mo.Variable([1 / 3, 0.1 + 0.2, 2.675], display_name="prec") + m.i.span = mo.Variable(timedelta(hours=36), display_name="span") + m.i.bigs2 = mo.Variable([1.2345678901234568e20, 1e-15, -1e-19], display_name="bigs2") + m.i.times3 = mo.Variable(["+1:30", "12:", "0:90"], display_name="times3") + m.i.pads = mo.Variable([1234, 5, -1234.5], display_name="pads") + m.x = mo.MultiVariable("X") + m.x.left = LEFT(m.i.names, 2).set_display_name("left") + m.x.left1 = LEFT(m.i.names).set_display_name("left1") + m.x.leftnum = LEFT(m.i.amounts, 3).set_display_name("leftnum") + m.x.leftdate = LEFT(m.i.dates, 4).set_display_name("leftdate") + m.x.leftnums = LEFT(m.i.nums, 20).set_display_name("leftnums") + m.x.leftbig = LEFT(m.i.bigs, 25).set_display_name("leftbig") + m.x.leftbig2 = LEFT(m.i.bigs2, 25).set_display_name("leftbig2") + m.x.leftspan = LEFT(m.i.span, 3).set_display_name("leftspan") + m.x.leftbool = LEFT(m.i.big, 2).set_display_name("leftbool") + m.x.right = RIGHT(m.i.codes, m.i.k).set_display_name("right") + m.x.mid = MID(m.i.codes, 4, 2).set_display_name("mid") + m.x.midpast = MID(m.i.codes, 9, 2).set_display_name("midpast") + m.x.trim = TRIM(m.i.padded).set_display_name("trim") + m.x.find = FIND("-", m.i.codes).set_display_name("find") + m.x.find2 = FIND("a", m.i.names, 2).set_display_name("find2") + m.x.findempty = FIND("", m.i.names).set_display_name("findempty") + m.x.findempty2 = FIND("", "abc", 4).set_display_name("findempty2") + m.x.search = SEARCH("a*f", m.i.names).set_display_name("search") + m.x.search2 = SEARCH("A~?", "xa?b").set_display_name("search2") + m.x.searchempty = SEARCH("", "abc", 4).set_display_name("searchempty") + m.x.searchempty2 = SEARCH("", m.i.codes, 2).set_display_name("searchempty2") + m.x.tilde = SEARCH("~a", "xab").set_display_name("tilde") + m.x.sub = SUBSTITUTE(m.i.codes, "-", "").set_display_name("sub") + m.x.sub2 = SUBSTITUTE(m.i.names, "a", "A", 1).set_display_name("sub2") + m.x.value = VALUE(m.i.texts).set_display_name("value") + m.x.value2 = VALUE(m.i.texts2).set_display_name("value2") + m.x.valuebool = VALUE(m.i.flag).set_display_name("valuebool") + m.x.valuebig = VALUE("1e999").set_display_name("valuebig") + m.x.valuetime = VALUE(m.i.times).set_display_name("valuetime") + m.x.valuetime2 = VALUE(m.i.times2).set_display_name("valuetime2") + m.x.valuetime3 = VALUE(m.i.times3).set_display_name("valuetime3") + m.x.text0 = TEXT(m.i.amounts, "0").set_display_name("text0") + m.x.text2 = TEXT(m.i.amounts, "#,##0.00").set_display_name("text2") + m.x.textp = TEXT(m.i.amounts, "0.0%").set_display_name("textp") + m.x.textg = TEXT(m.i.amounts, "General").set_display_name("textg") + m.x.textgbig = TEXT(m.i.bigs, "General").set_display_name("textgbig") + m.x.textz = TEXT(m.i.k, "000").set_display_name("textz") + m.x.textpad = TEXT(m.i.pads, "#,##000000").set_display_name("textpad") + m.x.textgroup = TEXT(m.i.bigs, "#,##0").set_display_name("textgroup") + m.x.textd = TEXT(m.i.dates, "mmm yyyy").set_display_name("textd") + m.x.textd2 = TEXT(m.i.dates, "dd.mm.yyyy").set_display_name("textd2") + m.x.textd3 = TEXT(m.i.dates, "yyyy-mm-dd").set_display_name("textd3") + m.x.textm = TEXT(m.i.dates, "mmmmm").set_display_name("textm") + m.x.textb = TEXT(m.i.big, "0").set_display_name("textb") + m.x.textlong = TEXT(m.i.prec, "0.000000000000000000").set_display_name("textlong") + m.x.textwide = TEXT(123456789012345678.0, "0").set_display_name("textwide") + m.x.textempty = TEXT(m.i.amounts, "").set_display_name("textempty") + m.x.rept = REPT("#", m.i.k).set_display_name("rept") + m.x.reptbig = REPT("a", 32768).set_display_name("reptbig") + m.x.lit = LEFT(["ab", "cd", "ef"], 1).set_display_name("lit") + m.x.join = TEXTJOIN(", ", True, m.i.names).set_display_name("join") + m.x.join2 = TEXTJOIN(m.i.sep, False, m.i.codes, "end").set_display_name("join2") + m.x.joinx = TEXTJOIN(",", "x", m.i.names).set_display_name("joinx") + m.x.quote = SUBSTITUTE(m.i.names, "a", '"').set_display_name("quote") + m.x.chain = mo.UPPER(LEFT(m.i.names, 3)).set_display_name("chain") + m.x.err = LEFT(m.i.bad, 1).set_display_name("err") + # a literal blank where text is read goes in as "", not as 0 + m.x.blank = LEFT(None, 2).set_display_name("blank") + m.x.blanklist = LEFT(["ab", None, "cd"], 1).set_display_name("blanklist") + m.x.blankfind = FIND("0", None).set_display_name("blankfind") + m.x.blanksub = SUBSTITUTE("a0b", "0", None).set_display_name("blanksub") + m.x.blankrept = REPT(None, 3).set_display_name("blankrept") + m.x.blankfmt = TEXT(m.i.amounts, None).set_display_name("blankfmt") + m.x.blankjoin = TEXTJOIN("-", False, "a", None, "b").set_display_name("blankjoin") + m.x.blanklen = mo.LEN(None).set_display_name("blanklen") + m.x.blankupper = mo.UPPER(["ab", None, "cd"]).set_display_name("blankupper") + m.x.blankcat = mo.CONCAT("a", None, "b").set_display_name("blankcat") + return m + + +def _rows(ws): + out = {} + for row in ws.iter_rows(): + label = row[0].value + if label: + out[str(label)] = [c.value for c in row[1:]] + return out + + +def _formulas(model, tmp_path): + """Sheet X's formulas by line — the period cells of a series, the one + cell of a scalar (the sheet keeps a column for scalars, blank on a + series row).""" + path = tmp_path / "text.xlsx" + model.to_excel(path) + return {label: [v for v in cells if v is not None] + for label, cells in _rows(load_workbook(path)["X"]).items()} + + +class TestWhatTheWorkbookSays: + def test_one_call_per_period(self, tmp_path): + x = _formulas(_model(), tmp_path) + assert x["left"] == ['=LEFT(I!C2, 2)', '=LEFT(I!D2, 2)', '=LEFT(I!E2, 2)'] + assert x["left1"] == ['=LEFT(I!C2)', '=LEFT(I!D2)', '=LEFT(I!E2)'] + assert x["leftdate"][0] == '=LEFT(I!C6, 4)' + assert x["right"] == ['=RIGHT(I!C3, I!B7)', '=RIGHT(I!D3, I!B7)', '=RIGHT(I!E3, I!B7)'] + assert x["mid"][0] == '=MID(I!C3, 4, 2)' + assert x["find"][0] == '=FIND("-", I!C3)' + assert x["find2"] == ['=FIND("a", I!C2, 2)', '=FIND("a", I!D2, 2)', '=FIND("a", I!E2, 2)'] + assert x["findempty"][0] == '=FIND("", I!C2)' + assert x["search"][0] == '=SEARCH("a*f", I!C2)' + assert x["sub"][0] == '=SUBSTITUTE(I!C3, "-", "")' + assert x["sub2"][0] == '=SUBSTITUTE(I!C2, "a", "A", 1)' + assert x["value"] == ['=VALUE(I!C5)', '=VALUE(I!D5)', '=VALUE(I!E5)'] + assert x["text0"][0] == '=TEXT(I!C4, "0")' + assert x["text2"][0] == '=TEXT(I!C4, "#,##0.00")' + assert x["textp"][0] == '=TEXT(I!C4, "0.0%")' + assert x["textd"] == ['=TEXT(I!C6, "mmm yyyy")', '=TEXT(I!D6, "mmm yyyy")', + '=TEXT(I!E6, "mmm yyyy")'] + assert x["textd2"][0] == '=TEXT(I!C6, "dd.mm.yyyy")' + assert x["textm"][0] == '=TEXT(I!C6, "mmmmm")' + assert x["textpad"] == ['=TEXT(I!C23, "#,##000000")', '=TEXT(I!D23, "#,##000000")', + '=TEXT(I!E23, "#,##000000")'] + assert x["valuetime3"][0] == '=VALUE(I!C22)' + assert x["leftbig2"][0] == '=LEFT(I!C21, 25)' + assert x["textb"][0] == '=TEXT(I!C12, "0")' + assert x["leftnums"][0] == '=LEFT(I!C13, 20)' + assert x["value2"][0] == '=VALUE(I!C14)' + assert x["err"][0] == '=LEFT(I!C11, 1)' + + def test_a_scalar_call_is_one_cell(self, tmp_path): + x = _formulas(_model(), tmp_path) + assert x["trim"] == ['=TRIM(I!B9)'] + assert x["rept"] == ['=REPT("#", I!B7)'] + assert x["search2"] == ['=SEARCH("A~?", "xa?b")'] + assert x["tilde"] == ['=SEARCH("~a", "xab")'] + assert x["findempty2"] == ['=FIND("", "abc", 4)'] + assert x["textz"] == ['=TEXT(I!B7, "000")'] + assert x["valuebool"] == ['=VALUE(I!B15)'] + assert x["valuebig"] == ['=VALUE("1e999")'] + + def test_textjoin_joins_the_row_as_a_range(self, tmp_path): + x = _formulas(_model(), tmp_path) + assert x["join"] == ['=_xlfn.TEXTJOIN(", ", TRUE, I!C2:E2)'] + assert x["join2"] == ['=_xlfn.TEXTJOIN(I!B8, FALSE, I!C3:E3, "end")'] + assert x["joinx"] == ['=_xlfn.TEXTJOIN(",", "x", I!C2:E2)'] + + def test_textjoin_over_a_literal_list_is_written_as_its_value(self, tmp_path): + # The range-function law: a list with no cells in the workbook + # cannot be a range, so the call's computed text is written in + # its place, with a warning. An array constant would keep it + # live; the renderer does not write one. + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=PERIODS) + with m: + m.x = mo.MultiVariable("X") + m.x.joinlist = TEXTJOIN(",", True, ["a", "b"], "c").set_display_name("joinlist") + with pytest.warns(CrossScopeReferenceWarning, match="literal list"): + m.to_excel(tmp_path / "text.xlsx") + assert _rows(load_workbook(tmp_path / "text.xlsx")["X"])["joinlist"][0] == "a,b,c" + + def test_a_literal_list_is_this_period_s_item(self, tmp_path): + x = _formulas(_model(), tmp_path) + assert x["lit"] == ['=LEFT("ab", 1)', '=LEFT("cd", 1)', '=LEFT("ef", 1)'] + + def test_a_literal_blank_where_text_is_read_is_empty_text(self, tmp_path): + x = _formulas(_model(), tmp_path) + assert x["blank"] == ['=LEFT("", 2)'] + assert x["blanklist"] == ['=LEFT("ab", 1)', '=LEFT("", 1)', '=LEFT("cd", 1)'] + assert x["blankfind"] == ['=FIND("0", "")'] + assert x["blanksub"] == ['=SUBSTITUTE("a0b", "0", "")'] + assert x["blankrept"] == ['=REPT("", 3)'] + assert x["blankfmt"][0] == '=TEXT(I!C4, "")' + assert x["blankjoin"] == ['=_xlfn.TEXTJOIN("-", FALSE, "a", "", "b")'] + assert x["blanklen"] == ['=LEN("")'] + assert x["blankupper"] == ['=UPPER("ab")', '=UPPER("")', '=UPPER("cd")'] + assert x["blankcat"] == ['="a" & "" & "b"'] + + def test_a_quote_in_a_text_is_doubled(self, tmp_path): + x = _formulas(_model(), tmp_path) + assert x["quote"][0] == '=SUBSTITUTE(I!C2, "a", """")' + + def test_a_call_inside_a_call(self, tmp_path): + x = _formulas(_model(), tmp_path) + assert x["chain"] == ['=UPPER(LEFT(I!C2, 3))', '=UPPER(LEFT(I!D2, 3))', + '=UPPER(LEFT(I!E2, 3))'] + + def test_every_name_is_known_to_the_renderer(self, tmp_path, caplog): + with caplog.at_level(logging.WARNING, logger="modeleon.compile.excel.renderer"): + _model().to_excel(tmp_path / "text.xlsx") + assert not [r for r in caplog.records if "Unknown function" in r.getMessage()] + + +def _soffice(): + for candidate in ("soffice", "libreoffice"): + found = shutil.which(candidate) + if found: + return found + mac = Path("/Applications/LibreOffice.app/Contents/MacOS/soffice") + return str(mac) if mac.exists() else None + + +def _cell(v): + return "" if v is None else v + + +@pytest.mark.slow +def test_the_workbook_recalculates_to_the_python_values(tmp_path): + soffice = _soffice() + if soffice is None: + pytest.skip("LibreOffice not installed") + m = _model() + path = tmp_path / "text.xlsx" + m.to_excel(path) + subprocess.run( + [soffice, f"-env:UserInstallation={(tmp_path / 'prof').as_uri()}", + "--headless", "--convert-to", "xlsx", "--outdir", + str(tmp_path / "calc"), str(path)], + check=True, capture_output=True, + ) + x = _rows(load_workbook(tmp_path / "calc" / "text.xlsx", data_only=True)["X"]) + assert set(DIVERGENT) <= set(LINES) + for name in LINES: + var = getattr(m.x, name) + if var.var_type == 'list': + got = [_cell(v) for v in x[name][-PERIODS:]] + else: + got = _cell(x[name][0]) + engine, libre = DIVERGENT.get(name, (var.value, var.value)) + assert var.value == engine, name + assert got == libre, name diff --git a/tests/test_excel_time_header.py b/tests/test_excel_time_header.py index 4bd3cfd..58f5741 100644 --- a/tests/test_excel_time_header.py +++ b/tests/test_excel_time_header.py @@ -250,17 +250,18 @@ def _serial(v): return round(_to_serial(v), 6) if isinstance(v, (int, float, date)) else v -@pytest.mark.parametrize('grain_book', ['month', 'quarter']) +@pytest.mark.parametrize('grain_book', ['month', 'quarter', 'half']) @pytest.mark.slow def test_the_workbook_recalculates_to_the_engine(tmp_path, grain_book): m = target(6) - if grain_book == 'quarter': - # A quarterly book of the same model (the debt loop re-grains by - # its balances; the rest is recomputed for quarters). + if grain_book in ('quarter', 'half'): + # A quarterly / half-yearly book of the same model (the debt loop + # re-grains by its balances; the rest is recomputed for the + # coarser periods). m.debt.interest.set_regrain(mo.frozen(values='sum')) m.debt.balance.set_regrain(mo.frozen(values='last')) m.inputs.capex.set_regrain(mo.up('sum')) - m = m.at('quarter') + m = m.at(grain_book) wb = _recalculated(m, tmp_path) rows = { 'Inputs': ['rate', 'loan', 'cod'], diff --git a/tests/test_excel_view.py b/tests/test_excel_view.py index 42a7351..36c0fa7 100644 --- a/tests/test_excel_view.py +++ b/tests/test_excel_view.py @@ -1,11 +1,19 @@ # SPDX-License-Identifier: Apache-2.0 """ExcelView value object + the declared-once cascade resolver.""" +import warnings + import pytest from openpyxl import load_workbook import modeleon as mo -from modeleon.compile.excel.view import ExcelView, resolve_excel_view +from modeleon.compile.excel.view import ( + ExcelView, + UnreadViewWarning, + resolve_excel_view, + unread_view_notes, + view_pass, +) class TestExcelViewValueObject: @@ -175,6 +183,68 @@ def test_extend_for_inherit_and_tweak(self): assert r.format_by_type is True +class TestViewPass: + """``view_pass`` keeps each node's resolved view for a read-only pass: + the same answers as without it, and nothing kept past what it read.""" + + @pytest.fixture(autouse=True) + def _restore_global(self): + import modeleon + saved = modeleon.default_excel_view + yield + modeleon.default_excel_view = saved + + @staticmethod + def _model(): + m = mo.Model("m") + m.default_excel_view = ExcelView( + font={"name": "Arial Narrow"}, formats={"date": "dd mmm yyyy"}) + m.sec = mo.MultiVariable("Sec") + m.sec.default_excel_view = ExcelView(formats={"number": "#,##0"}) + m.sec.x = mo.Variable(1) + m.y = mo.Variable(2) + return m + + def test_same_answers_as_without_a_pass(self): + m = self._model() + nodes = (m.sec.x, m.y, m.sec, m, None) + outside = [resolve_excel_view(n) for n in nodes] + with view_pass(): + assert [resolve_excel_view(n) for n in nodes] == outside + assert [resolve_excel_view(n) for n in nodes] == outside + + def test_a_node_resolves_once_per_pass(self): + m = self._model() + with view_pass(): + first = resolve_excel_view(m.sec.x) + assert resolve_excel_view(m.sec.x) is first + + def test_a_view_edited_in_the_pass_is_read_again(self): + m = self._model() + with view_pass(): + assert resolve_excel_view(m.y).orient == "across" # the house default's + m.default_excel_view.orient = mo.Variable("down") + assert resolve_excel_view(m.y).orient == "down" + + def test_a_house_view_or_a_new_default_in_the_pass_is_honoured(self): + import modeleon + m = self._model() + with view_pass(): + assert resolve_excel_view(m.y).time_label_format == "finance" + with mo.house_view(ExcelView(timeline={"label_format": "iso"})): + assert resolve_excel_view(m.y).time_label_format == "iso" + assert resolve_excel_view(m.y).time_label_format == "finance" + modeleon.default_excel_view = ExcelView(timeline={"label_format": "compact"}) + assert resolve_excel_view(m.y).time_label_format == "compact" + + def test_the_pass_ends_with_the_block(self): + m = self._model() + with view_pass(): + assert resolve_excel_view(m.y).base_font == {"name": "Arial Narrow"} + m.default_excel_view = ExcelView(font={"name": "Calibri"}) + assert resolve_excel_view(m.y).base_font == {"name": "Calibri"} + + class TestBaseFontWiring: """The resolved view's ``base_font`` reaches the emitted cells.""" @@ -1096,3 +1166,151 @@ def test_a_windowed_tracked_model_takes_the_view_kwarg(self, tmp_path): assert "default_excel_view" not in load_workbook( tmp_path / "v.xlsx" ).sheetnames + + + +class TestAViewHungWhereNothingReadsIt: + """An unnamed view under a name that says "view", held by no + ``default_excel_view`` slot, is a container nobody reads — the book + keeps its look, no total column appears — so the book warns with the + slot to use. A view the slot points at (also through ``extend``), a + named view (for ``applied_view``) and a view kept under another name pass; and a + view's settings are words, never a series placed on the window + (``totals=['quarter', 'year']`` was once refused as "2 values + in a 24-period window").""" + + @staticmethod + def _model(**kw): + m = mo.Model("m", default_grain="month", default_start="2026-01", + default_periods=24, **kw) + with m: + m.pnl = mo.MultiVariable("P&L") + with m.pnl as pnl: + pnl.revenue = mo.Variable([1.0] * 24, regrain=mo.up("sum")) + return m + + def test_a_view_under_a_name_that_says_view_is_noted_with_the_slot(self): + m = self._model() + m.excel_view = ExcelView(timeline={"totals": ["quarter", "year"]}) + (path, note), = unread_view_notes([m]) + assert path == "m.excel_view" + assert note.startswith("`m.excel_view = mo.ExcelView(...)` hangs the view on m ") + assert "`m.default_excel_view = mo.ExcelView(...)`" in note + assert "mo.ExcelView('Quarterly', ...)" in note + + def test_the_sentence_spells_the_holder_by_the_author_s_name(self, tmp_path): + # ``model = mo.Model('Acme Corp')``: the slot line the sentence says + # to write is the author's binding, never the declared name. + model = mo.Model("Acme Corp", default_grain="month", default_start="2026-01", + default_periods=3) + model.pnl = mo.MultiVariable("P&L") + model.pnl.revenue = mo.Variable([1.0] * 3, regrain=mo.up("sum")) + model.view = ExcelView(timeline={"totals": ["year"]}) + with pytest.warns(UnreadViewWarning, match=r"`model\.view = mo\.ExcelView") as rec: + model.to_excel(tmp_path / "t.xlsx") + assert "`model.default_excel_view = mo.ExcelView(...)`" in str(rec[0].message) + # the warning lands on the author's line, not inside the engine + assert rec[0].filename == __file__ + + def test_a_projection_keeps_what_the_slot_reads(self): + # ``.at('quarter')`` (a projection to a coarser grain) copies a hung view on adoption: + # the copy is the same view — pointed at, named, extended — and a + # view nothing reads is still noted on the projection. + m = self._model() + m.fin_view = ExcelView(timeline={"totals": ["year"]}) + m.default_excel_view = m.fin_view + m.quarterly_view = ExcelView("Quarterly", timeline={"totals": ["year"]}) + m.base_view = ExcelView(font={"name": "Arial Narrow", "size": 10}) + m.pnl.default_excel_view = m.base_view.extend(cell_types={"on": True}) + assert unread_view_notes([m]) == [] + assert unread_view_notes([m.at("quarter")]) == [] + hung = self._model() + hung.view = ExcelView(timeline={"totals": ["year"]}) + assert [path for path, _ in unread_view_notes([hung.at("quarter")])] == ["m.view"] + + def test_a_named_view_taken_by_a_second_model_keeps_its_name(self): + m = self._model() + m.board_view = ExcelView("Board", timeline={"totals": ["year"]}) + m2 = self._model() + m2.board_view = m.board_view # adopted by a second parent: a copy + assert unread_view_notes([m2]) == [] + + def test_an_ancestor_s_slot_reads_a_view_for_a_section_written_alone(self, tmp_path): + m = self._model() + m.pnl.fin_view = ExcelView(timeline={"totals": ["year"]}) + m.default_excel_view = m.pnl.fin_view + assert unread_view_notes([m.pnl]) == [] + with warnings.catch_warnings(): + warnings.simplefilter("error", UnreadViewWarning) + m.pnl.to_excel(tmp_path / "t.xlsx") + + def test_a_view_rendered_as_the_applied_look_is_read(self, tmp_path): + from modeleon.compile.excel.view import applied_view + + m = self._model() + m.board_view = ExcelView(timeline={"totals": ["year"]}) # unnamed, applied + assert unread_view_notes([m]) != [] + with applied_view(m.board_view): + assert unread_view_notes([m]) == [] + with warnings.catch_warnings(): + warnings.simplefilter("error", UnreadViewWarning) + m.to_excel(tmp_path / "t.xlsx") + assert unread_view_notes([m]) != [] + + def test_a_view_rendered_under_the_house_layer_is_read(self, tmp_path): + m = self._model() + m.house_view = ExcelView(timeline={"totals": ["year"]}) + assert unread_view_notes([m]) != [] + with mo.house_view(m.house_view): + assert unread_view_notes([m]) == [] + with warnings.catch_warnings(): + warnings.simplefilter("error", UnreadViewWarning) + m.to_excel(tmp_path / "t.xlsx") + assert unread_view_notes([m]) != [] # the layer ends with the block + + def test_the_view_keyword_is_noted_and_the_book_warns(self, tmp_path): + m = self._model(view=ExcelView(timeline={"totals": ["year"]})) + assert [path for path, _ in unread_view_notes([m])] == ["m.view"] + with pytest.warns(UnreadViewWarning, match=r"`m\.view = mo\.ExcelView"): + m.to_excel(tmp_path / "t.xlsx") + ws = load_workbook(tmp_path / "t.xlsx")["P&L"] + heads = {ws.cell(r, c).value for r in range(1, 4) for c in range(1, ws.max_column + 1)} + assert "Total 2026" not in heads # the book kept its look + + def test_a_view_under_a_section_is_spelled_by_its_path(self): + m = self._model() + m.pnl.view = ExcelView(timeline={"totals": ["year"]}) + (path, note), = unread_view_notes([m]) + assert path == "m.pnl.view" + assert note.startswith("`m.pnl.view = mo.ExcelView(...)` hangs the view on m.pnl ") + + def test_a_view_the_slot_points_at_is_read(self): + m = self._model() + m.fin_view = ExcelView(timeline={"totals": ["year"]}) + m.default_excel_view = m.fin_view + assert unread_view_notes([m]) == [] + + def test_a_view_read_through_its_extension_is_read(self): + m = self._model() + m.base_view = ExcelView(font={"name": "Arial Narrow", "size": 10}) + m.pnl.default_excel_view = m.base_view.extend(cell_types={"on": True}) + assert unread_view_notes([m]) == [] + + def test_a_named_view_and_a_view_under_another_name_pass(self, tmp_path): + m = self._model() + m.quarterly_view = ExcelView("Quarterly", timeline={"totals": ["quarter", "year"]}) + m.style = ExcelView(timeline={"totals": ["quarter", "year"]}) + assert unread_view_notes([m]) == [] + with warnings.catch_warnings(): + warnings.simplefilter("error", UnreadViewWarning) + m.to_excel(tmp_path / "t.xlsx") + + def test_the_slot_draws_the_total_columns(self, tmp_path): + m = self._model() + m.default_excel_view = ExcelView(timeline={"totals": ["quarter", "year"]}) + with warnings.catch_warnings(): + warnings.simplefilter("error", UnreadViewWarning) + m.to_excel(tmp_path / "t.xlsx") + ws = load_workbook(tmp_path / "t.xlsx")["P&L"] + heads = {ws.cell(r, c).value for r in range(1, 4) for c in range(1, ws.max_column + 1)} + assert {"Total Q1", "Total 2026", "Total 2027"} <= heads diff --git a/tests/test_excel_view_pass.py b/tests/test_excel_view_pass.py new file mode 100644 index 0000000..a55b9bb --- /dev/null +++ b/tests/test_excel_view_pass.py @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: Apache-2.0 +"""``view_pass`` serves exactly what a resolve outside it would. + +A tree with a view at three levels is resolved, node by node, through a +sequence of the changes a pass must see — a house view entered and left, +the process-wide default replaced, a view edited — once inside one pass +and once with no pass at all. Every answer must be the same, and two +threads, each in its own pass over its own tree, must not see each +other's answers. +""" + +from __future__ import annotations + +import threading +from contextlib import ExitStack + +import pytest + +import modeleon as mo +from modeleon.compile.excel.view import ExcelView, resolve_excel_view, view_pass + + +@pytest.fixture(autouse=True) +def _restore_global(): + saved = mo.default_excel_view + yield + mo.default_excel_view = saved + + +def _tree(font: str = "Arial Narrow"): + m = mo.Model("m") + m.default_excel_view = ExcelView( + font={"name": font}, formats={"number": "#,##0"}, bands={"section": {"bold": True}}) + m.a = mo.MultiVariable("A") + m.a.default_excel_view = ExcelView(formats={"%": "0.0%"}, nesting={"indent": 2}) + m.a.b = mo.MultiVariable("B") + m.a.b.default_excel_view = ExcelView(bands={"input": {"font_color": "0000FF"}}) + m.a.b.x = mo.Variable(1) + m.a.y = mo.Variable(2) + m.z = mo.Variable(3) + return m + + +def _nodes(m): + return [m.a.b.x, m.a.y, m.z, m.a.b, m.a, m, None] + + +def _script(m, stack: ExitStack, views: dict) -> list: + """Resolve every node (twice: the second ask is the memo's), change + something, again — the answers after each step. The views it installs + are built before any pass: building a view drops an open pass's memo, + which would hide whether the pass sees the change by itself.""" + out = [] + + def record(): + first = [resolve_excel_view(n) for n in _nodes(m)] + again = [resolve_excel_view(n) for n in reversed(_nodes(m))][::-1] + assert again == first + out.append(first) + + floor = mo.default_excel_view + record() + stack.enter_context(mo.house_view(views["house"])) + record() + mo.default_excel_view = views["floor"] + record() + m.a.default_excel_view.orient = views["orient"] # an edit of a view + record() + stack.close() # the house view ends + record() + mo.default_excel_view = floor + record() + return out + + +def _views() -> dict: + return { + "house": ExcelView(timeline={"label_format": "iso"}), + "floor": ExcelView(timeline={"label_format": "compact"}, formats={"number": "0.00"}), + "orient": mo.Variable("down"), + } + + +def test_a_pass_answers_as_no_pass_through_every_change(): + with ExitStack() as stack: + plain = _script(_tree(), stack, _views()) + tree, views = _tree(), _views() + with view_pass(), ExitStack() as stack: + passed = _script(tree, stack, views) + assert passed == plain + # Each step did change what the nodes resolve to. + assert all(a != b for a, b in zip(plain, plain[1:])) + + +def test_the_pass_ends_with_the_block(): + m = _tree() + deeper = ExcelView(nesting={"indent": 9}) # built outside the pass + with view_pass(): + assert resolve_excel_view(m.a.y).nesting_indent == 2 + m.a.default_excel_view = deeper # builds no view: drops no memo + assert resolve_excel_view(m.a.y).nesting_indent == 9 + + +def test_two_threads_in_their_own_passes(): + trees = {"Arial": _tree("Arial"), "Calibri": _tree("Calibri")} + expected = {k: [resolve_excel_view(n) for n in _nodes(t)] for k, t in trees.items()} + got: dict = {} + barrier = threading.Barrier(2, timeout=30) + + def worker(key): + with view_pass(): + first = [resolve_excel_view(n) for n in _nodes(trees[key])] + barrier.wait() + got[key] = (first, [resolve_excel_view(n) for n in _nodes(trees[key])]) + + threads = [threading.Thread(target=worker, args=(k,)) for k in trees] + for t in threads: + t.start() + for t in threads: + t.join() + for key in trees: + assert got[key] == (expected[key], expected[key]) + assert expected["Arial"][0].base_font != expected["Calibri"][0].base_font + + + +def _view_model(): + import modeleon as mo + + m = mo.MultiVariable("m") + m.sec = mo.MultiVariable("sec") + m.sec.x = mo.Variable(1) + m.default_excel_view = mo.ExcelView() + return m + + +def test_an_edit_inside_a_nested_pass_is_seen_by_the_outer_one(): + """The outer pass resolved a line; an inner pass edited the view. Once + the inner block ends, the outer pass answers with the edit.""" + import modeleon as mo + from modeleon.compile.excel.view import resolve_excel_view, view_pass + + m = _view_model() + before = resolve_excel_view(m.sec.x).orient + with view_pass(): + assert resolve_excel_view(m.sec.x).orient == before + with view_pass(): + m.default_excel_view.orient = mo.Variable("down" if before != "down" else "across") + served = resolve_excel_view(m.sec.x).orient + assert served != before + assert served == resolve_excel_view(m.sec.x).orient + + +def test_an_edit_from_another_thread_is_seen_inside_a_pass(): + """A view edited by another thread while a pass is open: the pass + answers with the edit, as a resolve without a pass does.""" + import threading + + import modeleon as mo + from modeleon.compile.excel.view import resolve_excel_view, view_pass + + m = _view_model() + before = resolve_excel_view(m.sec.x).orient + with view_pass(): + assert resolve_excel_view(m.sec.x).orient == before + target = "down" if before != "down" else "across" + t = threading.Thread(target=lambda: setattr(m.default_excel_view, "orient", mo.Variable(target))) + t.start() + t.join() + served = resolve_excel_view(m.sec.x).orient + assert served == target diff --git a/tests/test_financial_schedules.py b/tests/test_financial_schedules.py new file mode 100644 index 0000000..fe1eeb9 --- /dev/null +++ b/tests/test_financial_schedules.py @@ -0,0 +1,694 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The loan schedule — ``NPER``, ``IPMT``, ``PPMT``, ``RATE``, ``CUMIPMT``, +``CUMPRINC`` — and depreciation — ``SLN``, ``DB``, ``DDB``, ``SYD`` — +compute what the spreadsheet computes, with the formula as the product. + +The numbers are the spreadsheet's own: the documented examples, and a +recalculation of the fractional, zero-rate and negative cases. Each +function is one call per period through the shared helper, so a blank +is 0, numeric text its number and other text ``#VALUE!``, an error cell +passes through, scalars broadcast, tracks lift, an amount keeps its +unit, and a period-shaped operand — ``mo.time.index`` — makes the call +a schedule row. +""" + +from __future__ import annotations + +import math + +import numpy as np +import pytest + +import modeleon as mo +from modeleon.core.expr import FuncCall, Literal, VarRef +from modeleon.functions import financial +from modeleon.functions.financial import ( + CUMIPMT, + CUMPRINC, + DB, + DDB, + IPMT, + NPER, + PMT, + PPMT, + PV, + RATE, + SLN, + SYD, +) +from modeleon.functions.registry import ( + admits_in_loop, + excel_name, + file_prefix, + is_elementwise, + is_range_taking, + spec, +) + +NAMES = { + "NPER": NPER, "IPMT": IPMT, "PPMT": PPMT, "RATE": RATE, "CUMIPMT": CUMIPMT, + "CUMPRINC": CUMPRINC, "SLN": SLN, "DB": DB, "DDB": DDB, "SYD": SYD, +} + +MONTHLY = 0.09 / 12 + + +def close(value, expected, rel=1e-9): + return value == pytest.approx(expected, rel=rel, abs=1e-9) + + +# ══════════════════════════════════════════════════════════════════════ +# The loan schedule +# ══════════════════════════════════════════════════════════════════════ + + +class TestNper: + @pytest.mark.parametrize("args, expected", [ + ((0.01, -100, -1000, 10000, 1), 59.6738656742946), + ((0.01, -100, -1000, 10000), 60.0821228537617), + ((0.01, -100, -1000), -9.57859403981317), + ((0.0, -100, 1000), 10.0), + ((0.05, -100, 0, 0), 0.0), + ((0.05, 0, 0), 0.0), + ((0.1, -100, 1000, 0, 1), 25.1588579280968), + ((0.1, -100, 1000, 0, 2), 25.1588579280968), # any other type reads as 1 + ((0.05, 0, -1000, 2000), 14.2066990828226), # no payment: a sum doubling + ]) + def test_the_values(self, args, expected): + assert close(NPER(*args).value, expected) + + def test_the_spreadsheet_s_errors(self): + assert NPER(0, 0, 1000).value == '#NUM!' # no payment at no rate + assert NPER(0.1, 100, -1000).value == '#NUM!' # the payment is the interest + assert NPER(0.1, 50, -1000).value == '#NUM!' # and does not even cover it + # At -100% no number of periods pays anything off (LibreOffice + # divides by log1p(-1) = -inf and shows 0). + assert NPER(-1, -100, 1000).value == '#NUM!' + assert NPER(-1.5, -100, 1000).value == '#NUM!' + + def test_no_unit_and_a_float(self): + pmt = mo.Variable(-200.0, unit="$") + loan = mo.Variable(8000.0, unit="$") + out = NPER(0.01, pmt, loan) + assert out._unit is None and out.value_type == 'float' and out.var_type == 'scalar' + assert NPER(0.01, mo.Variable(-200), mo.Variable(8000)).value_type == 'float' + + def test_an_optional_argument_is_written_only_when_given(self): + rate = mo.Variable(0.01, display_name="rate") + pmt = mo.Variable(-100.0, display_name="pmt") + pv = mo.Variable(-1000.0, display_name="pv") + assert NPER(rate, pmt, pv).formula == "NPER(rate, pmt, pv)" + assert NPER(rate, pmt, pv, 10000.0).formula == "NPER(rate, pmt, pv, 10000.0)" + assert NPER(rate, pmt, pv, when=1).formula == "NPER(rate, pmt, pv, 0, 1)" + assert len(NPER(rate, pmt, pv)._expr.args) == 3 + assert len(NPER(rate, pmt, pv, when=1)._expr.args) == 5 + + +class TestIpmt: + @pytest.mark.parametrize("args, expected", [ + ((0.1 / 12, 1, 36, 8000), -66.6666666666667), + ((0.1, 3, 3, 8000), -292.447129909365), + ((0.1, 1, 3, 8000, 0, 1), 0.0), + ((0.1, 2, 3, 8000, 0, 1), -507.552870090635), + ((0.1, 2, 3, 8000, 0, 2), -507.552870090635), + ((0.0, 2, 10, 1000), 0.0), + ((0.1, 2.5, 3, 8000), -428.544451994763), + ((0.1, 2, 3.5, 8000), -597.961725204323), + ((0.1, 2, 3, 8000, 1000), -528.09667673716), + ((0.1, 1, 3, -8000), 800.0), + ]) + def test_the_values(self, args, expected): + assert close(IPMT(*args).value, expected) + + def test_a_zero_rate_is_a_plain_zero(self): + value = IPMT(0.0, 2, 10, 1000).value + assert value == 0.0 and math.copysign(1.0, value) == 1.0 + + def test_the_spreadsheet_s_errors(self): + assert IPMT(0.1, 0, 3, 8000).value == '#NUM!' + assert IPMT(0.1, 4, 3, 8000).value == '#NUM!' + assert IPMT(0.1, 1, 0, 8000).value == '#NUM!' + assert IPMT(-1.5, 1, 3, 8000).value == '#NUM!' + + def test_a_rate_of_minus_100_percent_is_computed_as_the_spreadsheet_computes_it(self): + # Every amount is gone after a period: the first period's interest + # is the whole loan and the payment nothing (LibreOffice's values). + assert IPMT(-1, 1, 3, 8000).value == 8000.0 + assert PPMT(-1, 1, 3, 8000).value == -8000.0 + assert IPMT(-1, 2, 3, 8000).value == 0.0 + assert PPMT(-1, 2, 3, 8000).value == 0.0 + assert IPMT(-1, 1, 3, 8000, 0, 1).value == 0.0 + + def test_the_first_period_needs_no_payment(self): + # (1 + rate) ** nper overflows on a term this long; the first + # period's interest is the loan's own, as the spreadsheet gives it, + # and the rest is #NUM! on both sides. + assert IPMT(0.1, 1, 10000, 1000).value == -100.0 + assert IPMT(0.1, 1, 10000, 1000, 500).value == -100.0 + assert CUMIPMT(0.1, 10000, 1000, 1, 1, 0).value == -100.0 + assert IPMT(0.1, 2, 10000, 1000).value == '#NUM!' + assert PPMT(0.1, 1, 10000, 1000).value == '#NUM!' + + def test_a_series_per_is_a_schedule_row(self): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with m: + m.rate = mo.Variable(0.1, display_name="rate") + m.loan = mo.Variable(8000.0, display_name="loan", unit="$") + m.interest = IPMT(m.rate, mo.time.index, 3, m.loan) + assert m.interest.var_type == 'list' and m.interest.value_type == 'float' + assert m.interest.value == [pytest.approx(IPMT(0.1, p, 3, 8000).value) for p in (1, 2, 3)] + assert str(m.interest._unit) == "$" + assert IPMT(0.1, [1, 2, 3], 3, 8000).value == m.interest.value + + +class TestPpmt: + @pytest.mark.parametrize("args, expected", [ + ((0.1 / 12, 1, 24, 2000), -75.6231860083664), + ((0.08, 10, 10, 200000), -27598.0534624214), + ((0.1, 2, 3, 8000, 0, 1), -2416.91842900302), + ((0.0, 2, 10, 1000), -100.0), + ((0.0, 2, 10, 1000, 500), -150.0), + ]) + def test_the_values(self, args, expected): + assert close(PPMT(*args).value, expected) + + def test_the_spreadsheet_s_errors(self): + assert PPMT(0.1, 4, 3, 8000).value == '#NUM!' + assert PPMT(0.1, 0.5, 3, 8000).value == '#NUM!' + + def test_a_small_rate_loses_no_digits(self): + # (1 + rate) ** nper - 1 by expm1, as the spreadsheet computes it: + # at 1e-12 the subtraction alone keeps four digits, at 3e-16 none + # (LibreOffice's values). + assert close(PPMT(1e-9, 2, 10, 1000).value, -99.99999965) + assert close(PPMT(1e-12, 2, 10, 1000).value, -99.99999999965) + assert close(PPMT(1e-12, 2, 10, 1000, 0, 1).value, -99.99999999955) + assert close(PPMT(3e-16, 2, 10, 1000).value, -99.9999999999999) + assert close(PPMT(1e-9, 7.5, 10, 1000, 300).value, -130.00000026) + assert close(CUMPRINC(1e-12, 360, 125000, 13, 24, 0).value, -4166.66666599167) + + @pytest.mark.parametrize("when", [0, 1]) + def test_interest_and_principal_make_the_payment_and_pay_the_loan_off(self, when): + payment = PMT(0.1, 3, 8000, 0, when).value + principal = [PPMT(0.1, p, 3, 8000, 0, when).value for p in (1, 2, 3)] + interest = [IPMT(0.1, p, 3, 8000, 0, when).value for p in (1, 2, 3)] + for i, p in zip(interest, principal): + assert i + p == pytest.approx(payment) + assert sum(principal) == pytest.approx(-8000.0) + + +class TestRate: + @pytest.mark.parametrize("args, expected", [ + ((48, -200, 8000), 0.00770147248823326), + ((48, -200, 8000, 0, 1), 0.00805298192393902), + ((10, -100, 0, 1500), 0.0873205217799398), + ((48, -200, 8000, 0, 0, 0.05), 0.00770147248820193), + ((10.5, -100, 800), 0.0504359702865107), + ((5, 50, -1000), -0.335280089136234), + ]) + def test_the_values(self, args, expected): + assert close(RATE(*args).value, expected) + + def test_the_rate_found_gives_the_payment_back(self): + rate = RATE(360, -1000, 125000).value + assert PMT(rate, 360, 125000).value == pytest.approx(-1000.0) + rate = RATE(360, -1000, 125000, 5000, 1).value + assert PMT(rate, 360, 125000, 5000, 1).value == pytest.approx(-1000.0) + + def test_a_loan_paid_back_exactly_has_a_rate_of_zero(self): + # A rate too small to move 1 + rate is 0 (LibreOffice stops at + # -7e-12), and the schedule reads it as no interest. + assert RATE(10, -100, 1000).value == 0.0 + assert RATE(10, -100, 1000, guess=0.5).value == 0.0 + assert RATE(1000, -1, 1000).value == 0.0 + rate = RATE(10, -100, 1000).value + assert IPMT(rate, 2, 10, 1000).value == 0.0 and PPMT(rate, 2, 10, 1000).value == -100.0 + + def test_the_spreadsheet_s_iteration_and_its_further_starts(self): + # A few payments the size of the loan: Newton from 10% lands below + # -100%, and the spreadsheet starts again at 20%, 5%, 30%, ... as + # the engine does. A thousand-period term settles inside the 150 + # steps. The numbers are LibreOffice's. + assert close(RATE(5, -1000, 1000).value, 0.965948236645501) + assert close(RATE(12, -1000, 1000).value, 0.999755500937318) + assert close(RATE(5.5, -1000, 1000).value, 0.976413003900949) + assert close(RATE(1200, -1, 1000).value, 0.000313468830444138) + assert close(RATE(2, -1, 1000).value, -0.967873270798253) + assert close(RATE(1, -2000, 1000).value, 1.0) + # The rate settled on is refined to full precision: the payment + # comes back to the last digits (LibreOffice stops within its step, + # at 0.000999954352834482). + rate = RATE(10000, -1, 1000).value + assert close(rate, 0.000999954352834482) + assert PMT(rate, 10000, 1000).value == pytest.approx(-1.0, rel=1e-12) + # An annuity due's balance is x (400 x^2 - 600 x - 600) in x = 1 + r: + # the rate is the quadratic's root, never the x = 0 at -100%. + assert close(RATE(3, -600, 1000, 0, 1).value, + (600 + math.sqrt(600 ** 2 + 4 * 400 * 600)) / 800 - 1) + + def test_an_end_that_is_no_rate_is_a_start_that_found_none(self): + # LibreOffice's iteration can stop at a point and show it as the + # rate: -100%, where an annuity due's balance and a loan's with no + # future value vanish trivially, or a point where the balance is + # merely small. The engine takes neither: it tries the next start, + # and with none left it is #NUM!. + assert RATE(10, 0, 1000).value == '#NUM!' # LibreOffice: -98.2% + assert RATE(36, 50, 0, 0, 1).value == '#NUM!' # LibreOffice: -100.0% + assert RATE(360, 0, -1000, 0, 1, 0.05).value == '#NUM!' # LibreOffice: -10.4% + assert RATE(1200, -10, 0).value == '#NUM!' # LibreOffice: 80% + # From 10% the iteration ends at -100% (LibreOffice shows -100%); + # a further start finds the rate the payments carry. + for args in [(120, -100, 1000, 0, 1), (48, -200, 1000, 0, 1)]: + rate = RATE(*args).value + assert isinstance(rate, float) and rate > 0.1, args + assert PV(rate, args[0], args[1], 0, 1).value == pytest.approx(1000.0) + + def test_a_given_guess_is_the_only_start(self): + # LibreOffice starts again only from its own default; from a guess + # given, an iteration that runs off is #NUM! (#N/A in LibreOffice). + assert RATE(10, -100, 800, guess=-0.5).value == '#NUM!' + assert RATE(10, -100, 1000, 0, 0, -2).value == '#NUM!' + assert close(RATE(10, -100, 800, guess=0.05).value, RATE(10, -100, 800).value) + + def test_with_no_flow_at_all_every_rate_balances_and_the_guess_is_the_answer(self): + assert RATE(10, 0, 0).value == 0.1 # LibreOffice: 0.1 + assert RATE(10, 0, 0, guess=0.05).value == 0.05 + # One period repaid as it opens balances at any rate too + # (LibreOffice's iteration wanders there on rounding, to 82.5%). + assert RATE(1, -1000, 1000, 0, 1).value == 0.1 + + def test_the_spreadsheet_s_errors(self): + assert RATE(0, -100, 1000).value == '#NUM!' + assert RATE(-5, -100, 1000).value == '#NUM!' + assert RATE(10, 100, 1000).value == '#NUM!' # every flow the same way + # Payments five times the loan: no start settles (LibreOffice: #N/A). + assert RATE(2, -5000, 1000).value == '#NUM!' + assert RATE(12, -5000, 1000).value == '#NUM!' + + def test_no_unit_and_the_guess_written_only_when_given(self): + n = mo.Variable(48, display_name="n") + pmt = mo.Variable(-200.0, display_name="pmt", unit="$") + loan = mo.Variable(8000.0, display_name="loan", unit="$") + out = RATE(n, pmt, loan) + assert out._unit is None and out.value_type == 'float' + assert out.formula == "RATE(n, pmt, loan)" and len(out._expr.args) == 3 + assert RATE(n, pmt, loan, guess=0.05).formula == "RATE(n, pmt, loan, 0, 0, 0.05)" + + def test_a_series_nper_is_a_row_of_rates(self): + out = RATE(mo.Variable([8, 9, 10]), -100, 800) + assert out.var_type == 'list' + assert out.value == [pytest.approx(RATE(n, -100, 800).value) for n in (8, 9, 10)] + + +class TestCumulative: + @pytest.mark.parametrize("fn, args, expected", [ + (CUMIPMT, (MONTHLY, 360, 125000, 13, 24, 0), -11135.2321307508), + (CUMIPMT, (MONTHLY, 360, 125000, 1, 1, 0), -937.5), + (CUMIPMT, (MONTHLY, 360, 125000, 1, 12, 1), -10201.3328844946), + (CUMIPMT, (MONTHLY, 360, 125000, 13.7, 24.2, 0), -11135.2321307508), + (CUMPRINC, (MONTHLY, 360, 125000, 13, 24, 0), -934.107123420899), + (CUMPRINC, (MONTHLY, 360, 125000, 1, 1, 0), -68.2782711809784), + (CUMPRINC, (MONTHLY, 360, 125000, 1, 12, 1), -1778.16017175526), + ]) + def test_the_values(self, fn, args, expected): + assert close(fn(*args).value, expected) + + def test_the_sum_of_the_periods(self): + interest = sum(IPMT(MONTHLY, p, 360, 125000).value for p in range(13, 25)) + principal = sum(PPMT(MONTHLY, p, 360, 125000).value for p in range(13, 25)) + assert CUMIPMT(MONTHLY, 360, 125000, 13, 24).value == pytest.approx(interest) + assert CUMPRINC(MONTHLY, 360, 125000, 13, 24).value == pytest.approx(principal) + whole = CUMIPMT(MONTHLY, 360, 125000, 1, 360).value + \ + CUMPRINC(MONTHLY, 360, 125000, 1, 360).value + assert whole == pytest.approx(PMT(MONTHLY, 360, 125000).value * 360) + + def test_the_type_is_read_whole_and_written_always(self): + assert CUMIPMT(MONTHLY, 360, 125000, 1, 12, 0.5).value == \ + CUMIPMT(MONTHLY, 360, 125000, 1, 12, 0).value + rate = mo.Variable(MONTHLY, display_name="rate") + out = CUMIPMT(rate, 360, 125000.0, 1, 12) + assert out.formula == "CUMIPMT(rate, 360, 125000.0, 1, 12, 0)" + assert len(out._expr.args) == 6 + for fn in (CUMIPMT, CUMPRINC): + given_none = fn(rate, 360, 125000.0, 1, 12, when=None) + assert given_none.formula == f"{fn.__name__}(rate, 360, 125000.0, 1, 12, 0)" + assert given_none.value == fn(rate, 360, 125000.0, 1, 12).value + + def test_a_period_is_read_as_the_cell_shows_it(self): + # 12.999999999999998 shows as 13, and the spreadsheet counts from 13. + assert close(CUMIPMT(MONTHLY, 360, 125000, 12.999999999999998, 24, 0).value, + -11135.2321307508) + assert close(CUMPRINC(MONTHLY, 360, 125000, 12.999999999999998, 24, 0).value, + -934.107123420899) + + @pytest.mark.parametrize("args", [ + (0.0, 360, 125000, 1, 12, 0), + (-0.01, 360, 125000, 1, 12, 0), + (MONTHLY, 0, 125000, 1, 12, 0), + (MONTHLY, 360, 0, 1, 12, 0), + (MONTHLY, 360, -125000, 1, 12, 0), + (MONTHLY, 360, 125000, 0, 12, 0), + (MONTHLY, 360, 125000, 5, 3, 0), + (MONTHLY, 360, 125000, 1, 361, 0), + (MONTHLY, 360, 125000, 1, 12, 2), + ]) + def test_the_spreadsheet_s_errors(self, args): + assert CUMIPMT(*args).value == '#NUM!' + assert CUMPRINC(*args).value == '#NUM!' + + +# ══════════════════════════════════════════════════════════════════════ +# Depreciation +# ══════════════════════════════════════════════════════════════════════ + + +class TestSln: + def test_the_values(self): + assert SLN(30000, 7500, 10).value == 2250.0 + assert SLN(-30000, 7500, 10).value == -3750.0 + assert SLN(1000, 2000, 10).value == -100.0 + assert SLN(mo.Variable([30000.0, 20000.0]), 7500, 10).value == [2250.0, 1250.0] + + def test_a_life_of_zero_divides_by_zero(self): + assert SLN(30000, 7500, 0).value == '#DIV/0!' + + +class TestDb: + @pytest.mark.parametrize("period, expected", [ + (1, 186083.333333333), (2, 259639.416666667), (3, 176814.44275), + (4, 120410.63551275), (5, 81999.6427841828), (6, 55841.7567360285), + (7, 15845.0984738481), + ]) + def test_the_documented_schedule_with_seven_months_in_the_first_year(self, period, expected): + assert close(DB(1000000, 100000, 6, period, 7).value, expected) + + def test_twelve_months_when_month_is_omitted(self): + assert close(DB(1000000, 100000, 6, 6).value, 46722.5182806209) + assert DB(1000000, 100000, 6, 7).value == 0.0 # no months left for a stub + assert DB(1000000, 100000, 6, 6, 12).value == DB(1000000, 100000, 6, 6).value + + def test_the_period_and_the_month_are_read_whole(self): + assert DB(1000000, 100000, 6, 1.5, 7).value == DB(1000000, 100000, 6, 1, 7).value + assert DB(1000000, 100000, 6, 2.5, 7).value == DB(1000000, 100000, 6, 2, 7).value + assert DB(1000000, 100000, 6, 6.5, 7).value == DB(1000000, 100000, 6, 7, 7).value + assert DB(1000000, 100000, 6, 2, 7.9).value == DB(1000000, 100000, 6, 2, 7).value + + def test_no_salvage_is_the_whole_cost_at_once(self): + assert DB(1000, 0, 5, 1).value == 1000.0 + assert DB(1000, 0, 5, 2).value == 0.0 + + @pytest.mark.parametrize("args, expected", [ + # A period that reads as 1 is the first period's charge, even past + # a life shorter than that; the stub is for a later period. + ((1000, 100, 0.5, 1), 990.0), + ((1000, 100, 0.5, 1.5, 6), 495.0), + ((1000, 100, 0.9, 1, 6), 461.5), + ((1000, 100, 0.9, 1.9, 6), 461.5), + ((1000, 100, 1, 1.5, 6), 450.0), + ((1000, 100, 1.2, 1.5, 6), 426.5), + ((1000, 0, 0.5, 1, 1), 83.3333333333333), + ((1000, 100, 0.75, 1, 6), 477.0), + ((1000, 100, 1, 2, 6), 247.5), + ((1000, 100, 1.2, 2, 6), 244.59775), + ((1000, 100, 1.5, 2.5, 6), 238.44375), + ]) + def test_a_life_under_two_periods(self, args, expected): + assert close(DB(*args).value, expected) + + def test_the_rate_period_and_month_are_read_as_the_cell_shows_them(self): + # 1 - 318 / 2400 is 0.8675, held as 0.86749999...: the sheet shows + # 0.8675 and rounds it to 0.868. + assert close(DB(2400, 318, 1, 1).value, 2083.2) + assert close(DB(2400, 318, 1, 2, 7).value, 428.502666666667) + # A period or a month a hair under 3 shows as 3. + assert close(DB(1000, 100, 5, 2.9999999999999996).value, 146.921409) + assert close(DB(1000, 100, 5, 2.9999999999999996, 7).value, 182.72040525) + assert close(DB(1000, 100, 5, 1, 2.9999999999999996).value, 92.25) + + @pytest.mark.parametrize("args", [ + (1000000, 100000, 6, 8, 7), + (0, 0, 5, 1), + (-1000, -100, 5, 1), + (1000, -1, 5, 1), + (1000, 2000, 5, 1), + (1000, 100, 0, 1), + (1000, 100, 5, 0), + (1000, 100, 5, 0.5), # Excel's domain; LibreOffice computes 0 + (1000, 100, 5, 1, 0), + (1000, 100, 5, 1, 13), + ]) + def test_the_spreadsheet_s_errors(self, args): + assert DB(*args).value == '#NUM!' + + def test_a_series_period_is_a_schedule_row(self): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=7) + with m: + m.cost = mo.Variable(1000000.0, display_name="cost", unit="$") + m.dep = DB(m.cost, 100000.0, 6, mo.time.index, 7) + assert m.dep.value == [pytest.approx(DB(1000000, 100000, 6, p, 7).value) + for p in range(1, 8)] + assert str(m.dep._unit) == "$" and m.dep.var_type == 'list' + + +class TestDdb: + @pytest.mark.parametrize("args, expected", [ + ((2400, 300, 3650, 1), 1.31506849315065), + ((2400, 300, 120, 1), 40.0), + ((2400, 300, 10, 1), 480.0), + ((2400, 300, 10, 2, 1.5), 306.0), + ((2400, 300, 10, 10), 22.1225472000002), + ((2400, 300, 10, 1.5), 429.325051679959), + ((1000, 100, 2, 1, 3), 900.0), + ((1000, 100, 2, 2, 3), 0.0), + ((0, 0, 5, 1), 0.0), + ]) + def test_the_values(self, args, expected): + assert close(DDB(*args).value, expected) + + def test_the_book_value_stops_at_the_salvage_value(self): + charges = [DDB(2400, 300, 10, p).value for p in range(1, 11)] + assert sum(charges) == pytest.approx(2400 - 300) + assert charges[-1] == pytest.approx(22.1225472) and charges[0] == 480.0 + + @pytest.mark.parametrize("args", [ + (2400, 300, 10, 11), + (2400, 300, 10, 0.5), + (1000, 0, 5, 0), + (1000, 0, 5, 1, 0), + (1000, 0, 5, 1, -2), + (1000, 2000, 5, 1), + (-1000, 0, 5, 1), + (1000, -1, 5, 1), + (1000, 0, 0, 1), + ]) + def test_the_spreadsheet_s_errors(self, args): + assert DDB(*args).value == '#NUM!' + + def test_a_series_period_is_a_schedule_row(self): + row = DDB(2400.0, 300.0, 10, mo.Variable([1, 2, 3])) + assert row.value == pytest.approx([480.0, 384.0, 307.2]) + assert row.var_type == 'list' and row.value_type == 'float' + + +class TestSyd: + def test_the_values(self): + assert close(SYD(30000, 7500, 10, 1).value, 4090.90909090909) + assert close(SYD(30000, 7500, 10, 10).value, 409.090909090909) + assert close(SYD(30000, 7500, 10, 2.5).value, 3477.27272727273) + assert sum(SYD(30000, 7500, 10, p).value for p in range(1, 11)) == pytest.approx(22500) + assert SYD(1000, 2000, 5, 1).value == pytest.approx(-333.333333333333) + + @pytest.mark.parametrize("args", [ + # Outside Excel's domain, where LibreOffice computes the formula + # as written: 0 the period after the life, negative after that. + (30000, 7500, 10, 11), + (30000, 7500, 10, 12), + (30000, 7500, 10, 10.5), + (30000, 7500, 10, 0), + (30000, 7500, 10, 0.5), + (30000, 7500, 0, 1), + (30000, 7500, -2, 1), + ]) + def test_the_spreadsheet_s_errors(self, args): + assert SYD(*args).value == '#NUM!' + + def test_a_series_period_is_a_schedule_row(self): + row = SYD(30000.0, 7500.0, 10, mo.Variable([1, 2, 10])) + assert row.value == [pytest.approx(4090.909090909091), pytest.approx(3681.818181818182), + pytest.approx(409.0909090909091)] + + +# ══════════════════════════════════════════════════════════════════════ +# The shared law: one call per period +# ══════════════════════════════════════════════════════════════════════ + + +class TestOneCallPerPeriod: + def test_a_blank_is_zero(self): + blank = mo.Variable([None]) + assert SLN(blank, 100, 10).value == [-10.0] + assert DDB(2400, blank, 10, 1).value == [480.0] + assert IPMT(blank, 2, 10, 1000).value == [0.0] + assert NPER(0.1, -100, blank).value == [0.0] + assert RATE(10, -100, 1000, blank).value == [pytest.approx(0.0, abs=1e-12)] + + def test_numeric_text_is_its_number_and_other_text_is_value(self): + # As the spreadsheet reads text in an argument (LibreOffice's values). + words = mo.Variable(["abc", "4"]) + assert SLN(words, 0, 1).value == ['#VALUE!', 4.0] + assert IPMT(0.1, "x", 3, 8000).value == '#VALUE!' + assert RATE(48, -200, "8000").value == RATE(48, -200, 8000).value + assert IPMT(0.1, "2", 3, 8000).value == IPMT(0.1, 2, 3, 8000).value + assert DB(1000, 100, 5, 1, "7").value == DB(1000, 100, 5, 1, 7).value + assert CUMIPMT(0.01, 12, 1000, 1, 2, "0").value == CUMIPMT(0.01, 12, 1000, 1, 2, 0).value + assert SLN(" 1,234.5 ", 0, 1).value == 1234.5 + assert SLN("12%", 0, 1).value == pytest.approx(0.12) + assert SLN("(1,000)", 0, 1).value == -1000.0 + + def test_a_boolean_is_one_or_zero(self): + assert SLN(True, False, True).value == 1.0 + assert NPER(0.1, -100, 1000, 0, True).value == NPER(0.1, -100, 1000, 0, 1).value + assert IPMT(0.1, True, 3, 8000).value == IPMT(0.1, 1, 3, 8000).value + + def test_a_numpy_number_is_a_number(self): + assert SLN(np.int64(1000), 100, 5).value == 180.0 + assert RATE(np.int64(48), np.float64(-200.0), 8000).value == RATE(48, -200, 8000).value + out = IPMT(0.1, mo.Variable(list(np.arange(1, 4))), 3, 8000) + assert out.value == [pytest.approx(IPMT(0.1, p, 3, 8000).value) for p in (1, 2, 3)] + # Read as a plain number, so a division by zero and an overflow are + # the spreadsheet's errors, never numpy's inf. + assert SLN(np.float64(1000.0), 100, 0).value == '#DIV/0!' + assert IPMT(np.float64(0.1), 2, 10000, 1000).value == '#NUM!' + assert type(SLN(np.float64(1000.0), 100, 5).value) is float + + def test_an_error_cell_is_the_period_s_result(self): + e = mo.Variable([2.0, 2.0]) / mo.Variable([1.0, 0.0]) + assert e.value == [2.0, '#DIV/0!'] + assert SLN(e, 0, 1).value == [2.0, '#DIV/0!'] + assert IPMT(0.1, 1, 3, e).value == [pytest.approx(-0.2), '#DIV/0!'] + assert RATE(2, -1, e).value == [pytest.approx(0.0, abs=1e-12), '#DIV/0!'] + assert SYD(e, 0, 1, 1).value == [2.0, '#DIV/0!'] + + def test_scalars_only_is_one_number(self): + out = SLN(30000, 7500, 10) + assert out.var_type == 'scalar' and out.value_type == 'float' + assert out.formula == "SLN(30000, 7500, 10)" + assert IPMT(mo.Variable(0.1), mo.Variable(1), mo.Variable(3), 8000.0).var_type == 'scalar' + + def test_scalars_and_a_one_value_series_broadcast(self): + assert DDB(mo.Variable([2400.0]), 300, 10, [1, 2]).value == pytest.approx([480.0, 384.0]) + assert IPMT(mo.Variable([0.1]), [1, 2], 3, mo.Variable(8000.0)).value == \ + [pytest.approx(-800.0), pytest.approx(IPMT(0.1, 2, 3, 8000).value)] + assert SLN(mo.Variable([1000.0, 2000.0]), 0, mo.Variable([10])).value == [100.0, 200.0] + + def test_series_of_different_lengths_are_refused_as_the_operators_refuse_them(self): + with pytest.raises(ValueError, match="different lengths"): + IPMT(0.1, mo.Variable([1, 2, 3]), 3, mo.Variable([8000.0, 9000.0])) + + def test_tracked_operands_lift_per_track(self): + m = mo.Model("m", tracks=mo.Tracks("plan", "actual"), + default_grain="month", default_start="2026-01", default_periods=2) + with m: + m.loan = mo.Variable(plan=[8000.0, 8000.0], actual=[9000.0, None]) + m.i = IPMT(0.1, 1, 3, m.loan) + m.d = SLN(m.loan, 0.0, 10) + assert dict(m.i._value.items()) == {"plan": [-800.0, -800.0], "actual": [-900.0, 0.0]} + assert dict(m.d._value.items()) == {"plan": [800.0, 800.0], "actual": [900.0, 0.0]} + assert m.i.var_type == 'list' + + def test_the_result_is_a_float_whatever_the_operands(self): + assert IPMT(mo.Variable(1), 1, 3, mo.Variable(8000)).value_type == 'float' + assert SLN(mo.Variable([30000, 20000]), 7500, 10).value_type == 'float' + assert DDB(mo.Variable(2400), 300, 10, 1).value_type == 'float' + assert NPER(mo.Variable(1), -100, 1000).value_type == 'float' + + def test_an_amount_keeps_the_unit_the_amounts_share(self): + cost = mo.Variable(2400.0, unit="$") + salvage = mo.Variable(300.0, unit="$") + hours = mo.Variable(300.0, unit="hr") + loan = mo.Variable(8000.0, unit="$") + rate = mo.Variable(0.1) + assert str(SLN(cost, salvage, 10)._unit) == "$" + assert str(SLN(cost, 300.0, 10)._unit) == "$" + # A cost and a salvage in two units are refused, as + refuses them. + with pytest.raises(ValueError, match="mo.SLN takes its values in one unit"): + SLN(cost, hours, 10) + for fn in (DB, DDB, SYD): + assert str(fn(cost, salvage, 10, 1)._unit) == "$", fn + assert str(IPMT(rate, 1, 3, loan)._unit) == "$" + assert str(PPMT(rate, 1, 3, loan)._unit) == "$" + assert str(CUMIPMT(rate, 3, loan, 1, 2)._unit) == "$" + assert str(CUMPRINC(rate, 3, loan, 1, 2)._unit) == "$" + assert NPER(rate, -200.0, loan)._unit is None + assert RATE(3, -200.0, loan)._unit is None + + def test_the_formula_names_the_operands_and_the_ast_is_a_call(self): + rate = mo.Variable(0.1, display_name="rate") + per = mo.Variable([1, 2, 3], display_name="per") + term = mo.Variable(3, display_name="term") + loan = mo.Variable(8000.0, display_name="loan") + out = IPMT(rate, per, term, loan) + assert out.formula == "IPMT(rate, per, term, loan)" + assert isinstance(out._expr, FuncCall) and out._expr.func == "IPMT" + assert all(isinstance(a, VarRef) for a in out._expr.args) + assert out._expr.args[1].var is per + lit = DDB(loan, 300.0, term, 2) + assert lit.formula == "DDB(loan, 300.0, term, 2)" + assert isinstance(lit._expr.args[1], Literal) and lit._expr.args[1].value == 300.0 + assert isinstance(lit._expr.args[3], Literal) and lit._expr.args[3].value == 2 + + @pytest.mark.parametrize("name", sorted(NAMES)) + def test_an_axised_operand_is_refused(self, name): + region = mo.Variable(["N", "S"], display_name="region") + by_region = mo.Variable([1.0, 2.0], indexed_by=[region]) + arity = {"NPER": 3, "RATE": 3, "SLN": 3, "IPMT": 4, "PPMT": 4, "DB": 4, "DDB": 4, + "SYD": 4, "CUMIPMT": 5, "CUMPRINC": 5}[name] + with pytest.raises(ValueError, match=name): + NAMES[name](by_region, *([1] * (arity - 1))) + + +# ══════════════════════════════════════════════════════════════════════ +# The registry and a loop container +# ══════════════════════════════════════════════════════════════════════ + + +class TestTheRegistry: + @pytest.mark.parametrize("name", sorted(NAMES)) + def test_each_is_element_wise_with_a_bare_classic_name(self, name): + assert is_elementwise(name) and not is_range_taking(name) + assert spec(name).passthrough and not spec(name).bare + assert file_prefix(name) == "" and excel_name(name) == name + assert admits_in_loop(name, 3) and admits_in_loop(name, 6) + + def test_rate_has_its_helper_now(self): + assert not spec("RATE").bare and spec("RATE").shape == "elementwise" + assert "RATE" in financial.__all__ and callable(financial.RATE) + + +class TestInsideALoop: + def test_a_schedule_runs_period_by_period(self): + # The opening balance reads the closing one through a lag, so the + # container is a loop; NPER reads the opening balance — a row of + # the loop — and says how many payments are left. + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with m: + m.i = mo.MultiVariable("I") + m.i.rate = mo.Variable(0.1, display_name="rate") + m.i.loan = mo.Variable(1000.0, display_name="loan") + m.i.payment = PMT(m.i.rate, 3, m.i.loan).set_display_name("payment") + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + d.open = mo.lag(d.close, fill=m.i.loan) + d.interest = d.open * m.i.rate + d.principal = PPMT(m.i.rate, mo.time.index, 3, m.i.loan) + d.close = d.open + d.principal + d.left = NPER(m.i.rate, m.i.payment, d.open) + d.rate = RATE(d.left, m.i.payment, d.open) + # Interest on the opening balance is IPMT, which carries the + # payment's sign: money out. + assert [-v for v in d.interest.value] == \ + [pytest.approx(IPMT(0.1, p, 3, 1000).value) for p in (1, 2, 3)] + assert d.close.value[-1] == pytest.approx(0.0, abs=1e-9) + assert d.left.value == [pytest.approx(3.0), pytest.approx(2.0), pytest.approx(1.0)] + # The rate recovered from the periods left and the balance is the input rate. + assert d.rate.value == pytest.approx([0.1, 0.1, 0.1]) diff --git a/tests/test_function_ast_completeness.py b/tests/test_function_ast_completeness.py index c4d7782..a1668e1 100644 --- a/tests/test_function_ast_completeness.py +++ b/tests/test_function_ast_completeness.py @@ -8,12 +8,29 @@ equivalent) node. This test pins the invariant across every public function in the -``functions/`` package. +``functions/`` package; the last test fails when a function is exported +without a call here. """ from __future__ import annotations +import datetime as dt +import re +from pathlib import Path + import modeleon as mo +from modeleon import functions +from modeleon.core.expr import FuncCall + + +def _calls(results): + """Each result carries the call itself as its AST root, named as the + function, and a second renderer walks it.""" + for name, out in results.items(): + assert out._expr is not None, name + assert isinstance(out._expr, FuncCall) and out._expr.func == name, (name, out._expr) + node = mo.to_json(out._expr) + assert node["kind"] == "funccall" and node["func"] == name, (name, node) class TestFunctionASTCompleteness: @@ -38,21 +55,99 @@ def test_math_functions(self): assert mo.ROUND(b, 1)._expr is not None assert mo.INT(b)._expr is not None assert mo.MOD(c, divisor)._expr is not None + _calls({ + "ROUNDUP": mo.ROUNDUP(b, 0), + "ROUNDDOWN": mo.ROUNDDOWN(b, 0), + "TRUNC": mo.TRUNC(b), + "CEILING": mo.CEILING(b, 1), + "FLOOR": mo.FLOOR(b, 1), + "POWER": mo.POWER(c, 0.5), + "SQRT": mo.SQRT(c), + "EXP": mo.EXP(b), + "LN": mo.LN(c), + "LOG": mo.LOG(c, 10), + "LOG10": mo.LOG10(c), + "SIGN": mo.SIGN(a), + "PRODUCT": mo.PRODUCT(a, b, c), + }) def test_conditional_function(self): cond = mo.Variable(True) a = mo.Variable(1) b = mo.Variable(2) assert mo.IF(cond, a, b)._expr is not None + keyed = mo.Variable({'a': 1, 'b': 2}) + assert mo.CHOOSE(mo.Variable(1), keyed)._expr is not None + other = mo.Variable(False) + _calls({ + "AND": mo.AND(cond, other), + "OR": mo.OR(cond, other), + "NOT": mo.NOT(cond), + "ISBLANK": mo.ISBLANK(a), + }) + + def test_logical_functions(self): + cond = mo.Variable(True) + other = mo.Variable(False) + x = mo.Variable(10.0) + zero = mo.Variable(0.0) + n = mo.Variable(12) + _calls({ + "IFERROR": mo.IFERROR(x / zero, 0), + "ISERROR": mo.ISERROR(x / zero), + "ISNUMBER": mo.ISNUMBER(x), + "XOR": mo.XOR(cond, other), + "IFS": mo.IFS(cond, x, True, zero), + "SWITCH": mo.SWITCH(n, 12, "monthly", 4, "quarterly", "other"), + }) + + def test_conditional_aggregates(self): + row = mo.Variable([5.0, -2.0, 7.0]) + _calls({ + "SUMIF": mo.SUMIF(row, ">0"), + "SUMIFS": mo.SUMIFS(row, row, ">0"), + "COUNTIF": mo.COUNTIF(row, ">0"), + "COUNTIFS": mo.COUNTIFS(row, ">0"), + "AVERAGEIF": mo.AVERAGEIF(row, ">0"), + "AVERAGEIFS": mo.AVERAGEIFS(row, row, ">0"), + "MAXIFS": mo.MAXIFS(row, row, ">0"), + "MINIFS": mo.MINIFS(row, row, ">0"), + "COUNT": mo.COUNT(row), + "COUNTA": mo.COUNTA(row), + }) + + def test_lookup_functions(self): + years = mo.Variable([2026, 2027, 2028]) + rates = mo.Variable([0.05, 0.04, 0.03]) + year = mo.Variable(2027) + _calls({ + "MATCH": mo.MATCH(year, years, 0), + "INDEX": mo.INDEX(rates, 2), + "XLOOKUP": mo.XLOOKUP(year, years, rates), + "VLOOKUP": mo.VLOOKUP(2026, [years, rates], 2, False), + "HLOOKUP": mo.HLOOKUP(year, [years, rates], 2, False), + }) def test_date_functions(self): - import datetime as dt d = mo.Variable(dt.date(2026, 4, 24)) assert mo.YEAR(d)._expr is not None assert mo.MONTH(d)._expr is not None assert mo.DAY(d)._expr is not None assert mo.EDATE(d, 3)._expr is not None assert mo.EOMONTH(d, 0)._expr is not None + end = mo.Variable(dt.date(2027, 1, 15)) + _calls({ + "DATE": mo.DATE(mo.Variable(2026), 1, 31), + "DAYS360": mo.DAYS360(d, end), + "YEARFRAC": mo.YEARFRAC(d, end), + "DAYS": mo.DAYS(end, d), + "TODAY": mo.TODAY(), + "DATEDIF": mo.DATEDIF(d, end, "M"), + "WEEKDAY": mo.WEEKDAY(d), + "WEEKNUM": mo.WEEKNUM(d), + "NETWORKDAYS": mo.NETWORKDAYS(d, end), + "WORKDAY": mo.WORKDAY(d, 10), + }) def test_text_functions(self): s = mo.Variable("hello") @@ -61,6 +156,20 @@ def test_text_functions(self): assert mo.UPPER(s)._expr is not None assert mo.LOWER(s)._expr is not None assert mo.CONCAT(s, t)._expr is not None + padded = mo.Variable(" hello world ") + _calls({ + "LEFT": mo.LEFT(t, 3), + "RIGHT": mo.RIGHT(t, 3), + "MID": mo.MID(t, 2, 3), + "TRIM": mo.TRIM(padded), + "FIND": mo.FIND("o", t), + "SEARCH": mo.SEARCH("O", t), + "SUBSTITUTE": mo.SUBSTITUTE(t, "o", "0"), + "VALUE": mo.VALUE(mo.Variable("12.5")), + "TEXT": mo.TEXT(mo.Variable(3.7), "0.00"), + "REPT": mo.REPT(t, 2), + "TEXTJOIN": mo.TEXTJOIN(", ", True, s, t), + }) def test_financial_functions(self): cf = mo.Variable([-1000.0, 300.0, 400.0, 500.0]) @@ -73,6 +182,52 @@ def test_financial_functions(self): assert mo.PMT(rate, nper, pv)._expr is not None assert mo.FV(rate, nper, pmt)._expr is not None assert mo.PV(rate, nper, pmt)._expr is not None + dates = mo.Variable([dt.date(2026, 1, 1), dt.date(2026, 7, 1), + dt.date(2027, 1, 1), dt.date(2027, 7, 1)]) + monthly = mo.Variable(0.01) + payment = mo.Variable(-100) + cost, salvage, life = mo.Variable(10_000), mo.Variable(1_000), mo.Variable(5) + _calls({ + "XIRR": mo.XIRR(cf, dates), + "XNPV": mo.XNPV(rate, cf, dates), + "NPER": mo.NPER(monthly, payment, pv), + "IPMT": mo.IPMT(monthly, 1, nper, pv), + "PPMT": mo.PPMT(monthly, 1, nper, pv), + "RATE": mo.RATE(nper, payment, pv), + "CUMIPMT": mo.CUMIPMT(monthly, nper, pv, 1, 12, 0), + "CUMPRINC": mo.CUMPRINC(monthly, nper, pv, 1, 12, 0), + "SLN": mo.SLN(cost, salvage, life), + "DB": mo.DB(cost, salvage, life, 1), + "DDB": mo.DDB(cost, salvage, life, 1), + "SYD": mo.SYD(cost, salvage, life, 1), + }) + + def test_statistics_functions(self): + row = mo.Variable([10.0, 40.0, 20.0, 30.0]) + k = mo.Variable(0.25) + x = mo.Variable(20.0) + _calls({ + "MEDIAN": mo.MEDIAN(row), + "STDEV_S": mo.STDEV_S(row), + "STDEV_P": mo.STDEV_P(row), + "VAR_S": mo.VAR_S(row), + "VAR_P": mo.VAR_P(row), + "PERCENTILE_INC": mo.PERCENTILE_INC(row, k), + "PERCENTILE_EXC": mo.PERCENTILE_EXC(row, 0.4), + "QUARTILE_INC": mo.QUARTILE_INC(row, 1), + "QUARTILE_EXC": mo.QUARTILE_EXC(row, 3), + "LARGE": mo.LARGE(row, 2), + "SMALL": mo.SMALL(row, 2), + "RANK_EQ": mo.RANK_EQ(x, row), + "RANK_AVG": mo.RANK_AVG(x, row, 1), + "STDEV": mo.STDEV(row), + "STDEVP": mo.STDEVP(row), + "VAR": mo.VAR(row), + "VARP": mo.VARP(row), + "PERCENTILE": mo.PERCENTILE(row, k), + "QUARTILE": mo.QUARTILE(row, 1), + "RANK": mo.RANK(x, row), + }) def test_recurrence_functions(self): series = mo.Variable([1, 2, 3, 4, 5]) @@ -88,3 +243,8 @@ def test_self_ref_produces_ast(self): periods=3, ) assert result._expr is not None + + def test_every_exported_function_is_called_here(self): + called = set(re.findall(r"\bmo\.([A-Z][A-Z0-9_]*)\(", Path(__file__).read_text())) + exported = {n for n in functions.__all__ if n.isupper()} + assert exported - called == set(), "call each new export in a test above" diff --git a/tests/test_function_registry.py b/tests/test_function_registry.py new file mode 100644 index 0000000..8560d30 --- /dev/null +++ b/tests/test_function_registry.py @@ -0,0 +1,801 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The function registry: one place that says how each Excel function is +shaped, read by the Excel renderer and the loop container in place of +the name sets they used to keep by hand. + +Every name the engine knew before the registry keeps the spec those +sets gave it — the old literals are kept here as the expected values, +and the three names that gained a helper since are named — and the +whole registry equals those names and the table of the names added +since, so no name is registered, or changes its shape, silently. + +:func:`elementwise`, the shared per-period helper ``MIN`` / ``MAX`` +moved onto, must keep their law: scalars broadcast as ``a + b`` does, a +one-value series broadcasts, other length mismatches are refused, +tracked operands lift per coordinate, an error marker is the period's +result, a shared unit is kept. +""" + +from __future__ import annotations + +import dataclasses +import datetime as dt +import logging +import shutil +import subprocess +from pathlib import Path + +import pytest +from openpyxl import load_workbook + +import modeleon as mo +from modeleon import functions +from modeleon.core.expr import FuncCall, ListExpr, Literal, VarRef +from modeleon.functions import registry +from modeleon.functions._helpers import elementwise, is_error, is_series +from modeleon.functions.registry import ( + FunctionSpec, + admits_in_loop, + elementwise_names, + excel_name, + file_prefix, + file_prefixes, + is_elementwise, + is_passthrough, + is_range_taking, + names, + passthrough_names, + per_period_arity, + range_positions, + range_taking_names, + register, + spec, +) + +# The sets the Excel renderer kept by hand (compile/excel/renderer.py) +# before the registry. +OLD_RANGE_TAKING = { + "SUM", "AVERAGE", "MIN", "MAX", "COUNT", + "IRR", "NPV", "XIRR", "XNPV", + "SUMPRODUCT", +} +OLD_PASSTHROUGH = OLD_RANGE_TAKING | { + "IF", "AND", "OR", "NOT", "CHOOSE", "ISBLANK", "ABS", "ROUND", "INT", + "MOD", + "EDATE", "EOMONTH", "YEAR", "MONTH", "DAY", "DATE", "DAYS360", "TODAY", + "YEARFRAC", "DAYS", + "LEN", "UPPER", "LOWER", "TEXT", + "PMT", "FV", "PV", "RATE", +} +OLD_FILE_PREFIX = {"DAYS": "_xlfn."} +# The set the loop container kept by hand (core/loops.py). +OLD_ELEMENTWISE = { + 'IF', 'AND', 'OR', 'NOT', 'ABS', 'ROUND', 'INT', 'MOD', 'CHOOSE', + 'ISBLANK', 'EDATE', 'EOMONTH', 'YEAR', 'MONTH', 'DAY', 'DATE', + 'DAYS360', 'TODAY', 'LEN', 'UPPER', 'LOWER', 'CONCAT', +} +# Every name the engine knew before the registry. +OLD_NAMES = OLD_PASSTHROUGH | OLD_ELEMENTWISE +# Of the range-taking ones, these read whole series and give one number; +# the others take fixed arguments. +OLD_REDUCTIONS = {"SUM", "AVERAGE", "MIN", "MAX", "COUNT"} +# Written as given, without a helper, until they gained one. +# TEXT and RATE, plain calls before, are one call per period now, so a +# loop container runs them; COUNT stays a reduction. +GAINED_A_HELPER = {"COUNT", "TEXT", "RATE"} +NOW_ELEMENTWISE = {"TEXT", "RATE"} + +# The names added since: shape, file prefix, the argument +# positions written as ranges (None: every list argument). +NEW_NAMES: dict[str, tuple[str, str, tuple[int, ...] | None]] = { + # Logical and error (functions/logical.py) + "IFERROR": ("elementwise", "", None), + "ISERROR": ("elementwise", "", None), + "ISNUMBER": ("elementwise", "", None), + "XOR": ("elementwise", "_xlfn.", None), + "IFS": ("elementwise", "_xlfn.", None), + "SWITCH": ("elementwise", "_xlfn.", None), + # Math and rounding (functions/mathfn.py) + "ROUNDUP": ("elementwise", "", None), + "ROUNDDOWN": ("elementwise", "", None), + "TRUNC": ("elementwise", "", None), + "CEILING": ("elementwise", "", None), + "FLOOR": ("elementwise", "", None), + "POWER": ("elementwise", "", None), + "SQRT": ("elementwise", "", None), + "EXP": ("elementwise", "", None), + "LN": ("elementwise", "", None), + "LOG": ("elementwise", "", None), + "LOG10": ("elementwise", "", None), + "SIGN": ("elementwise", "", None), + "PRODUCT": ("reduction", "", None), + # Financial (functions/financial.py) + "NPER": ("elementwise", "", None), + "IPMT": ("elementwise", "", None), + "PPMT": ("elementwise", "", None), + "CUMIPMT": ("elementwise", "", None), + "CUMPRINC": ("elementwise", "", None), + "SLN": ("elementwise", "", None), + "DB": ("elementwise", "", None), + "DDB": ("elementwise", "", None), + "SYD": ("elementwise", "", None), + # Dates (functions/dates.py): the holidays are a range every period's + # call reads. + "DATEDIF": ("elementwise", "", None), + "WEEKDAY": ("elementwise", "", None), + "WEEKNUM": ("elementwise", "", None), + "NETWORKDAYS": ("range", "", (2,)), + "WORKDAY": ("range", "", (2,)), + # Text (functions/text.py) + "LEFT": ("elementwise", "", None), + "RIGHT": ("elementwise", "", None), + "MID": ("elementwise", "", None), + "TRIM": ("elementwise", "", None), + "FIND": ("elementwise", "", None), + "SEARCH": ("elementwise", "", None), + "SUBSTITUTE": ("elementwise", "", None), + "VALUE": ("elementwise", "", None), + "REPT": ("elementwise", "", None), + "TEXTJOIN": ("range", "_xlfn.", None), + # Conditional aggregates and counts (functions/criteria.py) + "SUMIF": ("range", "", None), + "SUMIFS": ("range", "", None), + "COUNTIF": ("range", "", None), + "COUNTIFS": ("range", "", None), + "AVERAGEIF": ("range", "", None), + "AVERAGEIFS": ("range", "", None), + "MAXIFS": ("range", "_xlfn.", None), + "MINIFS": ("range", "_xlfn.", None), + "COUNTA": ("reduction", "", None), + # Lookup and reference (functions/lookup.py); NA() is the not-found + # value an XLOOKUP is written with, without a helper of its own. + "MATCH": ("range", "", (1,)), + "INDEX": ("range", "", (0,)), + "XLOOKUP": ("range", "_xlfn.", (1, 2)), + "VLOOKUP": ("range", "", (1,)), + "HLOOKUP": ("range", "", (1,)), + "NA": ("scalar", "", None), + # Statistics (functions/statistics.py): the reductions read every cell + # of one argument, and two values or more with a series among them one + # period at a time; the others one range (RANK's at its second + # argument), a list of two or more lines with a series among them a + # range per period. The names an older workbook carries are + # their own entries, written as themselves. + "MEDIAN": ("reduction", "", None), + "STDEV_S": ("reduction", "_xlfn.", None), + "STDEV_P": ("reduction", "_xlfn.", None), + "VAR_S": ("reduction", "_xlfn.", None), + "VAR_P": ("reduction", "_xlfn.", None), + "STDEV": ("reduction", "", None), + "STDEVP": ("reduction", "", None), + "VAR": ("reduction", "", None), + "VARP": ("reduction", "", None), + "PERCENTILE_INC": ("range", "_xlfn.", (0,)), + "PERCENTILE_EXC": ("range", "_xlfn.", (0,)), + "QUARTILE_INC": ("range", "_xlfn.", (0,)), + "QUARTILE_EXC": ("range", "_xlfn.", (0,)), + "PERCENTILE": ("range", "", (0,)), + "QUARTILE": ("range", "", (0,)), + "LARGE": ("range", "", (0,)), + "SMALL": ("range", "", (0,)), + "RANK_EQ": ("range", "_xlfn.", (1,)), + "RANK_AVG": ("range", "_xlfn.", (1,)), + "RANK": ("range", "", (1,)), +} +# The names the workbook spells otherwise: Excel's dot is an underscore in +# Python. +EXCEL_SPELLING = { + "STDEV_S": "STDEV.S", "STDEV_P": "STDEV.P", "VAR_S": "VAR.S", "VAR_P": "VAR.P", + "PERCENTILE_INC": "PERCENTILE.INC", "PERCENTILE_EXC": "PERCENTILE.EXC", + "QUARTILE_INC": "QUARTILE.INC", "QUARTILE_EXC": "QUARTILE.EXC", + "RANK_EQ": "RANK.EQ", "RANK_AVG": "RANK.AVG", +} +# The names written as given, without a helper of their own. +BARE = {"SUMPRODUCT", "NA"} +# The new names that are one call per period from two values, a +# series among them: every statistic, PRODUCT and COUNTA — every aggregate +# follows the rule. +PER_PERIOD = { + n: 2 for n in ( + "MEDIAN", "STDEV_S", "STDEV_P", "VAR_S", "VAR_P", "STDEV", "STDEVP", "VAR", "VARP", + "PERCENTILE_INC", "PERCENTILE_EXC", "QUARTILE_INC", "QUARTILE_EXC", "PERCENTILE", + "QUARTILE", "LARGE", "SMALL", "RANK_EQ", "RANK_AVG", "RANK", "PRODUCT", "COUNTA", + ) +} +# Of the old names, the reductions, one call per period from two operands, +# a series among them: MIN, MAX, SUM, AVERAGE and COUNT. +OLD_PER_PERIOD = {"MIN", "MAX", "SUM", "AVERAGE", "COUNT"} + + +@pytest.fixture +def scratch(): + """Register throwaway names in a test; forget them after it.""" + created: list[str] = [] + + def _register(name: str, **kwargs): + created.append(name) + return register(name, **kwargs) + + yield _register + for name in created: + registry._SPECS.pop(name, None) + + +def _rows(ws): + out = {} + for row in ws.iter_rows(): + label = row[0].value + if label: + out[str(label)] = [c.value for c in row[1:]] + return out + + +def _book(model, tmp_path, sheet): + path = tmp_path / "book.xlsx" + model.to_excel(path) + return _rows(load_workbook(path)[sheet]) + + +def _call_var(name, args, value): + """A list-valued Variable whose formula is a hand-built call.""" + v = mo.Variable(formula=FuncCall(name, args)) + v._value = value + v.var_type = 'list' if isinstance(value, list) else 'scalar' + v.value_type = 'float' + return v + + +# ══════════════════════════════════════════════════════════════════════ +# The names the engine knew before the registry keep their specs +# ══════════════════════════════════════════════════════════════════════ + + +class TestTheOldNamesKeepTheirSpecs: + def test_range_taking(self): + assert range_taking_names() & OLD_NAMES == OLD_RANGE_TAKING + + def test_passthrough(self): + assert passthrough_names() & OLD_NAMES == OLD_PASSTHROUGH + + def test_elementwise(self): + assert elementwise_names() & OLD_NAMES == OLD_ELEMENTWISE | NOW_ELEMENTWISE + + def test_file_prefix(self): + assert {n: p for n, p in file_prefixes().items() if n in OLD_NAMES} == OLD_FILE_PREFIX + + def test_shapes(self): + for name in OLD_NAMES: + if name in OLD_REDUCTIONS: + expected = "reduction" + elif name in OLD_RANGE_TAKING: + expected = "range" + elif name in OLD_ELEMENTWISE | NOW_ELEMENTWISE: + expected = "elementwise" + else: + expected = "scalar" + assert spec(name).shape == expected, name + + def test_loop_admission(self): + per_period = OLD_ELEMENTWISE | NOW_ELEMENTWISE + for name in OLD_NAMES: + for n_args in range(4): + expected = name in per_period or (name in OLD_PER_PERIOD and n_args >= 2) + assert admits_in_loop(name, n_args) == expected, (name, n_args) + + def test_per_period_arity_is_the_reductions_over_two_operands(self): + assert {n for n in OLD_NAMES if per_period_arity(n) is not None} == OLD_PER_PERIOD + assert {per_period_arity(n) for n in OLD_PER_PERIOD} == {2} + assert OLD_PER_PERIOD == OLD_REDUCTIONS + + def test_the_three_that_gained_a_helper(self): + assert not any(spec(n).bare for n in GAINED_A_HELPER) + assert all(callable(getattr(functions, n, None)) for n in GAINED_A_HELPER) + assert spec("COUNT").shape == "reduction" + assert {spec(n).shape for n in NOW_ELEMENTWISE} == {"elementwise"} + + +# ══════════════════════════════════════════════════════════════════════ +# The registry is exactly the old names and the new names' table +# ══════════════════════════════════════════════════════════════════════ + + +class TestTheRegistryIsTheTable: + def test_nothing_else_is_registered(self): + # CONCAT is the one element-wise name the renderer does not pass + # through: it writes the ``&`` operator. + assert names() == OLD_NAMES | set(NEW_NAMES) + assert not OLD_NAMES & set(NEW_NAMES) + assert all(excel_name(n) == n for n in OLD_NAMES) + assert not is_passthrough("CONCAT") and is_elementwise("CONCAT") + + @pytest.mark.parametrize("name", sorted(NEW_NAMES)) + def test_each_new_name_has_its_spec(self, name): + shape, prefix, positions = NEW_NAMES[name] + assert spec(name) == FunctionSpec( + name=name, shape=shape, excel=EXCEL_SPELLING.get(name, name), file_prefix=prefix, + range_args=positions, per_period_arity=PER_PERIOD.get(name), bare=name in BARE, + ) + + def test_the_names_one_call_per_period_from_two_values(self): + assert {n for n in names() if per_period_arity(n) is not None} == ( + OLD_PER_PERIOD | set(PER_PERIOD)) + + def test_the_views_over_the_whole_registry(self): + new_ranges = {n for n, (shape, _, _) in NEW_NAMES.items() + if shape in ("reduction", "range")} + new_elementwise = {n for n, (shape, _, _) in NEW_NAMES.items() if shape == "elementwise"} + new_prefixes = {n: prefix for n, (_, prefix, _) in NEW_NAMES.items() if prefix} + assert range_taking_names() == OLD_RANGE_TAKING | new_ranges + assert passthrough_names() == OLD_PASSTHROUGH | set(NEW_NAMES) + assert elementwise_names() == OLD_ELEMENTWISE | NOW_ELEMENTWISE | new_elementwise + assert file_prefixes() == OLD_FILE_PREFIX | new_prefixes + + +# ══════════════════════════════════════════════════════════════════════ +# The queries +# ══════════════════════════════════════════════════════════════════════ + + +class TestQueries: + def test_shapes(self): + assert spec("SUM").shape == "reduction" + assert spec("NPV").shape == "range" + assert spec("IF").shape == "elementwise" + assert spec("PMT").shape == "scalar" + assert is_range_taking("SUM") and is_range_taking("NPV") + assert not is_range_taking("IF") and not is_range_taking("PMT") + + def test_every_existing_range_function_ranges_every_argument(self): + assert all(range_positions(n) is None for n in OLD_RANGE_TAKING) + + def test_an_unregistered_name_answers_the_safe_way(self): + assert spec("NOPE") is None + assert not is_passthrough("NOPE") + assert not is_range_taking("NOPE") + assert not is_elementwise("NOPE") + assert file_prefix("NOPE") == "" + assert excel_name("NOPE") == "NOPE" + assert range_positions("NOPE") is None + assert per_period_arity("NOPE") is None + assert not admits_in_loop("NOPE", 1) + + def test_bare_names_have_no_helper_and_every_other_name_has_one(self): + bare = {n for n in names() if spec(n).bare} + assert bare == BARE + for name in names() - bare: + assert hasattr(functions, name), name + for name in bare: + assert not hasattr(functions, name), name + + def test_every_excel_cased_export_is_registered(self): + exported = {n for n in functions.__all__ if n.isupper()} + assert exported == {n for n in names() if not spec(n).bare} + + def test_loop_admission(self): + assert admits_in_loop("IF", 3) and admits_in_loop("ABS", 1) + assert admits_in_loop("TODAY", 0) + assert not admits_in_loop("SUM", 1) and not admits_in_loop("AVERAGE", 1) + assert admits_in_loop("SUM", 2) and admits_in_loop("AVERAGE", 3) + assert admits_in_loop("MAX", 2) and admits_in_loop("MIN", 3) + assert not admits_in_loop("MAX", 1) + assert not admits_in_loop("NPV", 2) and not admits_in_loop("SUMPRODUCT", 2) + assert not admits_in_loop("PMT", 3) and not admits_in_loop("YEARFRAC", 2) + + def test_the_prefix_and_the_excel_name(self): + assert file_prefix("DAYS") == "_xlfn." and excel_name("DAYS") == "DAYS" + assert file_prefix("SUM") == "" and excel_name("SUM") == "SUM" + assert file_prefix("PERCENTILE_INC") == "_xlfn." + assert excel_name("PERCENTILE_INC") == "PERCENTILE.INC" + assert {n for n in names() if excel_name(n) != n} == set(EXCEL_SPELLING) + + +# ══════════════════════════════════════════════════════════════════════ +# Registering +# ══════════════════════════════════════════════════════════════════════ + + +class TestRegister: + def test_returns_a_frozen_spec_with_the_defaults_filled(self, scratch): + s = scratch("ZZPLAIN", shape="elementwise") + assert s == FunctionSpec(name="ZZPLAIN", shape="elementwise", excel="ZZPLAIN") + assert s.file_prefix == "" and s.range_args is None + assert s.per_period_arity is None and s.passthrough and not s.bare + with pytest.raises(dataclasses.FrozenInstanceError): + s.shape = "scalar" + + def test_the_same_spec_twice_is_one_entry(self, scratch): + first = scratch("ZZSAME", shape="scalar", bare=True) + again = scratch("ZZSAME", shape="scalar", bare=True) + assert first == again + assert "ZZSAME" in names() + + def test_a_different_spec_for_the_same_name_is_refused(self, scratch): + scratch("ZZDIFF", shape="elementwise") + with pytest.raises(ValueError, match="ZZDIFF.*already registered"): + register("ZZDIFF", shape="scalar") + assert spec("ZZDIFF").shape == "elementwise" + + def test_excel_name_and_prefix(self, scratch): + scratch("ZZ_INC", shape="elementwise", excel="ZZ.INC", file_prefix="_xlfn.") + assert excel_name("ZZ_INC") == "ZZ.INC" + assert file_prefix("ZZ_INC") == "_xlfn." + assert file_prefixes()["ZZ_INC"] == "_xlfn." + assert is_passthrough("ZZ_INC") + + def test_range_positions(self, scratch): + scratch("ZZPOS", shape="range", range_args=[0, 2]) + scratch("ZZALL", shape="range") + assert range_positions("ZZPOS") == (0, 2) + assert range_positions("ZZALL") is None + assert is_range_taking("ZZPOS") and is_range_taking("ZZALL") + assert "ZZPOS" in range_taking_names() + + def test_per_period_arity(self, scratch): + scratch("ZZCAP", shape="reduction", per_period_arity=3) + assert per_period_arity("ZZCAP") == 3 + assert admits_in_loop("ZZCAP", 3) and not admits_in_loop("ZZCAP", 2) + # A range function counts the lines its ranges list. + scratch("ZZPICK", shape="range", range_args=[0], per_period_arity=2) + assert per_period_arity("ZZPICK") == 2 + assert admits_in_loop("ZZPICK", 2) and not admits_in_loop("ZZPICK", 1) + + def test_not_passthrough_and_bare(self, scratch): + scratch("ZZOP", shape="elementwise", passthrough=False) + scratch("ZZBARE", shape="scalar", bare=True) + assert not is_passthrough("ZZOP") and is_elementwise("ZZOP") + assert "ZZOP" not in passthrough_names() and "ZZOP" in elementwise_names() + assert spec("ZZBARE").bare and is_passthrough("ZZBARE") + + @pytest.mark.parametrize("kwargs, message", [ + ({"shape": "table"}, "shape"), + ({"shape": "elementwise", "range_args": [0]}, "range_args"), + ({"shape": "range", "range_args": [-1]}, "range_args"), + ({"shape": "reduction", "per_period_arity": 1}, "per_period_arity"), + ({"shape": "range", "per_period_arity": 1}, "per_period_arity"), + ({"shape": "elementwise", "per_period_arity": 2}, "per_period_arity"), + ({"shape": "scalar", "per_period_arity": 2}, "per_period_arity"), + ]) + def test_a_bad_spec_is_refused(self, kwargs, message): + with pytest.raises(ValueError, match=message): + register("ZZBAD", **kwargs) + assert spec("ZZBAD") is None + + @pytest.mark.parametrize("name", ["", "sum", "Sum", "_xlfn.DAYS", "A B", "1X"]) + def test_a_name_must_be_an_upper_case_identifier(self, name): + with pytest.raises(ValueError, match="upper-case"): + register(name, shape="elementwise") + + +# ══════════════════════════════════════════════════════════════════════ +# The Excel renderer reads the registry +# ══════════════════════════════════════════════════════════════════════ + + +def _model(): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with m: + m.i = mo.MultiVariable("I") + m.i.d0 = mo.Variable(dt.date(2026, 1, 1), display_name="d0") + m.i.d1 = mo.Variable(dt.date(2026, 3, 1), display_name="d1") + m.x = mo.MultiVariable("X") + m.x.a = mo.Variable([3.0, 8.0, 5.0], display_name="a") + m.x.k = mo.Variable([10.0, 20.0, 30.0], display_name="k") + return m + + +class TestTheRendererReadsTheRegistry: + def test_the_file_prefix_is_written(self, tmp_path): + m = _model() + with m: + m.i.span = mo.DAYS(m.i.d1, m.i.d0).set_display_name("span") + i = _book(m, tmp_path, "I") + assert i["span"][0] == "=_xlfn.DAYS(B2, B1)", i["span"] + + def test_the_excel_name_and_its_prefix_are_written(self, scratch, tmp_path): + scratch("ZZ_INC", shape="elementwise", excel="ZZ.INC", file_prefix="_xlfn.") + m = _model() + with m: + m.x.z = _call_var("ZZ_INC", [VarRef(m.x.a)], [3.0, 8.0, 5.0]) + m.x.z.set_display_name("z") + x = _book(m, tmp_path, "X") + assert x["z"] == ["=_xlfn.ZZ.INC(B2)", "=_xlfn.ZZ.INC(C2)", "=_xlfn.ZZ.INC(D2)"], x["z"] + + def test_a_range_position_is_a_range_and_the_others_this_period_s_cell( + self, scratch, tmp_path): + scratch("ZZPOS", shape="range", range_args=[0]) + m = _model() + with m: + m.x.z = _call_var("ZZPOS", [VarRef(m.x.a), VarRef(m.x.k)], [26.0, 36.0, 46.0]) + m.x.z.set_display_name("z") + x = _book(m, tmp_path, "X") + assert x["z"] == ["=ZZPOS(B2:D2, B3)", "=ZZPOS(B2:D2, C3)", "=ZZPOS(B2:D2, D3)"], x["z"] + + def test_every_argument_is_a_range_when_no_positions_are_given(self, scratch, tmp_path): + scratch("ZZALL", shape="range") + m = _model() + with m: + m.i.z = _call_var("ZZALL", [VarRef(m.x.a), VarRef(m.x.k)], 76.0) + m.i.z.set_display_name("z") + i = _book(m, tmp_path, "I") + assert i["z"][0] == "=ZZALL(X!B2:D2, X!B3:D3)", i["z"] + + def test_a_per_period_literal_at_a_cell_position_is_not_an_unlaid_list( + self, scratch, tmp_path): + # Only range positions can make the call fall back to its value: + # a list literal at a cell position is this period's number. + scratch("ZZPOS", shape="range", range_args=[0]) + m = _model() + with m: + m.x.z = _call_var("ZZPOS", [VarRef(m.x.a), Literal([1.0, 2.0, 3.0])], + [17.0, 18.0, 19.0]) + m.x.z.set_display_name("z") + x = _book(m, tmp_path, "X") + assert x["z"] == ["=ZZPOS(B2:D2, 1.0)", "=ZZPOS(B2:D2, 2.0)", "=ZZPOS(B2:D2, 3.0)"], x["z"] + + def test_a_list_of_lines_at_a_per_period_range_is_this_column_s_run( + self, scratch, tmp_path): + # A range function one call per period: the lines a range lists are + # the run of their cells in this period's column, another argument + # this period's cell. + scratch("ZZPICK", shape="range", range_args=[0], per_period_arity=2) + m = _model() + with m: + m.x.z = _call_var("ZZPICK", [ListExpr([VarRef(m.x.a), VarRef(m.x.k)]), + VarRef(m.x.a)], [3.0, 8.0, 5.0]) + m.x.z.set_display_name("z") + m.x.one = _call_var("ZZPICK", [VarRef(m.x.a), VarRef(m.x.k)], [3.0, 8.0, 5.0]) + m.x.one.set_display_name("one") + x = _book(m, tmp_path, "X") + assert x["z"] == ["=ZZPICK(B2:B3, B2)", "=ZZPICK(C2:C3, C2)", "=ZZPICK(D2:D3, D2)"], x["z"] + # One line at the range position is the whole row, as before. + assert x["one"] == ["=ZZPICK(B2:D2, B3)", "=ZZPICK(B2:D2, C3)", "=ZZPICK(B2:D2, D3)"] + + def test_an_unknown_name_is_written_as_given_and_points_at_the_registry( + self, tmp_path, caplog): + m = _model() + with m: + m.x.z = _call_var("ZZNOPE", [VarRef(m.x.a)], [3.0, 8.0, 5.0]) + m.x.z.set_display_name("z") + with caplog.at_level(logging.WARNING, logger="modeleon"): + x = _book(m, tmp_path, "X") + assert x["z"] == ["=ZZNOPE(B2)", "=ZZNOPE(C2)", "=ZZNOPE(D2)"], x["z"] + assert any("ZZNOPE" in r.getMessage() and "registry" in r.getMessage() + for r in caplog.records) + + def test_the_old_paths_still_hold(self, tmp_path): + m = _model() + with m: + m.x.lo = mo.MIN(m.x.a, m.x.k).set_display_name("lo") + m.x.r = mo.ROUND(m.x.a / 3, 1).set_display_name("r") + m.i.total = mo.SUM(m.x.a).set_display_name("total") + x = _book(m, tmp_path, "X") + i = _book(m, tmp_path, "I") + assert x["lo"] == ["=MIN(B2, B3)", "=MIN(C2, C3)", "=MIN(D2, D3)"], x["lo"] + assert x["r"] == ["=ROUND(B2 / 3, 1)", "=ROUND(C2 / 3, 1)", "=ROUND(D2 / 3, 1)"], x["r"] + assert i["total"][0] == "=SUM(X!B2:D2)", i["total"] + + +# ══════════════════════════════════════════════════════════════════════ +# The loop container reads the registry +# ══════════════════════════════════════════════════════════════════════ + + +class TestTheLoopReadsTheRegistry: + def test_an_elementwise_function_runs_period_by_period(self): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with m: + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + d.bal = mo.lag(d.r, fill=10.0) * 1.37 + d.cap = mo.MIN(d.bal, 15.0) + d.r = mo.ROUND(d.cap, 1) + assert d.bal.value == pytest.approx([13.7, 18.769, 20.55]) + assert d.cap.value == pytest.approx([13.7, 15.0, 15.0]) + assert d.r.value == pytest.approx([13.7, 15.0, 15.0]) + + def test_a_reduction_is_refused(self): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with pytest.raises(ValueError, match="AVERAGE reads a whole series"): + with m: + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + d.bal = mo.lag(d.x, fill=10.0) * 1.37 + d.avg = mo.AVERAGE(d.bal) + d.x = d.bal - d.avg + + def test_a_name_the_engine_lacks_is_refused_by_name(self): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with pytest.raises(ValueError, match="ZZNOPE cannot be run one period at a time"): + with m: + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + d.bal = mo.lag(d.z, fill=10.0) * 1.37 + d.z = _call_var("ZZNOPE", [VarRef(d.bal)], [1.0, 1.0, 1.0]) + + +# ══════════════════════════════════════════════════════════════════════ +# The shared per-period helper +# ══════════════════════════════════════════════════════════════════════ + + +def _hypot(a, b): + return (a * a + b * b) ** 0.5 + + +class TestElementwiseHelper: + def test_scalars_only_is_one_call(self): + m = _model() + with m: + m.i.p = mo.Variable(3.0) + out = elementwise("ZZHYP", _hypot, [m.i.p, 4.0]) + assert out.value == 5.0 and out.var_type == "scalar" + assert out.formula == "ZZHYP(p, 4.0)" + assert isinstance(out._expr, FuncCall) and out._expr.func == "ZZHYP" + assert isinstance(out._expr.args[0], VarRef) + assert isinstance(out._expr.args[1], Literal) and out._expr.args[1].value == 4.0 + + def test_a_series_gives_a_series_and_scalars_broadcast(self): + a = mo.Variable([3.0, 6.0, 5.0]) + assert elementwise("ZZHYP", _hypot, [a, 4.0]).value == [5.0, _hypot(6.0, 4.0), _hypot(5.0, 4.0)] + assert elementwise("ZZHYP", _hypot, [4.0, a]).var_type == "list" + assert elementwise("ZZHYP", _hypot, [a, [4.0, 8.0, 12.0]]).value == [5.0, 10.0, 13.0] + + def test_a_one_value_series_broadcasts_and_other_lengths_are_refused(self): + a = mo.Variable([3.0, 6.0, 5.0]) + assert elementwise("ZZHYP", _hypot, [a, mo.Variable([4.0])]).value[0] == 5.0 + with pytest.raises(ValueError, match="different lengths"): + elementwise("ZZHYP", _hypot, [a, mo.Variable([4.0, 8.0])]) + + def test_tracked_operands_lift_per_track(self): + m = mo.Model("m", tracks=mo.Tracks("plan", "actual"), + default_grain="month", default_start="2026-01", default_periods=2) + with m: + m.x = mo.Variable(plan=[3.0, 6.0], actual=[5.0, 8.0]) + m.h = elementwise("ZZHYP", _hypot, [m.x, 4.0]) + assert dict(m.h._value.items()) == {"plan": [5.0, _hypot(6.0, 4.0)], + "actual": [_hypot(5.0, 4.0), _hypot(8.0, 4.0)]} + assert m.h.var_type == "list" + + def test_an_error_marker_is_the_period_s_result_first_in_reading_order(self): + e = mo.Variable([2.0, 2.0, 2.0]) / mo.Variable([1.0, 0.0, 2.0]) + out = elementwise("ZZHYP", _hypot, [e, mo.Variable([1.0, "#VALUE!", 1.0])]) + assert out.value == [_hypot(2.0, 1.0), "#DIV/0!", _hypot(1.0, 1.0)] + + def test_errors_can_be_handed_to_the_function_instead(self): + e = mo.Variable([2.0, "#DIV/0!"]) + caught = elementwise("ZZCATCH", lambda v, alt: alt if is_error(v) else v, + [e, 0.0], propagate_errors=False) + assert caught.value == [2.0, 0.0] + + def test_a_blank_reaches_the_function_unless_a_stand_in_is_given(self): + b = mo.Variable([None, 3.0]) + seen = elementwise("ZZSEEN", lambda v: v is None, [b], value_type='bool') + assert seen.value == [True, False] + assert elementwise("ZZZERO", lambda v: v + 1, [b], blank=0).value == [1, 4.0] + + def test_a_failed_call_is_the_spreadsheet_s_error(self): + assert elementwise("ZZDIV", lambda a, b: a / b, [mo.Variable([1.0, 1.0]), [0.0, 2.0]] + ).value == ["#DIV/0!", 0.5] + assert elementwise("ZZTYPE", lambda a: a + 1, [mo.Variable(["x"])]).value == ["#VALUE!"] + assert elementwise("ZZBIG", lambda a: float(10 ** a), [mo.Variable([400])] + ).value == ["#NUM!"] + + def test_exceptions_raise_lets_a_failed_call_propagate(self): + a = mo.Variable([1.0, 1.0]) + with pytest.raises(ZeroDivisionError): + elementwise("ZZDIV", lambda x, y: x / y, [a, [0.0, 2.0]], exceptions='raise') + with pytest.raises(TypeError): + elementwise("ZZTYPE", lambda x: x + 1, [mo.Variable(["x"])], exceptions='raise') + + def test_a_function_that_cannot_take_the_operands_is_refused_at_entry(self): + with pytest.raises(TypeError, match=r"ZZONE: the function given takes \(a\), " + r"but the call has 2 operands"): + elementwise("ZZONE", lambda a: a, [mo.Variable([1.0, 2.0]), 1.0]) + with pytest.raises(TypeError, match="has 1 operand"): + elementwise("ZZTWO", _hypot, [mo.Variable([1.0, 2.0])]) + # A builtin without a signature is trusted. + assert elementwise("ZZMAX", max, [mo.Variable([1.0, 5.0]), 3.0]).value == [3.0, 5.0] + + def test_value_type_follows_the_first_variable_unless_given(self): + assert elementwise("ZZHYP", _hypot, [mo.Variable([3, 4]), 4]).value_type == "int" + assert elementwise("ZZHYP", _hypot, [3.0, 4.0]).value_type == "float" + assert elementwise("ZZLEN", lambda s: len(s), [mo.Variable(["ab"])], + value_type='int').value_type == "int" + + def test_a_shared_unit_is_kept_and_a_mixed_one_dropped(self): + a = mo.Variable([3.0, 8.0], unit="$") + b = mo.Variable([4.0, 4.0], unit="$") + c = mo.Variable([4.0, 4.0], unit="hr") + assert str(elementwise("ZZHYP", _hypot, [a, b])._unit) == "$" + assert str(elementwise("ZZHYP", _hypot, [a, 1.0])._unit) == "$" + assert elementwise("ZZHYP", _hypot, [a, c])._unit is None + # A count or a span is not in the operands' unit. + assert elementwise("COUNT", lambda *v: len(v), [a, b], value_type='int')._unit is None + + def test_source_code_and_render_backends_are_forwarded(self): + a = mo.Variable([3.0, 8.0]) + out = elementwise("ZZHYP", _hypot, [a, 4.0], source_code="hyp(a, 4)", + render_backends=frozenset({'excel'})) + assert out._source_code == "hyp(a, 4)" + assert out._expr.render_backends == frozenset({'excel'}) + + def test_an_axised_operand_is_refused(self): + region = mo.Variable(["N", "S"], display_name="region") + by_region = mo.Variable([1.0, 2.0], indexed_by=[region]) + with pytest.raises(ValueError, match="ZZHYP"): + elementwise("ZZHYP", _hypot, [by_region, 4.0]) + + def test_is_series_and_is_error(self): + assert is_series([1.0]) and is_series(mo.Variable([1.0, 2.0])) + assert not is_series(1.0) and not is_series(mo.Variable(1.0)) + assert is_error("#DIV/0!") and is_error("#N/A") + assert not is_error("text") and not is_error(None) and not is_error(0) + + def test_min_and_max_still_use_it(self): + a = mo.Variable([3.0, 8.0, 5.0]) + assert mo.MIN(a, 4.0).value == [3.0, 4.0, 4.0] + assert mo.MAX(a, mo.Variable([None, None, None])).value == [3.0, 8.0, 5.0] + + def test_min_and_max_refuse_a_text_operand_at_build_time(self): + # The spreadsheet skips text in a referenced cell, so =MAX(B2, B7) + # over words is a number no marker could match; the call is + # refused as it always was, and nothing is written. + a = mo.Variable([3.0, -8.0, 0.0, 5.5]) + with pytest.raises(TypeError): + mo.MAX(a, mo.Variable(["x", "Yy", "", "zZz"])) + with pytest.raises(TypeError): + mo.MIN(a, mo.Variable("abc")) + with pytest.raises(TypeError): + mo.MAX(a, ["x", "Yy", "", "zZz"]) + + +# ══════════════════════════════════════════════════════════════════════ +# The spreadsheet computes the same numbers +# ══════════════════════════════════════════════════════════════════════ + + +def _soffice(): + for candidate in ("soffice", "libreoffice"): + found = shutil.which(candidate) + if found: + return found + mac = Path("/Applications/LibreOffice.app/Contents/MacOS/soffice") + return str(mac) if mac.exists() else None + + +@pytest.mark.slow +def test_the_workbook_recalculates_to_the_python_values(tmp_path): + """One function of each registry shape, written through the registry + and recalculated by LibreOffice: a reduction over a range, a + per-period cap, an element-wise call, a prefixed name, a range + function with a cell argument, a scalar financial call.""" + soffice = _soffice() + if soffice is None: + pytest.skip("LibreOffice not installed") + m = _model() + with m: + m.x.lo = mo.MIN(m.x.a, m.x.k).set_display_name("lo") + m.x.r = mo.ROUND(m.x.a / 3, 1).set_display_name("r") + m.x.pos = mo.ABS(m.x.a - m.x.k).set_display_name("pos") + m.i.total = mo.SUM(m.x.a).set_display_name("total") + m.i.span = mo.DAYS(m.i.d1, m.i.d0).set_display_name("span") + m.i.rate = mo.Variable(0.1, display_name="rate") + m.i.npv = mo.NPV(m.i.rate, m.x.k).set_display_name("npv") + m.i.pmt = mo.PMT(m.i.rate, 3, 100.0).set_display_name("pmt") + path = tmp_path / "registry.xlsx" + m.to_excel(path) + subprocess.run( + [soffice, f"-env:UserInstallation={(tmp_path / 'prof').as_uri()}", + "--headless", "--convert-to", "xlsx", "--outdir", + str(tmp_path / "calc"), str(path)], + check=True, capture_output=True, + ) + book = load_workbook(tmp_path / "calc" / "registry.xlsx", data_only=True) + x = _rows(book["X"]) + i = _rows(book["I"]) + for name in ("lo", "r", "pos"): + assert [round(v, 6) for v in x[name][:3]] == getattr(m.x, name).value, name + assert i["total"][0] == m.i.total.value == 16.0 + assert i["span"][0] == m.i.span.value == 59 + assert round(i["npv"][0], 6) == round(m.i.npv.value, 6) + assert round(i["pmt"][0], 6) == round(m.i.pmt.value, 6) diff --git a/tests/test_grain_vocabulary.py b/tests/test_grain_vocabulary.py new file mode 100644 index 0000000..9493668 --- /dev/null +++ b/tests/test_grain_vocabulary.py @@ -0,0 +1,132 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The grain vocabulary is one list, and every entry point says no by name. + +A word the engine does not know — a grain, a totals kind, a day label +that is really a week — must be refused where it is given, never +accepted and left to die (or lie) in a writer later. +""" +from datetime import date + +import pytest + +import modeleon as mo +from modeleon.compile.excel.header import spell_anchor, spell_inline +from modeleon.compile.excel.layout import LayoutEngine +from modeleon.compile.excel.view import resolve_excel_view +from modeleon.core.time import GRAINS, _parse + + +class TestDayLabels: + def test_a_week_label_is_not_a_day_start(self): + # date.fromisoformat reads '2026-W01' as the Monday it names; a + # day window would then start on 2025-12-29 under a label that + # never spells that day. + with pytest.raises(ValueError, match="Time day start must be 'YYYY-MM-DD'"): + mo.Model('m', default_grain='day', default_start='2026-W01', default_periods=3) + + @pytest.mark.parametrize("label", ["20260105", "2026-1-5", "2026-01-05T10:00", 20260105]) + def test_only_the_dashed_form_is_a_day_label(self, label): + with pytest.raises(ValueError, match="Time day start must be 'YYYY-MM-DD'"): + _parse(label, 'day') + + def test_a_dashed_day_label_still_parses(self): + assert _parse('2026-01-05', 'day') == date(2026, 1, 5) + m = mo.Model('m', default_grain='day', default_start='2026-01-05', default_periods=2) + with m: + m.x = mo.Variable([1.0, 2.0]) + assert m.x.time.start == '2026-01-05' + + +class TestGrainOnAssignment: + def test_an_unknown_grain_is_refused_when_assigned(self): + # The constructor refused it; plain assignment used to accept it + # and the workbook writer died on it later, naming no attribute. + m = mo.Model('m') + with pytest.raises(ValueError, match="Unknown default_grain 'minute'; supported: hour, "): + m.default_grain = 'minute' + with pytest.raises(ValueError, match="Unknown default_grain 'fortnight'"): + m.default_grain = 'fortnight' + m.default_grain = 'hour' + assert m.default_grain == 'hour' + + def test_a_known_grain_assigns(self): + m = mo.Model('m') + m.default_grain = 'quarter' + assert m.default_grain == 'quarter' + m.default_grain = None + assert m.default_grain is None + + +class TestTotalsKinds: + def test_a_view_refuses_a_totals_kind_it_cannot_draw(self): + # An unknown kind used to be accepted and dropped: the model + # declared the columns and the book showed none, without a word. + with pytest.raises(ValueError, match="ExcelView timeline totals may name day, week, month, quarter, half, year, iso_year; got 'fortnight'"): + mo.ExcelView(timeline={'totals': ['fortnight']}) + # Nothing is finer than an hour: it is never a total. + with pytest.raises(ValueError, match="got 'hour'"): + mo.ExcelView(timeline={'totals': ['quarter', 'hour']}) + + def test_the_known_kinds_resolve_in_their_order(self): + m = mo.Model('m', default_grain='month', default_start='2026-01', default_periods=6, + default_excel_view=mo.ExcelView(timeline={'totals': ['year', 'quarter']})) + with m: + m.x = mo.Variable([1.0] * 6, regrain=mo.up('sum')) + assert resolve_excel_view(m).time_totals == ('year', 'quarter') + + def test_a_layout_override_refuses_an_unknown_kind(self): + m = mo.Model('m', default_grain='month', default_start='2026-01', default_periods=3) + with m: + m.x = mo.Variable([1.0] * 3, regrain=mo.up('sum')) + with pytest.raises(ValueError, match="A totals column is one of hour, day, week, month, quarter, half, year, iso_year; got 'fortnight'"): + LayoutEngine([m], totals_override=('fortnight',)) + # The override's own vocabulary is the engine's: a caller may ask + # for a kind the declaration cannot (a month column over days). + LayoutEngine([m], totals_override=('month',)) + LayoutEngine([m], totals_override=()) + + +class TestHeaderSpelling: + def test_the_header_refuses_a_grain_it_cannot_step(self): + # An unknown word used to be spelled as one month (STEP.get(grain, + # 1)) — a silent 'C2-C2+1' in the days row. + with pytest.raises(ValueError, match="got grain 'minute'"): + spell_anchor('days', 1, 'minute', 'B2', lambda f, i: None) + # An hour spells: its fraction of a day. + assert spell_anchor('days', 1, 'hour', 'B2', lambda f, i: None) == '1/24' + # Every ruler of the calendar is spelled; a word it does not know + # is refused by name, the spelled rulers listed. + with pytest.raises(ValueError, match="spells periods of hour, day, week, month, quarter, half, year or iso_year; got grain 'minute'"): + spell_inline('end', 1, 'minute', 'B2') + assert spell_inline('end', 0, 'iso_year', 'B2').startswith('((DATE(YEAR(B2+3)+1,1,4)') + + def test_the_known_grains_still_spell(self): + assert spell_inline('end', 1, 'quarter', 'B2') == 'EOMONTH(B2,5)' + assert spell_inline('end', 0, 'day', 'B2') == 'B2' + + def test_a_day_windows_anchor_spells_its_index_row(self, tmp_path): + # The index chain needs no month stride; refusing a day grain + # there would break every day model with a header that + # counts its periods. + cell = lambda f, i: f"I{i + 3}" if i >= 0 else None # noqa: E731 + assert spell_anchor('index', 1, 'day', 'B2', cell) == 'I3+1' + assert spell_anchor('index', 0, 'day', 'B2', lambda f, i: None) == '1' + m = mo.Model('m', default_grain='day', default_start='2026-01-05', default_periods=3) + with m: + m.header = mo.MultiVariable('Time') + with m.header as h: + h.start = mo.time.start + h.index = mo.time.index + m.default_header = m.header + m.x = mo.Variable([1.0, 2.0, 3.0]) * mo.time.index + path = tmp_path / 'day.xlsx' + m.to_excel(path) + from openpyxl import load_workbook + ws = load_workbook(path)['Time'] + row = next(r for r in ws.iter_rows() if r[0].value == 'Index') + assert [c.value for c in row[1:4]] == ['=1', '=B3+1', '=C3+1'] + assert m.x.value == [1.0, 4.0, 9.0] + + +def test_the_vocabulary_is_one_tuple(): + assert GRAINS == ('hour', 'day', 'week', 'month', 'quarter', 'half', 'year', 'iso_year') diff --git a/tests/test_half_year.py b/tests/test_half_year.py new file mode 100644 index 0000000..8a35e26 --- /dev/null +++ b/tests/test_half_year.py @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The half-year: a ruler of the gregorian calendar. + +Six whole months, keyed ``YYYY-Hn``, written ``H1 2026``; a window of +its own, a bucket between quarter and year, a target of ``.at()`` — +every answer derived from the calendar, nothing per-grain added. +""" +from datetime import date + +import pytest +from openpyxl import load_workbook + +import modeleon as mo +from modeleon.compile.excel.header import spell_anchor, spell_inline +from modeleon.compile.excel.totals import build_totals_plan, display_label, header_rows, short_label +from modeleon.core.time import ( + _period_labels, _target_label, bucket_ranges, period_weights, time_field_series, + window_start_label, +) + + +class TestTheRuler: + def test_labels_in_three_styles(self): + assert _period_labels('2026-H1', None, 3, 'half', 'finance') == ['H1 2026', 'H2 2026', 'H1 2027'] + assert _period_labels('2026-H2', None, 2, 'half') == ['2026-H2', '2027-H1'] + assert _period_labels('2026-H1', None, 2, 'half', 'compact') == ['H1-26', 'H2-26'] + + def test_a_half_window_reads_its_time(self): + m = mo.Model('m', default_grain='half', default_start='2026-H1', default_periods=4) + with m: + m.days = mo.time.days + m.months = mo.time.months + m.fee = mo.Variable(100.0) * mo.time.days + assert m.days.value == [181, 184, 181, 184] + assert m.months.value == [6, 6, 6, 6] + assert m.fee.value == [18100.0, 18400.0, 18100.0, 18400.0] + assert time_field_series('end', '2026-H1', 'half', 2) == [date(2026, 6, 30), date(2026, 12, 31)] + + def test_a_half_window_may_open_inside_its_first_half(self): + # A valuation on 1 September: the first half is September to + # December, keeps its label and counts its own days. + m = mo.Model('m', default_grain='half', default_start=mo.DATE(2026, 9, 1), default_periods=3) + with m: + m.days = mo.time.days + assert m.default_start == '2026-H2' + assert m.days.value == [122, 181, 184] + assert window_start_label(date(2026, 7, 1), 'half') == '2026-H2' + assert window_start_label(date(2026, 2, 1), 'half') == '2026-H1' + with pytest.raises(ValueError, match="A half window starts on the 1st of a month"): + window_start_label(date(2026, 2, 15), 'half') + + def test_months_and_quarters_roll_into_halves_and_halves_into_years(self): + assert bucket_ranges('month', 'half', '2026-11', 8) == [('2026-H2', 0, 2), ('2027-H1', 2, 8)] + assert bucket_ranges('quarter', 'half', '2026-Q2', 4) == [('2026-H1', 0, 1), ('2026-H2', 1, 3), ('2027-H1', 3, 4)] + assert bucket_ranges('half', 'year', '2026-H2', 3) == [('2026', 0, 1), ('2027', 1, 3)] + assert _target_label((2026, 11), 'month', 'half') == '2026-H2' + assert period_weights('half', '2026-H1', 2) == [181, 184] + with pytest.raises(ValueError, match="Re-grain coarsens only: half -> quarter"): + bucket_ranges('half', 'quarter', '2026-H1', 2) + + def test_at_half_sums_a_monthly_line(self): + m = mo.Model('m', default_grain='month', default_start='2026-01', default_periods=12) + with m: + m.flow = mo.Variable([float(i + 1) for i in range(12)], regrain=mo.up('sum')) + m.level = mo.Variable([10.0] * 12, regrain=mo.up('last')) + assert m.flow.at('half').value == [21.0, 57.0] + assert m.flow.at('half').time.start == '2026-H1' + assert m.level.at('half').value == [10.0, 10.0] + + +class TestTheBook: + def test_half_totals_interleave_between_quarters_and_the_year(self): + plan = build_totals_plan(('quarter', 'half', 'year'), 'month', '2026-01', 12, 3) + assert plan is not None + kinds = [k for k, _i in plan.columns] + # … Jun (Q2) (H1) Jul … Dec (Q4) (H2) (2026): the half closes + # after the quarter that ends with it, the year after both. + assert kinds[6:10] == ['period', 'quarter', 'half', 'period'] + assert kinds[-4:] == ['period', 'quarter', 'half', 'year'] + tiers = [row['cells'][0]['kind'] for row in header_rows(plan, _period_labels('2026-01', None, 12, 'month', 'finance'))] + assert tiers == ['year', 'half', 'quarter', 'period'] + assert display_label('2026-H1', 'half', 'finance') == 'H1 2026' + assert display_label('2026-H1', 'half', 'iso') == '2026-H1' + assert short_label('2026-H2', 'half', 'finance', True) == 'H2' + labels = [c['label'] for row in header_rows(plan, _period_labels('2026-01', None, 12, 'month', 'finance')) + for c in row['cells'] if c['kind'] == 'half'] + assert labels == ['H1', 'Total H1', 'H2', 'Total H2'] + + def test_a_half_bucket_is_a_live_sum_over_its_months(self, tmp_path): + m = mo.Model('m', default_grain='month', default_start='2026-01', default_periods=12, + default_excel_view=mo.ExcelView(timeline={'totals': ['half']})) + with m: + m.flow = mo.Variable([1.0] * 12, regrain=mo.up('sum'), display_name='Flow') + path = tmp_path / 'half.xlsx' + m.to_excel(path) + ws = load_workbook(path)['M'] + row = next(r for r in ws.iter_rows() if r[0].value == 'Flow') + formulas = [c.value for c in row if isinstance(c.value, str) and c.value.startswith('=')] + assert formulas == ['=SUM(B3,C3,D3,E3,F3,G3)', '=SUM(I3,J3,K3,L3,M3,N3)'] + # The half is the top tier of this book: 'H1' spans January to June. + assert [c.value for c in ws[1] if c.value][:1] == ['H1'] + + def test_the_header_spells_a_half_window_by_six_months(self): + assert spell_inline('end', 1, 'half', 'B2') == 'EOMONTH(B2,11)' + assert spell_inline('months', 0, 'half', 'B2') == '6' + cell = lambda f, i: None # noqa: E731 + assert spell_anchor('end', 0, 'half', 'B2', cell) == 'EOMONTH(B2,5)' + # A short first half: each period still ends with its half. + assert spell_inline('end', 0, 'half', 'B2', date(2026, 9, 1)) == 'EOMONTH(B2,5-MOD(MONTH(B2)-1,6))' + + +def test_the_words_a_reader_sees(): + from modeleon.core.calendar import HALF + assert (HALF.word, HALF.letter, HALF.grammar) == ('Half-yearly', 'H', "'YYYY-Hn'") diff --git a/tests/test_helper_functions.py b/tests/test_helper_functions.py index b083ad9..9d6aafa 100644 --- a/tests/test_helper_functions.py +++ b/tests/test_helper_functions.py @@ -94,8 +94,12 @@ def test_sum_single_list_variable(self): assert mo.SUM(mo.Variable([10, 20, 30]))._value == 60 def test_sum_mixed_variable_and_scalar(self): + # A series and a scalar: the scalar is added in every period, as + # ``a + 100`` adds it; the total of the series and the scalar is + # ``mo.SUM(row) + 100``. result = mo.SUM(mo.Variable([10, 20]), 100) - assert result._value == 130 + assert result._value == [110, 120] + assert (mo.SUM(mo.Variable([10, 20])) + 100)._value == 130 def test_sum_all_scalars_returns_variable(self): # =SUM(1, 2, 3) is a valid Excel formula — so we return a Variable, diff --git a/tests/test_hour.py b/tests/test_hour.py new file mode 100644 index 0000000..4433143 --- /dev/null +++ b/tests/test_hour.py @@ -0,0 +1,343 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Hours: the wall clock of one fixed-offset zone. + +An hour is ``[13:00, 14:00)``, keyed ``2026-10-21T13``. Its instants are +the clock of the window's zone (``default_zone='+01:00'``), where every +day has 24 hours — a zone that moves its clocks is refused where it is +declared. ``mo.time.end`` is the hour's stop, ``mo.time.days`` its 1/24 +of a day; an hour nests in its day, and so in everything a day nests in. +A sheet holds 16,384 columns: an hourly window that needs more is +refused with the count and the two ways out, never cut. +""" +import warnings +from datetime import date, datetime, timezone + +import openpyxl +import pytest + +import modeleon as mo +from modeleon.compile.excel import header as H +from modeleon.compile.excel.layout import LayoutEngine +from modeleon.compile.excel.renderer import _value_to_excel_literal +from modeleon.compile.excel.totals import build_totals_plan, header_rows +from modeleon.core.calendar import GREGORIAN, HOUR +from modeleon.core.time import ( + SheetWidthError, _period_labels, bucket_ranges, period_bounds, period_weights, + time_field_series, window_start_label, zone_offset, +) +from modeleon.core.variable_ops import _to_serial + +from test_excel_time_header import _recalculated + + +def _hourly(n=30, start='2026-10-21T22', totals=None, zone='+01:00'): + kw = {'default_excel_view': mo.ExcelView(timeline={'totals': totals})} if totals else {} + m = mo.Model('hr', default_grain='hour', default_start=start, default_periods=n, + default_zone=zone, **kw) + with m: + m.header = mo.MultiVariable('Time') + with m.header as h: + h.start = mo.time.start + h.end = mo.time.end + h.days = mo.time.days + h.index = mo.time.index + m.default_header = m.header + m.ops = mo.MultiVariable('Ops') + with m.ops as o: + o.load = mo.Variable([float(i % 5) for i in range(n)], regrain=mo.up('sum'), + display_name='Load') + o.rate = mo.Variable(120.0, display_name='Rate per day') + o.cost = (o.rate * mo.time.days).set_display_name('Cost') + return m + + +def _spans(n=30): + """Time read inside formulas on an hourly sheet with no Time header.""" + m = mo.Model('sp', default_grain='hour', default_start='2026-10-21T22', default_periods=n, + default_zone='+01:00', default_excel_view=mo.ExcelView(timeline={'totals': ['day']})) + with m: + m.ops = mo.MultiVariable('Ops') + with m.ops as o: + o.span = ((mo.time.end - mo.time.start) * 24).set_display_name('Span') + o.twice = (mo.time.start * 2).set_display_name('Twice start') + return m + + +def _book(m, path): + out = path / 'hours.xlsx' + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + m.to_excel(str(out)) + return openpyxl.load_workbook(out) + + +def _row(ws, label): + return next(r for r in ws.iter_rows() if r[0].value == label) + + +class TestTheRuler: + def test_an_hour_is_keyed_and_read_back_strictly(self): + first = HOUR.parse('2026-10-21T13') + assert first == datetime(2026, 10, 21, 13) + assert HOUR.label(first) == '2026-10-21T13' + assert HOUR.label(first, 'finance') == '21 Oct 2026 13:00' + assert HOUR.label(first, 'compact') == '21-Oct-26 13:00' + for bad in ('2026-10-21', '2026-10-21T24', '2026-10-21T13:30', '2026-10-21 13'): + with pytest.raises(ValueError, match="'YYYY-MM-DDTHH'|00-23"): + HOUR.parse(bad) + + def test_an_hour_stops_where_the_next_starts(self): + assert HOUR.next_start(datetime(2026, 12, 31, 23)) == datetime(2027, 1, 1, 0) + assert HOUR.start_of(datetime(2026, 10, 21, 13, 59, 59)) == datetime(2026, 10, 21, 13) + assert HOUR.start_of(date(2026, 10, 21)) == datetime(2026, 10, 21, 0) + assert HOUR.measure(datetime(2026, 10, 21, 13)) == 1 / 24 + assert HOUR.count(datetime(2026, 10, 21, 22), datetime(2026, 10, 23, 3)) == 30 + + def test_a_fixed_offset_day_has_24_hours_even_when_europe_moves_its_clocks(self): + # 29 March 2026: clocks move in Europe. The wall clock of a fixed + # offset never skips an hour. + assert len(bucket_ranges('hour', 'day', '2026-03-29T00', 24)) == 1 + assert _period_labels('2026-03-29T01', None, 3, 'hour') == [ + '2026-03-29T01', '2026-03-29T02', '2026-03-29T03'] + + def test_an_hour_nests_in_its_day_and_everything_above(self): + assert GREGORIAN.rolls_up_to('hour') == ( + 'day', 'week', 'month', 'quarter', 'half', 'year', 'iso_year') + GREGORIAN.verify() + + +class TestTheWindow: + def test_a_window_starts_on_the_hour(self): + assert window_start_label(datetime(2026, 10, 21, 13), 'hour') == '2026-10-21T13' + assert window_start_label(date(2026, 10, 21), 'hour') == '2026-10-21T00' + with pytest.raises(ValueError, match="starts on the hour; 2026-10-21T13:30:00"): + window_start_label(datetime(2026, 10, 21, 13, 30), 'hour') + with pytest.raises(ValueError, match="wall-clock time of its zone"): + window_start_label(datetime(2026, 10, 21, 13, tzinfo=timezone.utc), 'hour') + # The other rulers still refuse a clock. + with pytest.raises(ValueError, match="starts at a day, not at a time of day"): + window_start_label(datetime(2026, 10, 21, 13), 'day') + + def test_the_zone_is_a_fixed_offset(self): + assert zone_offset('UTC') == zone_offset('Z') == '+00:00' + assert zone_offset('+0100') == '+01:00' and zone_offset('-03:30') == '-03:30' + with pytest.raises(ValueError, match="fixed offset from UTC.*Asia/Tokyo is \\+09:00"): + zone_offset('Asia/Tokyo') + with pytest.raises(ValueError, match="fixed offset"): + zone_offset('+01:20') + m = mo.Model('m', default_grain='hour', default_start='2026-10-21T00', default_periods=2, + default_zone='UTC') + assert m.default_zone == '+00:00' + m.default_zone = '+01:00' + assert m.default_zone == '+01:00' + with pytest.raises(ValueError, match="fixed offset"): + m.default_zone = 'Europe/Berlin' + + def test_mo_time_of_an_hour(self): + assert time_field_series('start', '2026-10-21T23', 'hour', 2) == [ + datetime(2026, 10, 21, 23), datetime(2026, 10, 22, 0)] + assert time_field_series('end', '2026-10-21T23', 'hour', 2) == [ + datetime(2026, 10, 22, 0), datetime(2026, 10, 22, 1)] + assert time_field_series('days', '2026-10-21T23', 'hour', 2) == [1 / 24, 1 / 24] + assert period_bounds('2026-10-21T23', 'hour', 1) == ( + [datetime(2026, 10, 21, 23)], [datetime(2026, 10, 22, 0)]) + m = _hourly(n=3) + assert m.header.days.value_type == 'float' + assert m.ops.cost.value == pytest.approx([5.0, 5.0, 5.0]) + + def test_an_hour_has_no_months(self): + m = mo.Model('m', default_grain='hour', default_start='2026-10-21T00', default_periods=2) + with m, pytest.raises(ValueError, match="An hour window has none - read mo.time.days"): + m.months = mo.time.months + + +class TestRollUps: + def test_hours_roll_up_to_days_by_sum_last_and_weighted_mean(self): + m = mo.Model('m', default_grain='hour', default_start='2026-10-21T22', default_periods=5) + with m: + m.flow = mo.Variable([1.0, 2.0, 3.0, 4.0, 5.0], regrain=mo.up('sum')) + m.level = mo.Variable([1.0, 2.0, 3.0, 4.0, 5.0], regrain=mo.up('last')) + m.price = mo.Variable([1.0, 2.0, 3.0, 4.0, 5.0], regrain=mo.up('mean')) + assert m.flow.at('day').value == [3.0, 12.0] + assert m.level.at('day').value == [2.0, 5.0] + assert m.price.at('day').value == pytest.approx([1.5, 4.0]) + assert m.flow.at('week').value == [15.0] and m.flow.at('month').value == [15.0] + assert period_weights('hour', '2026-10-21T22', 2) == [1 / 24, 1 / 24] + + def test_a_day_line_is_never_viewed_by_hours(self): + with pytest.raises(ValueError, match="coarsens only: day -> hour"): + bucket_ranges('day', 'hour', '2026-10-21', 2) + + def test_an_hourly_line_and_a_daily_line_meet_only_through_at(self): + h = mo.Model('h', default_grain='hour', default_start='2026-10-21T00', default_periods=2) + with h: + h.a = mo.Variable([1.0, 2.0], regrain=mo.up('sum')) + d = mo.Model('d', default_grain='day', default_start='2026-10-21', default_periods=1) + with d: + d.b = mo.Variable([10.0], regrain=mo.up('sum')) + with pytest.raises(ValueError, match=r"\(hours\) and 'b' \(days\).*a.at\('day'\)"): + h.a + d.b + + +class TestTheBook: + def test_an_hourly_header_is_a_closed_form_off_the_start_cell(self, tmp_path): + ws = _book(_hourly(n=4), tmp_path)['Time'] + start, end, days, index = (_row(ws, k) for k in ('Start', 'End', 'Days', 'Index')) + assert [c.value for c in start[1:4]] == [ + '=DATE(2026,10,21)+TIME(22,0,0)', + '=DATE(2026,10,21)+TIME(22,0,0)+1/24', + '=DATE(2026,10,21)+TIME(22,0,0)+2/24', + ] + assert [c.value for c in end[1:3]] == ['=B2+1/24', '=C2+1/24'] + assert [c.value for c in days[1:3]] == ['=B3-B2', '=C3-C2'] + assert days[1].number_format == '0.0000' + assert [c.value for c in index[1:3]] == ['=1', '=B5+1'] + + def test_captions_show_the_hour_itself(self, tmp_path): + ws = _book(_hourly(n=3), tmp_path)['Ops'] + # A flat sheet's caption is the hour's start cell — its own + # mirrored start row — shown as a clock. + assert ws['A2'].value == 'Start' and ws['C1'].value == '=C2' + assert ws['C1'].number_format == 'dd mmm yyyy hh:mm' + + def test_day_totals_are_live_sums_of_their_hours(self, tmp_path): + wb = _book(_hourly(n=30, totals=['day']), tmp_path) + ws = wb['Ops'] + load = _row(ws, 'Load') + sums = [c.value for c in load if isinstance(c.value, str) and c.value.startswith('=SUM')] + assert sums[0] == '=SUM(C8,D8)' # 22:00 and 23:00 of the 21st + assert sums[1].count(',') == 23 # all 24 hours of the 22nd + assert len(sums) == 3 + # The day reads in full over hours alone; the hours keep their clock. + assert ws['C1'].value == '21 Oct 2026' and ws['C2'].value == '22:00' + assert ws['E1'].value == 'Total 21 Oct 2026' + + def test_a_days_time_in_its_total_is_read_off_its_hours(self, tmp_path): + ws = _book(_hourly(n=30, totals=['day']), tmp_path)['Time'] + totals = [c for c in range(1, ws.max_column + 1) + if str(ws.cell(1, c).value or '').startswith('Total')] + start, end, days = (_row(ws, k) for k in ('Start', 'End', 'Days')) + # The 21st holds 22:00 and 23:00: its total starts at the first + # hour's start, stops at the last hour's stop, lasts their days. + # (Two header rows over the time rows: the day tier, the hours.) + assert start[totals[0] - 1].value == '=B3' + assert end[totals[0] - 1].value == '=C4' + assert days[totals[0] - 1].value == '=SUM(B5,C5)' + assert days[totals[1] - 1].value.count(',') == 23 + # Each keeps what the engine computed, for a reader that does not + # recalculate. + cached = openpyxl.load_workbook(tmp_path / 'hours.xlsx', data_only=True)['Time'] + col = openpyxl.utils.get_column_letter(totals[0]) + assert cached[f'{col}3'].value == datetime(2026, 10, 21, 22) + assert cached[f'{col}4'].value == datetime(2026, 10, 22) + assert cached[f'{col}5'].value == pytest.approx(2 / 24) + + def test_an_instant_inside_a_formula_stands_in_brackets(self, tmp_path): + ws = _book(_spans(), tmp_path)['Ops'] + span, twice = _row(ws, 'Span'), _row(ws, 'Twice start') + # 22:00 is DATE()+TIME(), a sum: wherever a formula reads it — the + # first hour, a day's total — it stands in brackets, or Excel reads + # x - start as (x - DATE()) + TIME(). + assert span[1].value == ('=((DATE(2026,10,21)+TIME(22,0,0)+1/24)' + ' - (DATE(2026,10,21)+TIME(22,0,0))) * 24') + assert twice[1].value == '=(DATE(2026,10,21)+TIME(22,0,0)) * 2' + assert span[3].value == '=(DATE(2026, 10, 22) - (DATE(2026, 10, 21)+TIME(22, 0, 0))) * 24' + assert twice[3].value == '=(DATE(2026, 10, 21)+TIME(22, 0, 0)) * 2' + + def test_a_month_tier_over_days_over_hours(self): + plan = build_totals_plan(('day', 'month'), 'hour', '2026-10-31T22', 4, 3) + rows = header_rows(plan, _period_labels('2026-10-31T22', None, 4, 'hour', 'finance')) + tiers = [[c['label'] for c in r['cells']] for r in rows] + assert tiers[0][0] == 'Oct' and tiers[1][0] == '31' + assert [c['label'] for c in rows[-1]['cells'] if c['kind'] == 'period'] == [ + '22:00', '23:00', '00:00', '01:00'] + + def test_the_opening_column_is_where_the_first_hour_starts(self): + assert H.spell_anchor('start', 0, 'hour', 'B2', lambda f, i: 'O9' if i < 0 else None) == 'B2' + from modeleon.compile.excel.writer import opening_value + m = _hourly(n=2) + assert opening_value(m.header.end) == datetime(2026, 10, 21, 22) + + def test_an_instant_literal_keeps_its_clock(self): + assert _value_to_excel_literal(datetime(2026, 1, 5, 13)) == '(DATE(2026, 1, 5)+TIME(13, 0, 0))' + assert _value_to_excel_literal(datetime(2026, 1, 5)) == 'DATE(2026, 1, 5)' + assert H.date_literal(datetime(2026, 1, 5, 13)) == 'DATE(2026,1,5)+TIME(13,0,0)' + assert H.date_literal(date(2026, 1, 5)) == 'DATE(2026,1,5)' + + @pytest.mark.slow + def test_the_hourly_book_recalculates_to_the_engine(self, tmp_path): + m = _hourly(n=30, totals=['day']) + wb = _recalculated(m, tmp_path) + for sheet, mv, names in (('Time', m.header, ('start', 'end', 'days')), + ('Ops', m.ops, ('cost',))): + ws = wb[sheet] + caption = next(r for r in ws.iter_rows(max_row=4) + if any(isinstance(c.value, str) and c.value.endswith(':00') for c in r)) + hours = [c.column for c in caption if isinstance(c.value, str) and c.value.endswith(':00')] + assert len(hours) == 30 + rows = {r[0].value: r for r in ws.iter_rows() if r[0].value is not None} + for name in names: + var = getattr(mv, name) + got = [round(_to_serial(rows[var.display_name][c - 1].value), 6) for c in hours] + want = [round(_to_serial(v), 6) for v in var.value] + assert got == pytest.approx(want), (sheet, name, got[:3], want[:3]) + # The Time sheet's day totals recalculate to the hours they hold. + t = wb['Time'] + tot = [c for c in range(1, t.max_column + 1) if str(t.cell(1, c).value or '').startswith('Total')] + days_row = next(r for r in t.iter_rows() if r[0].value == 'Days') + assert [round(days_row[c - 1].value, 6) for c in tot] == [round(2 / 24, 6), 1.0, round(4 / 24, 6)] + # A day's total of the cost row is its hours' cost: 2, 24 and 4 hours. + ws = wb['Ops'] + totals = [c for c in _row(ws, 'Cost') if c.column in + [x.column for x in ws[1] if isinstance(x.value, str) and x.value.startswith('Total')]] + assert [round(c.value, 6) for c in totals] == [10.0, 120.0, 20.0] + + @pytest.mark.slow + def test_an_instant_inside_a_formula_recalculates_to_the_engine(self, tmp_path): + m = _spans() + ws = _recalculated(m, tmp_path)['Ops'] + hours = [c.column for c in ws[2] if isinstance(c.value, str) and c.value.endswith(':00')] + totals = [c.column for c in ws[1] if isinstance(c.value, str) and c.value.startswith('Total')] + assert len(hours) == 30 and len(totals) == 3 + span, twice = _row(ws, 'Span'), _row(ws, 'Twice start') + assert [span[c - 1].value for c in hours] == pytest.approx([1.0] * 30) + assert [span[c - 1].value for c in totals] == pytest.approx([2.0, 24.0, 4.0]) + want = [round(_to_serial(v), 6) for v in m.ops.twice.value] + assert [round(twice[c - 1].value, 6) for c in hours] == pytest.approx(want) + + +class TestTheWidthLaw: + def test_a_window_wider_than_a_sheet_is_refused_where_it_is_declared(self): + with pytest.raises(SheetWidthError, match="A window of 20000 hours needs 20000 columns.*" + "a sheet holds 16384; hours fit about 22 months. " + "View it by day"): + mo.Model('w', default_grain='hour', default_start='2026-01-01T00', default_periods=20000) + m = mo.Model('w', default_grain='hour', default_start='2026-01-01T00', default_periods=2) + with pytest.raises(SheetWidthError, match="20000 hours"): + m.default_periods = 20000 + + def test_totals_count_toward_the_width(self): + m = mo.Model('big', default_grain='hour', default_start='2026-01-01T00', default_periods=16100, + default_excel_view=mo.ExcelView(timeline={'totals': ['day']})) + with m: + m.v = mo.Variable([1.0] * 16100, regrain=mo.up('sum')) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + with pytest.raises(SheetWidthError, match="16100 hours needs 16771 columns with its totals"): + LayoutEngine([m]).compute_addresses() + fits = mo.Model('fits', default_grain='hour', default_start='2026-01-01T00', default_periods=15000) + with fits: + fits.v = mo.Variable([1.0] * 15000, regrain=mo.up('sum')) + LayoutEngine([fits]).compute_addresses() + + +def test_growth_by_the_hour_a_year_back(): + from modeleon.compile.excel.writer import _growth_by_calendar + from modeleon.core.time import DefaultWindow + n = 8761 + child = mo.Variable([float(i + 1) for i in range(n)]) + out = _growth_by_calendar(child, DefaultWindow('2025-01-01T00', 'hour', n), 'hour', n, child.value) + # 2026-01-01T00 (index 8760) grows on the same hour a year back, + # 2025-01-01T00 (index 0). + assert out.value[8760] == 8761.0 / 1.0 + assert all(v is None for v in out.value[:8760]) diff --git a/tests/test_indexed_by.py b/tests/test_indexed_by.py index 2f2e9d8..cd0d1d2 100644 --- a/tests/test_indexed_by.py +++ b/tests/test_indexed_by.py @@ -15,6 +15,8 @@ from __future__ import annotations +import operator + import pytest import modeleon as mo @@ -548,3 +550,248 @@ def test_rank1_untouched(self): _reject_rank2_emission( mo.Variable([1.0, 2.0], indexed_by=[months]) ) # no raise + + +class TestKeysAlong: + """``keys_along`` is the one predicate for "this line runs along keys, + not periods": a keyed list, or a line on one finite non-time axis. + The layout's ``line_keys`` is the same answer as text.""" + + def test_keyed_list_gives_its_keys_as_given(self): + from modeleon.core.shape import keys_along + assert keys_along(mo.Variable({'Bear': 1, 'Base': 2})) == ['Bear', 'Base'] + byyear = mo.Variable([1.0, 2.0, 3.0], keys=[2026, 2027, 2028]) + assert keys_along(byyear) == [2026, 2027, 2028] + + def test_one_finite_axis_gives_its_labels(self): + from modeleon.core.shape import keys_along + scen = mo.Variable(['Bear', 'Bull']) + assert keys_along(mo.Variable([1, 2], indexed_by=[scen])) == ['Bear', 'Bull'] + + def test_none_for_a_series_a_scalar_a_time_axis_rank2_tracks_and_a_keyed_cell(self): + from modeleon.core.shape import keys_along + assert keys_along(mo.Variable([1, 2, 3])) is None + assert keys_along(mo.Variable(7)) is None + t = mo.Time('2026-01', periods=3) + assert keys_along(mo.Variable([1, 2, 3], indexed_by=[t])) is None + a = mo.Variable(['x', 'y']) + b = mo.Variable(['p', 'q']) + assert keys_along(mo.Variable([1, 2, 3, 4], indexed_by=[a, b])) is None + mt = mo.Model('t', default_grain='month', default_start='2026-01', default_periods=3, + tracks=mo.Tracks('plan', 'actual')) + mt.rev = mo.Variable(plan=[1, 2, 3], actual=[1, 2, 3]) + assert keys_along(mt.rev) is None + assert keys_along(mo.Variable({'Bear': 1, 'Base': 2})['Base']) is None + + def test_layout_line_keys_is_keys_along_as_text(self): + from modeleon.compile.excel.layout import line_keys + from modeleon.core.shape import keys_along + scen = mo.Variable(['Bear', 'Bull']) + a = mo.Variable(['x', 'y']) + b = mo.Variable(['p', 'q']) + lines = [ + mo.Variable({'Bear': 1, 'Base': 2}), + mo.Variable([1.0, 2.0, 3.0], keys=[2026, 2027, 2028]), + mo.Variable([1, 2], indexed_by=[scen]), + mo.Variable([1, 2, 3]), + mo.Variable(7), + mo.Variable([1, 2, 3], indexed_by=[mo.Time('2026-01', periods=3)]), + mo.Variable([1, 2, 3, 4], indexed_by=[a, b]), + mo.Variable({'Bear': 1, 'Base': 2})['Base'], + ] + for v in lines: + raw = keys_along(v) + assert line_keys(v) == (None if raw is None else [str(k) for k in raw]) + + +def _keyed_model(): + m = mo.Model('m', default_grain='month', default_start='2026-01', default_periods=3) + m.growth = mo.Variable({'Bear': 0.9, 'Base': 1.0, 'Bull': 1.1}) + m.base = mo.Variable([100, 110, 120]) + return m + + +class TestKeysAgainstPeriods: + """A line that runs along keys against a period series is refused: + zipped cell by cell it pairs Bear with January — numbers that look + right and are wrong. Everything the engine aligned on purpose keeps + broadcasting as before.""" + + def test_keyed_times_series_is_refused_naming_both_lines(self): + m = _keyed_model() + with pytest.raises(ValueError) as e: + m.base * m.growth + msg = str(e.value) + assert "'growth' runs along its keys (Bear, Base, Bull)" in msg + assert "'base' runs along periods (3 values)" in msg + assert 'Bear with the first period, Base with the second' in msg + assert 'Pick one key, growth["Bear"]' in msg + assert "mo.Variable({'Bear': …, 'Base': …, 'Bull': …})" in msg + + def test_axised_times_series_is_refused_floating_and_named(self): + scen = mo.Variable(['Bear', 'Base', 'Bull']) + growth = mo.Variable([0.9, 1.0, 1.1], indexed_by=[scen]) + base = mo.Variable([100, 110, 120]) + with pytest.raises(ValueError) as e: + base * growth + msg = str(e.value) + assert 'runs along an axis (Bear, Base, Bull)' in msg + assert 'runs along periods (3 values)' in msg + m = mo.Model('m', default_grain='month', default_start='2026-01', default_periods=3) + m.scen = mo.Variable(['Bear', 'Base', 'Bull']) + m.g = mo.Variable([0.9, 1.0, 1.1], indexed_by=[m.scen], display_name='Growth by scenario') + m.base = mo.Variable([100, 110, 120]) + with pytest.raises(ValueError) as e: + m.base * m.g + msg = str(e.value) + assert "'Growth by scenario' runs along the axis 'scen' (Bear, Base, Bull)" in msg + assert 'Pick one cell, g[0]' in msg + assert 'indexed_by=[scen]' in msg + + @pytest.mark.parametrize('op', [ + operator.add, operator.sub, operator.mul, operator.truediv, operator.floordiv, + operator.mod, operator.pow, operator.lt, operator.le, operator.gt, operator.ge, + operator.eq, operator.ne, + ]) + def test_every_operator_and_comparison_refuses(self, op): + m = _keyed_model() + with pytest.raises(ValueError, match='runs along'): + op(m.growth, m.base) + with pytest.raises(ValueError, match='runs along'): + op(m.base, m.growth) + + def test_unequal_lengths_get_this_message_not_the_length_error(self): + m = _keyed_model() + with pytest.raises(ValueError) as e: + m.growth * mo.Variable([1, 2, 3, 4]) + assert 'runs along periods (4 values)' in str(e.value) + assert 'different lengths' not in str(e.value) + + def test_a_located_series_and_time_are_periods_too(self): + m = _keyed_model() + m.early = mo.Variable([1.0, 2.0, 3.0], start='2026-01', grain='month') + with pytest.raises(ValueError, match="'early' runs along periods \\(3 values\\)"): + m.growth * m.early + with m: + with pytest.raises(ValueError, match='the line mo.time.days runs along periods') as e: + m.growth * mo.time.days + # a time property cannot be given keys — the switch is offered + assert 'switch with mo.CHOOSE(pick, growth)' in str(e.value) + + def test_operator_results_and_slices_still_run_along_keys(self): + m = _keyed_model() + with pytest.raises(ValueError, match=r'the line growth \* 2 runs along its keys'): + (m.growth * 2) * m.base + with pytest.raises(ValueError, match=r'runs along its keys \(Bear, Base\)'): + m.growth[0:2] * m.base + + def test_year_keys_and_a_single_key_count(self): + m = _keyed_model() + m.byyear = mo.Variable([1.0, 2.0, 3.0], keys=[2026, 2027, 2028]) + with pytest.raises(ValueError) as e: + m.byyear * m.base + msg = str(e.value) + assert 'runs along its keys (2026, 2027, 2028)' in msg + assert 'Pick one cell, byyear[0]' in msg + assert 'mo.Variable({2026: …, 2027: …, 2028: …})' in msg + with pytest.raises(ValueError, match=r'runs along its keys \(Only\)'): + mo.Variable({'Only': 5.0}) * mo.Variable([1, 2, 3]) + + def test_scalars_and_one_value_lists_still_broadcast(self): + m = _keyed_model() + for r in (m.growth * 2, 2 * m.growth, m.growth * mo.Variable(2), + m.growth * mo.Variable([2])): + assert r._value == pytest.approx([1.8, 2.0, 2.2]) + assert r._keys == ['Bear', 'Base', 'Bull'] + + def test_a_raw_list_stays_positional(self): + m = _keyed_model() + r = m.growth * [1, 2, 3] + assert r._value == pytest.approx([0.9, 2.0, 3.3]) + assert r._keys == ['Bear', 'Base', 'Bull'] + r2 = m.growth * m.base.value + assert r2._value == pytest.approx([90.0, 110.0, 132.0]) + assert r2._keys == ['Bear', 'Base', 'Bull'] + + def test_keyed_times_keyed_keeps_its_rules(self): + m = _keyed_model() + r = m.growth * mo.Variable({'Bull': 2, 'Base': 3, 'Bear': 4}) + assert r._value == pytest.approx([3.6, 3.0, 2.2]) + assert r._keys == ['Bear', 'Base', 'Bull'] + with pytest.raises(ValueError, match='different key sets'): + m.growth * mo.Variable({'X': 1, 'Base': 2, 'Bear': 3}) + + def test_a_keyed_cell_is_a_scalar_and_broadcasts(self): + m = _keyed_model() + r = m.growth['Base'] * m.base + assert r._value == [100.0, 110.0, 120.0] + assert r._keys is None + + def test_axes_and_the_time_axis_are_unchanged(self): + t = mo.Time('2026-01', periods=3) + rev = mo.Variable([100, 110, 120], indexed_by=[t]) + scen = mo.Variable(['Bear', 'Base', 'Bull']) + growth = mo.Variable([0.9, 1.0, 1.1], indexed_by=[scen]) + r = mo.Variable([100, 110, 120]) * rev + assert r._value == [10000, 12100, 14400] + assert r._indexed_by == (t,) + cross = rev * growth + assert cross._indexed_by == (t, scen) + assert len(cross._value) == 9 + same = growth * mo.Variable([1, 2, 3], indexed_by=[scen]) + assert same._value == pytest.approx([0.9, 2.0, 3.3]) + assert same._indexed_by == (scen,) + + def test_series_times_series_stays_positional(self): + m = _keyed_model() + assert (m.base * mo.Variable([1, 2, 3]))._value == [100, 220, 360] + with m: + days = (m.base * mo.time.days)._value + assert days == [3100, 3080, 3720] + + def test_a_tracked_series_is_periods_too(self): + mt = mo.Model('t', default_grain='month', default_start='2026-01', default_periods=3, + tracks=mo.Tracks('plan', 'actual')) + mt.rev = mo.Variable(plan=[1, 2, 3], actual=[1, 2, 3]) + mt.g = mo.Variable({'Bear': 0.9, 'Base': 1.0, 'Bull': 1.1}) + for a, b in ((mt.rev, mt.g), (mt.g, mt.rev)): + with pytest.raises(ValueError, match=r"'rev' runs along periods \(3 values\)"): + a * b + with pytest.raises(ValueError, match='runs along periods'): + mt.rev > mt.g + + def test_a_series_on_the_time_axis_meets_a_keyed_list_and_is_refused(self): + t = mo.Time('2026-01', periods=3) + rev = mo.Variable([100, 110, 120], indexed_by=[t]) + g = mo.Variable({'Bear': 0.9, 'Base': 1.0, 'Bull': 1.1}) + with pytest.raises(ValueError, match='runs along periods'): + g * rev + + def test_a_located_one_value_series_is_periods(self): + m = mo.Model('m', default_grain='month', default_start='2026-01', default_periods=6) + m.growth = mo.Variable({'Bear': 0.9, 'Base': 1.0, 'Bull': 1.1}) + m.fee = mo.Variable([100.0], start='2026-01', grain='month', extend=mo.zero()) + with pytest.raises(ValueError, match=r"'fee' runs along periods \(1 value\)"): + m.fee * m.growth + # a bare one-value list is still a single value + assert (m.growth * mo.Variable([2.0]))._value == pytest.approx([1.8, 2.0, 2.2]) + + def test_keyed_list_against_an_axis_of_other_labels_is_refused(self): + regions = mo.Variable(['North', 'South', 'East']) + by_region = mo.Variable([10, 20, 30], indexed_by=[regions]) + g = mo.Variable({'Bear': 0.9, 'Base': 1.0, 'Bull': 1.1}) + with pytest.raises(ValueError, match='labels would not match'): + g * by_region + scen = mo.Variable(['Bear', 'Base', 'Bull']) + same = mo.Variable([1, 2, 3], indexed_by=[scen]) + assert (g * same)._value == pytest.approx([0.9, 2.0, 3.3]) + with pytest.raises(ValueError, match='labels would not match'): + mo.Variable({'Bull': 1.1, 'Base': 1.0, 'Bear': 0.9}) * same + + def test_a_computed_series_side_gets_the_switch_as_the_fix(self): + m = _keyed_model() + m.doubled = m.base * 2 + with pytest.raises(ValueError) as e: + m.growth * m.doubled + assert 'switch with mo.CHOOSE(pick, growth)' in str(e.value) + assert 'mo.Variable({' not in str(e.value) diff --git a/tests/test_lag.py b/tests/test_lag.py index d008f74..5aa328a 100644 --- a/tests/test_lag.py +++ b/tests/test_lag.py @@ -171,6 +171,22 @@ def test_lag_of_an_unnamed_running_total_carries_its_values(self, tmp_path): def _row(cells, label): return next(coord for coord, v in cells.items() if v == label)[1:] + def test_a_shift_fill_emits_a_cell_reference(self, tmp_path): + # As mo.lag keeps it: the head cell references the opening cell, + # never a frozen copy of its number. + model = mo.MultiVariable("m") + model.s = mo.MultiVariable("S", excel_props={"tab": True}) + with model.s as s: + s.opening = mo.Variable(500.0, display_name="Opening") + s.closing = mo.Variable([90.0, 80.0], display_name="Closing") + s.prior = s.closing.shift(1, fill_value=s.opening).set_display_name("Prior") + path = tmp_path / "shift_fill.xlsx" + model.to_excel(path) + cells = self._formulas(path, "S") + opening_row = next(coord for coord, v in cells.items() if v == "Opening")[1:] + prior_row = next(coord for coord, v in cells.items() if v == "Prior")[1:] + assert cells[f"B{prior_row}"] == f"=B{opening_row}" + def test_variable_fill_emits_cell_reference(self, tmp_path): model = mo.MultiVariable("m") model.s = mo.MultiVariable("S", excel_props={"tab": True}) diff --git a/tests/test_lines_on_other_windows.py b/tests/test_lines_on_other_windows.py new file mode 100644 index 0000000..4e742c8 --- /dev/null +++ b/tests/test_lines_on_other_windows.py @@ -0,0 +1,1752 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A line on another time window is read by DATE in every function. + +``a + b`` aligns two lines on other windows by date: each is continued +by its own ``extend=`` rule — ``mo.zero()`` for a flow, ``mo.hold()`` +for a rate — a line without one is refused, two grains are refused, and +the result runs on the combined window. Every function that combines two +or more lines one period at a time reads them the same way — ``IF``, +``AND`` / ``OR``, ``CHOOSE``, ``MOD``, the comparisons, ``CONCAT``, the +dates, the lookups, the statistics, a recurrence — and the workbook reads +such a line by date too: a formula on the model's window reading a line +on a sub-container's window inside it takes the cell of the same date, +and for a period the line has no column for, its rule (``0`` for a flow, +the first or last cell for a rate held). Where Python read a line by +position — a formula over ONE other window, which no second window +meets — the workbook reads it by position too, so the two always agree. +An unnamed formula (``rev * 1``, ``fee > 15``) runs on the window of the +lines it reads: it meets other windows by date and, holding no rule, +must cover the combined window — one that does not is refused, naming +the line to make of it. A one-value line on a window of more periods is +a constant, read in every period; a line along keys runs on no window; +a plain line beside a tracked one is read on the tracked line's window; +windows are ordered by date, never as text. A line of another length +with no window at all is refused, as ``a + b`` refuses it, while a list +typed in keeps its positional reading (its last value standing in, as +the cell writes it). A loop container reads such lines the same way: +each period of the loop reads the line's value of its date, and the +workbook reads the cell of that date. A formula whose operands met by +date on a window of its own, set on a sheet of another window, is +refused by the export as the bare sum is. + +Lines along keys combine key by key, as ``a + b`` combines them: the +same keys in another order are read in the first line's order (the +workbook names each key's cell), other keys and a period series are +refused. A list of lines given to ``SUM`` (and its kin) as one argument +is one number over scalar lines and refused over series: per period is +``mo.SUM(a, b)``, the total ``mo.SUM(a) + mo.SUM(b)``. +""" + +import shutil +import subprocess +from datetime import date, datetime +from pathlib import Path + +import pytest +from openpyxl import load_workbook + +import modeleon as mo + +pytestmark = pytest.mark.filterwarnings("ignore::UserWarning") + + +def _rows(ws): + out = {} + for row in ws.iter_rows(): + label = row[0].value + if label: + out[str(label)] = [c.value for c in row[1:]] + return out + + +def _lines(): + """A model from January (X, four months) with sub-containers on + windows inside its own: T from March (two months: flows continued by + zero, rates and dates held, one line without a rule), U from January + ending early, O of one period (March: a flow, a rate held, one line + without a rule), K on the model's window (a block to look up in), I + of scalars.""" + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.x = mo.MultiVariable("X") + m.x.rev = mo.Variable([1.0, 2.0, 3.0, 4.0], display_name="rev") + m.x.flag = mo.Variable([0.0, 1.0, 0.0, 1.0], display_name="flag") + m.x.idx = mo.Variable([1.0, 2.0, 1.0, 2.0], display_name="idx") + m.x.big = mo.Variable([7.0, 9.0, 11.0, 13.0], display_name="big") + m.x.tx = mo.Variable(["a", "b", "c", "d"], display_name="tx") + m.x.dt = mo.Variable([date(2026, 1, 31), date(2026, 2, 28), date(2026, 3, 31), + date(2026, 4, 30)], display_name="dt") + m.x.lk = mo.Variable([1.0, 10.0, 1.0, 10.0], display_name="lk") + m.t = mo.MultiVariable("T", default_grain="month", default_start="2026-03", + default_periods=2) + m.t.fee = mo.Variable([10.0, 20.0], display_name="fee", extend=mo.zero()) + m.t.rate = mo.Variable([0.5, 0.25], display_name="rate", extend=mo.hold()) + m.t.nofill = mo.Variable([7.0, 8.0], display_name="nofill") + m.t.td = mo.Variable([date(2026, 6, 30), date(2026, 7, 31)], display_name="td", + extend=mo.hold()) + m.t.tw = mo.Variable(["p", "q"], display_name="tw", extend=mo.hold()) + m.t.kk = mo.Variable([1.0, 2.0], display_name="kk", extend=mo.hold()) + m.t.rw = mo.Variable([2.0, 1.0], display_name="rw", extend=mo.hold()) + m.t.cc = mo.Variable([3.0, 4.0], display_name="cc", extend=mo.hold()) + m.u = mo.MultiVariable("U", default_grain="month", default_start="2026-01", + default_periods=2) + m.u.early = mo.Variable([100.0, 200.0], display_name="early", extend=mo.zero()) + m.u.cap = mo.Variable([9.0, 8.0], display_name="cap", extend=mo.hold()) + m.k = mo.MultiVariable("K") + m.k.yrs = mo.Variable([1.0, 2.0, 3.0, 4.0], display_name="yrs") + m.k.vals = mo.Variable([10.0, 20.0, 30.0, 40.0], display_name="vals") + m.k.dec = mo.Variable([1.23456, 2.34567, 3.45678, 4.56789], display_name="dec") + m.i = mo.MultiVariable("I") + m.i.k1 = mo.Variable(1.0, display_name="k1") + m.i.k2 = mo.Variable(3.0, display_name="k2") + m.i.v1 = mo.Variable(10.0, display_name="v1") + m.i.v2 = mo.Variable(30.0, display_name="v2") + m.i.hol = mo.Variable(date(2026, 5, 1), display_name="hol") + m.o = mo.MultiVariable("O", default_grain="month", default_start="2026-03", + default_periods=1) + m.o.once = mo.Variable([50.0], display_name="once", extend=mo.zero()) + m.o.level = mo.Variable([2.0], display_name="level", extend=mo.hold()) + m.o.bare = mo.Variable([7.0], display_name="bare") + return m + + +#: Each call on X over lines of T, U and the model's own window, with the +#: values read by date: fee (zero) [0, 0, 10, 20], rate (hold) [0.5, 0.5, +#: 0.5, 0.25], td (hold) [Jun 30, Jun 30, Jun 30, Jul 31], tw (hold) [p, p, +#: p, q], kk [1, 1, 1, 2], rw [2, 2, 2, 1], cc [3, 3, 3, 4], early (zero) +#: [100, 200, 0, 0], cap (hold) [9, 8, 8, 8]. +_CALLS = { + "if_fee": (lambda m: mo.IF(m.x.rev > 2, m.x.rev, m.t.fee), [0.0, 0.0, 3.0, 4.0]), + "if_rate": (lambda m: mo.IF(m.x.rev > 2, m.t.rate, m.u.cap), [9.0, 8.0, 0.5, 0.25]), + "if_flag": (lambda m: mo.IF(m.x.flag, m.t.fee, m.u.early), [100.0, 0.0, 0.0, 20.0]), + "and_": (lambda m: mo.AND(m.x.rev, m.t.fee), [False, False, True, True]), + "or_": (lambda m: mo.OR(m.x.flag, m.u.early), [True, True, False, True]), + "ch_series": (lambda m: mo.CHOOSE(m.x.idx, m.t.fee, m.u.cap), [0.0, 8.0, 10.0, 8.0]), + "ch_scalar": (lambda m: mo.CHOOSE(2, m.x.rev, m.t.fee), [0.0, 0.0, 10.0, 20.0]), + "md": (lambda m: mo.MOD(m.x.big, m.u.cap), [7.0, 1.0, 3.0, 5.0]), + "gt": (lambda m: m.x.rev > m.t.fee, [True, True, False, False]), + "ge": (lambda m: m.u.cap >= m.x.big, [True, False, False, False]), + "cc_": (lambda m: mo.CONCAT(m.x.tx, m.t.tw), ["ap", "bp", "cp", "dq"]), + "dd": (lambda m: mo.DATE(2026, m.x.rev, m.t.fee), + [date(2025, 12, 31), date(2026, 1, 31), date(2026, 3, 10), date(2026, 4, 20)]), + "d360": (lambda m: mo.DAYS360(m.x.dt, m.t.td), [150, 120, 90, 90]), + "yf": (lambda m: mo.YEARFRAC(m.x.dt, m.t.td, 3), + [150 / 365, 122 / 365, 91 / 365, 92 / 365]), + "dy": (lambda m: mo.DAYS(m.t.td, m.x.dt), [150, 122, 91, 92]), + "nwd": (lambda m: mo.NETWORKDAYS(m.x.dt, m.t.td, m.i.hol), None), + "xl": (lambda m: mo.XLOOKUP(m.x.rev, [m.i.k1, m.i.k2], [m.i.v1, m.i.v2], m.t.fee), + [10.0, 0.0, 30.0, 20.0]), + "hl": (lambda m: mo.HLOOKUP(m.x.rev, [m.k.yrs, m.k.vals], m.t.rw, False), + [10.0, 20.0, 30.0, 4.0]), + "ix": (lambda m: mo.INDEX([m.k.yrs, m.k.vals], m.x.idx, m.t.cc), [3.0, 30.0, 3.0, 40.0]), + "vl": (lambda m: mo.VLOOKUP(m.x.lk, [m.k.yrs, m.k.vals], m.t.cc, False), + [3.0, 30.0, 3.0, 40.0]), + "med": (lambda m: mo.MEDIAN(m.x.rev, m.t.fee, m.u.cap), [1.0, 2.0, 8.0, 8.0]), + "sdp": (lambda m: mo.STDEV_P(m.x.rev, m.u.early), [49.5, 99.0, 1.5, 2.0]), + "mdl": (lambda m: mo.MEDIAN([m.x.rev, m.t.fee, m.u.cap]), [1.0, 2.0, 8.0, 8.0]), + "pct": (lambda m: mo.PERCENTILE_INC([m.x.rev, m.t.fee, m.u.cap], 0.5), + [1.0, 2.0, 8.0, 8.0]), + "lg": (lambda m: mo.LARGE([m.x.rev, m.x.idx], m.t.kk), [1.0, 2.0, 3.0, 2.0]), + "rk": (lambda m: mo.RANK_EQ(m.t.fee, [m.x.rev, m.t.fee]), [2, 2, 1, 1]), + "rc": (lambda m: mo.recurrence(0.0, "{prev} + {a} + {b}", a=m.x.rev, b=m.t.fee), + [0.0, 2.0, 15.0, 39.0]), + "ie": (lambda m: mo.IFERROR(m.x.rev / m.t.fee, 0), [0.0, 0.0, 0.3, 0.2]), + # O's one period, March: once (zero) [0, 0, 50, 0], level (hold) [2, 2, 2, 2]. + "o_if": (lambda m: mo.IF(m.x.rev > 3, m.x.rev, m.o.once), [0.0, 0.0, 50.0, 4.0]), + "o_if2": (lambda m: mo.IF(m.x.flag, m.o.level, m.o.once), [0.0, 2.0, 50.0, 2.0]), + "o_med": (lambda m: mo.MEDIAN(m.x.rev, m.o.once, m.o.level), [1.0, 2.0, 3.0, 2.0]), + "o_cat": (lambda m: mo.CONCAT(m.x.tx, m.o.once), ["a0", "b0", "c50", "d0"]), + "o_rnd": (lambda m: mo.ROUND(m.k.dec, m.o.level), [1.23, 2.35, 3.46, 4.57]), + "o_ed": (lambda m: mo.EDATE(m.x.dt, m.o.level), + [date(2026, 3, 31), date(2026, 4, 28), date(2026, 5, 31), date(2026, 6, 30)]), + "o_pmt": (lambda m: mo.PMT(0.05, m.o.level, m.k.vals), + [mo.PMT(0.05, 2, v).value for v in (10.0, 20.0, 30.0, 40.0)]), + # A formula over X's lines runs on X's window, meeting others by date. + "w_add": (lambda m: (m.x.rev * 1) + m.t.fee, [1.0, 2.0, 13.0, 24.0]), + "w_if": (lambda m: mo.IF(m.x.rev * 1 > 2, m.x.rev * 1, m.t.fee), [0.0, 0.0, 3.0, 4.0]), + "w_once": (lambda m: m.x.rev * 1 + m.o.once, [1.0, 2.0, 53.0, 4.0]), + "w_max": (lambda m: mo.MAX(m.x.rev * 1, m.t.fee, m.u.cap), [9.0, 8.0, 10.0, 20.0]), + "w_lag": (lambda m: mo.lag(m.x.rev, 1) + m.t.fee, [0.0, 1.0, 12.0, 23.0]), + "w_cmp": (lambda m: mo.IF(m.x.rev > 2, m.t.fee, m.u.cap), [9.0, 8.0, 10.0, 20.0]), +} + + +def _model(): + """``_lines()`` with every call of ``_CALLS`` a row of X.""" + m = _lines() + with m: + for name, (make, _) in _CALLS.items(): + setattr(m.x, name, make(m).set_display_name(name)) + return m + + +def _networkdays(): + # The working days from each period's date to the held one, the + # holiday on 1 May, each counted on its own. + pairs = [(date(2026, 1, 31), date(2026, 6, 30)), (date(2026, 2, 28), date(2026, 6, 30)), + (date(2026, 3, 31), date(2026, 6, 30)), (date(2026, 4, 30), date(2026, 7, 31))] + return [mo.NETWORKDAYS(s, e, date(2026, 5, 1)).value for s, e in pairs] + + +# ══════════════════════════════════════════════════════════════════════ +# Python reads by date +# ══════════════════════════════════════════════════════════════════════ + + +class TestPythonReadsByDate: + @pytest.mark.parametrize("name", list(_CALLS)) + def test_each_call_reads_each_line_on_the_date_s_cell(self, name): + m = _lines() + make, expected = _CALLS[name] + got = make(m) + if expected is None: + expected = _networkdays() + if isinstance(expected[0], float): + assert got.value == pytest.approx(expected) + else: + assert got.value == expected + if name == "ie": + # The windows met in ``rev / fee``; IFERROR over that one + # series and a number places nothing, as ``(rev / fee) * 2``. + assert got.time is None + else: + # On the combined window, as an operator puts its result there. + assert got.time is not None and got.time.start == "2026-01", name + assert len(got.value) == 4 + + def test_the_call_reads_what_the_operator_reads(self): + m = _lines() + rev, fee, rate = m.x.rev, m.t.fee, m.t.rate + fee_on_x = (rev + fee - rev).value # fee read on X's dates + assert fee_on_x == [0.0, 0.0, 10.0, 20.0] + assert (rev > fee).value == [a > b for a, b in zip(rev.value, fee_on_x)] + assert (rev > fee).time == (rev + fee).time + assert mo.MOD(rev, rate).value == (rev % rate).value + assert mo.CONCAT(rev, fee).value == ["10", "20", "310", "420"] + + def test_three_windows_each_continued_by_its_own_rule(self): + m = _lines() + got = mo.IF(m.x.flag, m.t.fee, m.u.cap) + assert got.value == [9.0, 0.0, 8.0, 20.0] + assert mo.AND(m.x.rev, m.t.fee, m.u.early).value == [False, False, False, False] + assert mo.MEDIAN(m.x.rev, m.t.fee, m.u.early).value == [1.0, 2.0, 3.0, 4.0] + + def test_equal_windows_are_read_by_position_as_before(self): + m = _lines() + x = m.x + assert mo.IF(x.flag, x.rev, x.big).value == [7.0, 2.0, 11.0, 4.0] + assert mo.AND(x.rev, x.flag).value == [False, True, False, True] + assert mo.CHOOSE(x.idx, x.rev, x.big).value == [1.0, 9.0, 3.0, 13.0] + assert (x.rev > x.idx).value == [False, False, True, True] + assert mo.MOD(x.big, x.rev).value == [0.0, 1.0, 2.0, 1.0] + # Plain lists keep their positional reading. + assert mo.IF([1, 0, 1], [1.0, 2.0, 3.0], [9.0, 9.0, 9.0]).value == [1.0, 9.0, 3.0] + assert mo.AND([1, 1, 0], [1]).value == [True, True, False] + assert mo.AND([1, 1, 0], [1, 0]).value == [True, False, False] + # Nothing aligned, nothing placed: the line takes its container's + # window, as ``x.rev + x.big`` does. + assert mo.IF(x.flag, x.rev, x.big).time is None + assert (x.rev + x.big).time is None + + def test_a_formula_runs_on_the_window_of_the_lines_it_reads(self): + # ``revz > 2`` holds X's four months: beside a line from November + # the combined window runs six months, and a formula has no rule + # to continue it — refused, naming it, where a positional pairing + # would have read X's January in November. + m = _lines() + with m: + m.t2 = mo.MultiVariable("T2", default_grain="month", default_start="2025-11", + default_periods=2) + m.t2.old = mo.Variable([5.0, 6.0], display_name="old", extend=mo.zero()) + m.x.revz = mo.Variable([1.0, 2.0, 3.0, 4.0], display_name="revz", + extend=mo.zero()) + for formula, call in ( + ("revz > 2", lambda: mo.IF(m.x.revz > 2, m.x.revz, m.t2.old)), + ("revz > 2", lambda: mo.AND(m.x.revz > 2, m.x.revz, m.t2.old)), + ("idx \\* 1", lambda: mo.CHOOSE(m.x.idx * 1, m.x.revz, m.t2.old)), + ("revz > 2", lambda: mo.CONCAT(m.x.revz > 2, m.x.revz, m.t2.old)), + ("big \\* 1", lambda: mo.DATE(m.x.big * 1, m.x.revz, m.t2.old)), + ("revz > 2", lambda: mo.MEDIAN(m.x.revz > 2, m.x.revz, m.t2.old)), + ("revz \\* 1", lambda: (m.x.revz * 1) + m.t2.old)): + with pytest.raises(ValueError, match=f"the line {formula} does not cover the " + f"combined window") as refused: + call() + assert "the periods of the lines it reads" in str(refused.value) + assert "mo.Variable(" in str(refused.value) + # The line itself, continued by its rule, is read by date. + assert (m.x.revz + m.t2.old).value == [5.0, 6.0, 1.0, 2.0, 3.0, 4.0] + + +_NO_RULE = { + "IF": lambda m: mo.IF(m.x.rev > 2, m.x.rev, m.t.nofill), + "AND": lambda m: mo.AND(m.x.rev, m.t.nofill), + "OR": lambda m: mo.OR(m.t.nofill, m.x.rev), + "CHOOSE": lambda m: mo.CHOOSE(m.x.idx, m.t.nofill, m.t.fee), + "CHOOSE_scalar": lambda m: mo.CHOOSE(1, m.t.nofill, m.x.rev), + "MOD": lambda m: mo.MOD(m.x.rev, m.t.nofill), + "compare": lambda m: m.x.rev > m.t.nofill, + "CONCAT": lambda m: mo.CONCAT(m.x.tx, m.t.nofill), + "DATE": lambda m: mo.DATE(2026, m.x.rev, m.t.nofill), + "DAYS": lambda m: mo.DAYS(m.t.nofill, m.x.rev), + "XLOOKUP": lambda m: mo.XLOOKUP(m.x.rev, [m.i.k1, m.i.k2], [m.i.v1, m.i.v2], m.t.nofill), + "INDEX": lambda m: mo.INDEX([m.k.yrs, m.k.vals], m.x.idx, m.t.nofill), + "MEDIAN": lambda m: mo.MEDIAN(m.x.rev, m.t.nofill), + "MEDIAN_list": lambda m: mo.MEDIAN([m.x.rev, m.t.nofill]), + "LARGE": lambda m: mo.LARGE([m.x.rev, m.x.idx], m.t.nofill), + "recurrence": lambda m: mo.recurrence(0.0, "{prev} + {a}", a=m.t.nofill, b=m.x.rev), +} + + +class TestALineWithoutARuleIsRefused: + @pytest.mark.parametrize("name", list(_NO_RULE)) + def test_as_a_plus_b_refuses_it(self, name): + m = _lines() + with pytest.raises(ValueError, match="'nofill' does not cover the combined window"): + m.x.rev + m.t.nofill + with pytest.raises(ValueError, match="'nofill' does not cover the combined window") \ + as refused: + _NO_RULE[name](m) + assert "extend=mo.zero()" in str(refused.value) + assert "extend=mo.hold()" in str(refused.value) + + def test_a_formula_over_one_window_meets_another_by_date(self): + # ``rev * 1`` runs on X's window, the one of the line it reads: + # beside fee it is read by date, operators and functions alike, + # where a positional reading paired January with March. + m = _lines() + wide = m.x.rev * 1 + assert (wide + m.t.fee).value == [1.0, 2.0, 13.0, 24.0] + for call, expected in ( + (lambda: mo.IF(wide > 2, wide, m.t.fee), [0.0, 0.0, 3.0, 4.0]), + (lambda: mo.AND(wide, m.t.fee), [False, False, True, True]), + (lambda: mo.OR(m.t.fee, wide), [True, True, True, True]), + (lambda: mo.CHOOSE(1, wide, m.t.fee), [1.0, 2.0, 3.0, 4.0]), + (lambda: mo.CONCAT(wide, m.t.tw), ["1p", "2p", "3p", "4q"]), + (lambda: mo.DATE(2026, wide, m.t.fee), + [date(2025, 12, 31), date(2026, 1, 31), date(2026, 3, 10), + date(2026, 4, 20)]), + (lambda: mo.DAYS360(mo.EDATE(m.x.dt, 0), m.t.td), [150, 120, 90, 90]), + (lambda: mo.DAYS(m.t.td, mo.EDATE(m.x.dt, 0)), [150, 122, 91, 92]), + (lambda: mo.recurrence(0.0, "{prev} + {a} + {b}", a=wide, b=m.t.fee), + [0.0, 2.0, 15.0, 39.0]), + (lambda: mo.recurrence(0.0, "{prev} + {b} + {a}", b=m.t.fee, a=wide), + [0.0, 2.0, 15.0, 39.0]), + # A comparison is such a formula: the commonest condition. + (lambda: mo.IF(m.x.rev > 2, m.t.fee, 0), [0.0, 0.0, 10.0, 20.0]), + (lambda: mo.IF(m.x.rev > 2, m.t.fee, m.u.cap), [9.0, 8.0, 10.0, 20.0])): + got = call() + assert got.value == expected + assert got.time.start == "2026-01" + + def test_a_formula_the_combined_window_outruns_is_refused(self): + # Ops runs from July: ``cfads > 0`` holds July's periods, which no + # rule continues back to January — refused, where a positional + # pairing let July's cash decide January. + m = _lines() + with m: + m.ops = mo.MultiVariable("Ops", default_grain="month", default_start="2026-07", + default_periods=4) + m.ops.cfads = mo.Variable([-5.0, -1.0, 2.0, 3.0], display_name="cfads") + m.b = mo.MultiVariable("B", default_grain="month", default_start="2025-11", + default_periods=2) + m.b.bz = mo.Variable([1000.0, 2000.0], display_name="bz", extend=mo.zero()) + m.x.revz = mo.Variable([1.0, 2.0, 3.0, 4.0], display_name="revz", + extend=mo.zero()) + # Every line here says what continues it; the formula cannot. + cf, rev, fee = m.ops.cfads, m.x.revz, m.t.fee + for formula, call in ( + ("cfads > 0", lambda: mo.IF(cf > 0, rev, fee)), + ("cfads \\* 1", lambda: mo.MAX(cf * 1, rev, fee)), + ("cfads \\* 1", lambda: mo.CHOOSE(1, cf * 1, rev)), + ("cfads \\* 1", lambda: mo.SUM(cf * 1, rev, fee)), + ("cfads \\* 1", lambda: mo.MEDIAN(cf * 1, rev, fee)), + ("cfads \\* 1", + lambda: mo.recurrence(0.0, "{prev} + {a} + {b}", a=cf * 1, b=fee)), + ("cfads / 3", lambda: mo.ROUND(cf / 3, rev)), + ("cfads \\* 1", lambda: mo.PMT(0.05, rev, cf * 1)), + ("cfads \\* 1", lambda: (cf * 1) + rev), + ("cfads > 0", lambda: (cf > 0) + rev), + # The combined window may start before the sheet's. + ("revz > 3", lambda: mo.IF(rev > 3, m.b.bz, fee))): + with pytest.raises(ValueError, match=f"the line {formula} does not cover"): + call() + with pytest.raises(ValueError, match="'cfads' does not cover"): + cf + rev + + def test_the_formula_made_a_line_with_a_rule_is_read_by_date(self): + m = _lines() + with pytest.raises(ValueError, match="m.t. = mo.Variable") as refused: + mo.IF(m.t.fee > 15, m.x.rev, 0) + assert "the container 'fee' runs on" in str(refused.value) + with m: + m.t.big = mo.Variable(m.t.fee > 15, extend=mo.zero()) + assert mo.IF(m.t.big, m.x.rev, 0).value == [0, 0, 0, 4.0] + assert mo.OR(m.x.rev > 3, m.t.big).value == [False, False, False, True] + + def test_a_line_with_no_window_of_another_length_is_refused_by_name(self): + m = _lines() + pair = mo.Variable([1.0, 0.0], display_name="pair") + for call in (lambda: mo.AND(m.x.rev, pair), lambda: mo.IF(m.x.rev > 2, pair, 0), + lambda: mo.CONCAT(m.x.tx, pair)): + with pytest.raises(ValueError, match="'pair' cannot be paired with the call's " + "other series period by period " + r"\(different lengths: 2 vs 4\)"): + call() + + def test_one_condition_chooses_a_branch_whole(self): + # Nothing is paired: the branch comes back as it is. + a, b = mo.Variable([1.0, 2.0, 3.0]), mo.Variable([5.0, 6.0]) + assert mo.IF(True, a, b).value == [1.0, 2.0, 3.0] + assert mo.IF(False, a, b).value == [5.0, 6.0] + + def test_a_list_typed_in_keeps_its_positional_reading(self, tmp_path): + # A list typed in has no cells: its last value stands in the + # periods it lacks, as the cell writes it. + m = _lines() + assert mo.AND(m.x.rev, [1, 0]).value == [True, False, False, False] + assert mo.CONCAT(m.x.tx, ["x", "y"]).value == ["ax", "by", "cy", "dy"] + assert mo.DATE(2026, m.x.rev, [10, 20]).value[3] == date(2026, 4, 20) + # IF too, where it raised IndexError past the list's end. + with m: + m.x.ty = mo.IF(m.x.rev > 9, m.x.rev, [7.0, 8.0]).set_display_name("ty") + assert m.x.ty.value == [7.0, 8.0, 8.0, 8.0] + m.to_excel(tmp_path / "typed.xlsx") + assert _rows(load_workbook(tmp_path / "typed.xlsx")["X"])["ty"] == [ + "=IF(B2 > 9, B2, 7.0)", "=IF(C2 > 9, C2, 8.0)", "=IF(D2 > 9, D2, 8.0)", + "=IF(E2 > 9, E2, 8.0)"] + + def test_a_one_period_line_without_a_rule_is_refused(self): + m = _lines() + for call in (lambda: m.x.rev + m.o.bare, + lambda: mo.IF(m.x.rev > 2, m.x.rev, m.o.bare), + lambda: mo.MEDIAN(m.x.rev, m.o.bare), + lambda: mo.ROUND(m.k.dec, m.o.bare)): + with pytest.raises(ValueError, match="'bare' does not cover the combined window"): + call() + + def test_two_grains_are_refused(self): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=3) + with m: + m.x = mo.MultiVariable("X") + m.x.a = mo.Variable([1.0, 2.0, 3.0], display_name="a") + m.q = mo.MultiVariable("Q", default_grain="quarter", default_start="2026-Q1", + default_periods=2) + m.q.b = mo.Variable([1.0, 2.0], display_name="b") + for call in (lambda: mo.IF(m.x.a, m.x.a, m.q.b), lambda: mo.AND(m.x.a, m.q.b), + lambda: mo.MOD(m.x.a, m.q.b), lambda: m.x.a > m.q.b, + lambda: mo.MEDIAN(m.x.a, m.q.b), lambda: mo.CONCAT(m.x.a, m.q.b)): + with pytest.raises(ValueError, match="different grains"): + call() + + +# ══════════════════════════════════════════════════════════════════════ +# What runs on a window, and how windows are ordered +# ══════════════════════════════════════════════════════════════════════ + + +def _windows(): + """``_lines()`` with a constant on X (one value on four months), lines + along keys on X and on T, and a quarterly line.""" + m = _lines() + with m: + m.x.c = mo.Variable([5.0], display_name="c") + m.x.k = mo.Variable({"Bear": 1.0, "Base": 2.0, "Bull": 3.0}, display_name="k") + m.t.k2 = mo.Variable({"Bull": 30.0, "Base": 20.0, "Bear": 10.0}, display_name="k2") + m.q = mo.MultiVariable("Q", default_grain="quarter", default_start="2026-Q1", + default_periods=2) + m.q.h = mo.Variable([7.0, 8.0], display_name="h") + return m + + +def _unpadded(): + # September written without its leading zero, a line from October. + m = mo.Model("m", default_grain="month", default_start="2026-9", default_periods=6) + with m: + m.x = mo.MultiVariable("X") + m.x.rev = mo.Variable([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], display_name="rev") + m.t = mo.MultiVariable("T", default_grain="month", default_start="2026-10", + default_periods=2) + m.t.fee = mo.Variable([10.0, 20.0], display_name="fee", extend=mo.zero()) + return m + + +class TestWhatRunsOnAWindow: + def test_a_one_value_line_is_a_constant(self): + # One value on a window of four months is a constant (a bare list + # holds one value or one per period), read in every period — by the operators too, + # which refused it as a one-month line. + m = _windows() + rev, c = m.x.rev, m.x.c + assert (rev + c).value == [6.0, 7.0, 8.0, 9.0] + assert mo.MAX(rev, c).value == [5.0, 5.0, 5.0, 5.0] + assert mo.SUM(rev, c).value == [6.0, 7.0, 8.0, 9.0] + assert mo.IF(rev > 3, c, m.t.fee).value == [0.0, 0.0, 10.0, 5.0] + assert (rev * 1 + c).value == [6.0, 7.0, 8.0, 9.0] + + def test_lines_along_keys_run_on_no_window(self): + # k on X, k2 on T: keys, not periods — combined key by key, where + # T's window refused them as a line of March. + m = _windows() + k, k2 = m.x.k, m.t.k2 + for got, expected in ((mo.MAX(k, k2), [10.0, 20.0, 30.0]), + (k + k2, [11.0, 22.0, 33.0]), + (mo.IF(k > 1, k2, 0), [0, 20.0, 30.0])): + assert got.value == expected + assert got._keys == ["Bear", "Base", "Bull"] + + def test_windows_are_ordered_by_date(self): + # '2026-9' is before '2026-10', though not as text: September is + # kept, not dropped. + m = _unpadded() + rev, fee = m.x.rev, m.t.fee + for got, expected in ((rev + fee, [1.0, 12.0, 23.0, 4.0, 5.0, 6.0]), + (mo.SUM(rev, fee), [1.0, 12.0, 23.0, 4.0, 5.0, 6.0]), + (mo.MAX(rev, fee), [1.0, 10.0, 20.0, 4.0, 5.0, 6.0]), + (mo.IF(rev > 2, fee, rev), [1.0, 2.0, 20.0, 0.0, 0.0, 0.0])): + assert got.value == expected + assert got.time.start == "2026-9" + assert mo.SUM(mo.SUM(rev, fee)).value == 51.0 + + def test_a_formula_reads_lines_of_one_grain(self): + # ``h * 1`` runs on quarters: beside months it is refused as the + # line itself is, where its quarters were added to months. + m = _windows() + for call in (lambda: m.x.rev + m.q.h * 1, lambda: mo.MAX(m.x.rev * 1, m.q.h), + lambda: mo.IF(m.q.h > 7, m.x.rev, 0)): + with pytest.raises(ValueError, match="different grains"): + call() + + +# ══════════════════════════════════════════════════════════════════════ +# The order a model is built in does not move a formula +# ══════════════════════════════════════════════════════════════════════ + + +def _x_and_t(): + """X on the model's window (January, four months); T from March (two + months, a flow continued by zero).""" + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.x = mo.MultiVariable("X") + m.t = mo.MultiVariable("T", default_grain="month", default_start="2026-03", + default_periods=2) + m.t.fee = mo.Variable([10.0, 20.0], display_name="fee", extend=mo.zero()) + return m + + +def _attached_later(): + """Lines built floating — ``a``, ``b``, and over them the formulas + ``d = a * 1`` and ``e = d + b`` — then attached to X, beside T and T4 + from March (four months).""" + a = mo.Variable([1.0, 2.0, 3.0, 4.0], display_name="a") + b = mo.Variable([5.0, 6.0, 7.0, 8.0], display_name="b") + d = a * 1 + e = d + b + m = _x_and_t() + with m: + m.x.a = a + m.x.b = b + m.t4 = mo.MultiVariable("T4", default_grain="month", default_start="2026-03", + default_periods=4) + m.t4.fee = mo.Variable([10.0, 20.0, 30.0, 40.0], display_name="fee4", + extend=mo.zero()) + return m, d, e + + +def _six(): + return [mo.Variable([float(i + j) for j in range(4)], display_name=f"l{i}") + for i in range(6)] + + +def _total(lines): + total = lines[0] + for v in lines[1:]: + total = total + v + return total + + +class TestTheBuildOrder: + def test_a_floating_formula_attached_later_runs_on_its_lines_window(self): + # ``d`` met ``b`` while ``a`` had no window: once ``a`` is on X, + # ``d`` runs on January as a formula built after does — read by + # date beside T, refused beside T4, which it does not cover. + m, d, e = _attached_later() + assert e.value == [6.0, 8.0, 10.0, 12.0] + fresh = (m.x.a * 1) + m.t.fee + assert (d + m.t.fee).value == fresh.value == [1.0, 2.0, 13.0, 24.0] + assert mo.MAX(d, m.t.fee).value == [1.0, 2.0, 10.0, 20.0] + assert (e + m.t.fee).value == [6.0, 8.0, 20.0, 32.0] + for call in (lambda: d + m.t4.fee, lambda: mo.MAX(d, m.t4.fee), + lambda: e + m.t4.fee, lambda: (m.x.a * 1) + m.t4.fee): + with pytest.raises(ValueError, match="does not cover the combined window"): + call() + + def test_a_window_moved_after_a_formula_was_placed(self): + # ``g`` was placed on January through ``rev``; the model moves to + # March before ``g`` meets a line of March. + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.x = mo.MultiVariable("X") + m.x.rev = mo.Variable([1.0, 2.0, 3.0, 4.0], display_name="rev") + m.x.one = mo.Variable([1.0, 1.0, 1.0, 1.0], display_name="one") + g = m.x.rev * 1 + h = g + m.x.one + m.default_start = "2026-03" + m.t = mo.MultiVariable("T", default_grain="month", default_start="2026-03", + default_periods=4) + m.t.fee = mo.Variable([10.0, 20.0, 30.0, 40.0], display_name="fee", + extend=mo.zero()) + expected = [11.0, 22.0, 33.0, 44.0] + assert (g + m.t.fee).value == ((m.x.rev * 1) + m.t.fee).value == expected + assert (m.x.rev + m.t.fee).value == expected + assert (h + m.t.fee).value == [12.0, 23.0, 34.0, 45.0] + + def test_a_chain_answers_the_same_whatever_the_order_it_was_built_in(self): + # A running total over six lines meets T the same whether it was + # built before the lines were attached, after, or a step ahead of + # each attachment. + expected = [15.0, 21.0, 37.0, 53.0] + lines = _six() + before = _total(lines) + m = _x_and_t() + with m: + for i, v in enumerate(lines): + setattr(m.x, f"l{i}", v) + after = _total(lines) + assert (before + m.t.fee).value == (after + m.t.fee).value == expected + m = _x_and_t() + with m: + total = None + for i, v in enumerate(_six()): + total = v if total is None else total + v + setattr(m.x, f"l{i}", v) + assert (total + m.t.fee).value == expected + + def test_the_workbook_reads_a_formula_attached_later_by_date(self, tmp_path): + # ``s``, ``mx`` and ``es`` read ``a`` (B2) and ``b`` (B3) in their + # own column and fee by date — the cells ``fs``, built after the + # attachment, holds. + m, d, e = _attached_later() + with m: + m.x.s = (d + m.t.fee).set_display_name("s") + m.x.mx = mo.MAX(d, m.t.fee).set_display_name("mx") + m.x.es = (e + m.t.fee).set_display_name("es") + m.x.fs = ((m.x.a * 1) + m.t.fee).set_display_name("fs") + path = tmp_path / "book.xlsx" + m.to_excel(path) + x = _rows(load_workbook(path)["X"]) + assert x["s"] == x["fs"] == ["=B2 * 1 + 0", "=C2 * 1 + 0", "=D2 * 1 + T!B2", + "=E2 * 1 + T!C2"] + assert x["mx"] == ["=MAX(B2 * 1, 0)", "=MAX(C2 * 1, 0)", "=MAX(D2 * 1, T!B2)", + "=MAX(E2 * 1, T!C2)"] + assert x["es"] == ["=B2 * 1 + B3 + 0", "=C2 * 1 + C3 + 0", "=D2 * 1 + D3 + T!B2", + "=E2 * 1 + E3 + T!C2"] + + def test_a_copy_of_floating_lines_is_placed_as_a_fresh_formula(self): + # A pickle or a copy of floating lines and a formula over them, + # made after the model is built: once the copies are attached, the + # copied formula answers what a formula built after the attachment + # answers. + import copy + import pickle + for clone in (copy.deepcopy, lambda o: pickle.loads(pickle.dumps(o))): + m = _x_and_t() + with m: + m.t4 = mo.MultiVariable("T4", default_grain="month", + default_start="2026-03", default_periods=4) + m.t4.fee = mo.Variable([10.0, 20.0, 30.0, 40.0], display_name="fee4", + extend=mo.zero()) + a = mo.Variable([1.0, 2.0, 3.0, 4.0], display_name="a") + b = mo.Variable([5.0, 6.0, 7.0, 8.0], display_name="b") + d = a * 1 + assert (d + b).value == [6.0, 8.0, 10.0, 12.0] + a2, b2, d2 = clone((a, b, d)) + m.x.a = a2 + m.x.b = b2 + for call in (lambda: d2 + m.t4.fee, lambda: (m.x.a * 1) + m.t4.fee): + with pytest.raises(ValueError, match="does not cover the combined window"): + call() + + def test_a_refusal_names_the_container_of_the_line_with_a_window(self): + # ``z`` has no window yet; the formula runs on X's window through + # ``a``, so the way out is a line of X. + m, _d, _e = _attached_later() + z = mo.Variable([0.0, 0.0, 0.0, 0.0], display_name="z") + for build in (lambda: z + (m.x.a * 1), lambda: (m.x.a * 1) + z): + with pytest.raises(ValueError, match=r"make it a line of m\.x, the container 'a'"): + build() + m.t4.fee + + +# ══════════════════════════════════════════════════════════════════════ +# A window changed elsewhere does not walk every formula again +# ══════════════════════════════════════════════════════════════════════ + + +def _walks(monkeypatch, build, n): + """How many times the placement walk runs while ``build(n)`` builds.""" + from modeleon.core import variable_ops + count = [0] + walk = variable_ops._scan + + def counted(*args, **kwargs): + count[0] += 1 + return walk(*args, **kwargs) + + monkeypatch.setattr(variable_ops, "_scan", counted) + build(n) + monkeypatch.setattr(variable_ops, "_scan", walk) + return count[0] + + +def _sections(n): + # A running total over sections, each a container declaring the + # model's window, built and filled one after another. + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.assets = mo.MultiVariable("Assets") + total = None + for i in range(n): + s = mo.MultiVariable(f"A{i}", default_grain="month", default_start="2026-01", + default_periods=4) + setattr(m.assets, f"a{i}", s) + s.cf = mo.Variable([float(i + j) for j in range(4)], extend=mo.zero()) + total = s.cf * 1 if total is None else total + s.cf + return total + + +def _models(n): + # A running total over the lines of one model while another small + # model is built at every step (its constructor declares a window). + m = _x_and_t() + with m: + for i in range(n): + setattr(m.x, f"l{i}", mo.Variable([float(i + j) for j in range(4)])) + total = None + for i in range(n): + mo.Model(f"side{i}", default_grain="month", default_start="2026-01", + default_periods=4) + line = getattr(m.x, f"l{i}") + total = line * 1 if total is None else total + line + return total + + +class TestWindowChangesStayLinear: + @pytest.mark.parametrize("build", [_sections, _models]) + def test_the_walks_grow_with_the_lines_not_their_square(self, monkeypatch, build): + small = _walks(monkeypatch, build, 40) + large = _walks(monkeypatch, build, 80) + assert large <= 2.5 * max(small, 1), (small, large) + + +def _outcome(thunk): + try: + result = thunk() + except ValueError as error: + return ("refused", str(error).split(".")[0]) + return (result.value, result.time.start if result.time is not None else None) + + +def _kept_and_walked(thunk): + """The answer of a formula's kept placement, then the answer of the + same formula once every placement is walked again.""" + from modeleon.core import variable_ops + kept = _outcome(thunk) + variable_ops._windows_changed() + return kept, _outcome(thunk) + + +class TestAKeptPlacementFollowsItsLines: + """A formula's kept placement answers what a fresh walk answers after + the lines it reads move — a container detached and set elsewhere, a + window declared or deleted, a row read ahead becoming its line.""" + + def test_a_variance_over_track_slices_after_its_container_moved(self): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4, + tracks=mo.Tracks("plan", "actual")) + with m: + m.x = mo.MultiVariable("X") + m.x.trk = mo.Variable(plan=[10.0, 20.0, 30.0, 40.0], + actual=[11.0, 22.0, 33.0, 44.0], display_name="trk") + m.t = mo.MultiVariable("T", default_grain="month", default_start="2026-03", + default_periods=2) + m.t.fee = mo.Variable([1.0, 2.0], display_name="fee", extend=mo.zero()) + variance = m.x.trk.at(track="actual") - m.x.trk.at(track="plan") + x = m.remove("x") + with m: + m.y = mo.MultiVariable("Y", default_grain="month", default_start="2026-03", + default_periods=4) + m.y.x = x + kept, walked = _kept_and_walked(lambda: variance + m.t.fee) + assert kept == walked == ([2.0, 4.0, 3.0, 4.0], "2026-03") + + def test_a_row_read_ahead_after_its_loop_container_moved(self): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.t = mo.MultiVariable("T", default_grain="month", default_start="2026-03", + default_periods=2) + m.t.fee = mo.Variable([10.0, 20.0], display_name="fee", extend=mo.zero()) + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + y = mo.lag(d.bal, fill=0.0) + mo.Variable(0.0) + d.bal = mo.lag(d.bal, fill=1.0) + 1.0 + d = m.remove("d") + with m: + m.y = mo.MultiVariable("Y", default_grain="month", default_start="2026-03", + default_periods=4) + m.y.d = d + kept, walked = _kept_and_walked(lambda: y + m.t.fee) + fresh = _outcome(lambda: (mo.lag(m.y.d.bal, fill=0.0) + mo.Variable(0.0)) + m.t.fee) + assert kept == walked == fresh + + def test_a_loop_container_declaring_its_window_after_a_row_was_read_ahead(self): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.t = mo.MultiVariable("T", default_grain="month", default_start="2026-03", + default_periods=2) + m.t.fee = mo.Variable([10.0, 20.0], display_name="fee", extend=mo.zero()) + m.d = mo.MultiVariable("D", loop=True) + y = mo.lag(m.d.bal, fill=0.0) + mo.Variable(0.0) + m.d.default_grain = "month" + m.d.default_start = "2026-03" + m.d.default_periods = 4 + m.d.bal = mo.lag(m.d.bal, fill=1.0) + 1.0 + kept, walked = _kept_and_walked(lambda: y + m.t.fee) + fresh = _outcome(lambda: (mo.lag(m.d.bal, fill=0.0) + mo.Variable(0.0)) + m.t.fee) + assert kept == walked == fresh + + def test_a_container_replaced_after_a_line_became_a_plain_number(self): + # A notebook re-run: a line overwritten with a number, then the + # cell that builds its container runs again. + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.inputs = mo.MultiVariable("Inputs") + m.inputs.rate = mo.Variable(0.05, display_name="rate") + m.inputs.rate = 0.05 + old = m.inputs + m.inputs = mo.MultiVariable("Inputs") + assert old._parent is None and m.inputs is not old + m.remove("inputs") + assert "inputs" not in m._component_names + + def test_a_constant_moved_into_a_one_period_container(self): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.x = mo.MultiVariable("X") + m.x.a = mo.Variable([1.0, 2.0, 3.0, 4.0], display_name="a") + m.x.c = mo.Variable([5.0], display_name="c") + m.t = mo.MultiVariable("T", default_grain="month", default_start="2026-03", + default_periods=2) + m.t.fee = mo.Variable([10.0, 20.0], display_name="fee", extend=mo.zero()) + d = m.x.c + m.x.a + c = m.x.remove("c") + with m: + m.one = mo.MultiVariable("One", default_grain="month", default_start="2026-06", + default_periods=1) + m.one.c = c + kept, walked = _kept_and_walked(lambda: d + m.t.fee) + assert kept == walked + + @pytest.mark.parametrize("target_first", [True, False]) + def test_a_loop_class_instance_built_floating(self, target_first): + class Debt(mo.MultiVariableClass, loop=True): + def compute(self, rev=None): + self._y = mo.lag(self.bal, fill=0.0) + rev + self.bal = mo.lag(self.bal, fill=1.0) + 1.0 + + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.x = mo.MultiVariable("X") + m.x.rev = mo.Variable([1.0, 2.0, 3.0, 4.0], display_name="rev") + m.t = mo.MultiVariable("T", default_grain="month", default_start="2026-03", + default_periods=2) + m.t.fee = mo.Variable([10.0, 20.0], display_name="fee", extend=mo.zero()) + if target_first: + m.y = mo.MultiVariable("Y", default_grain="month", default_start="2026-03", + default_periods=4) + inst = Debt(rev=m.x.rev) + if not target_first: + m.y = mo.MultiVariable("Y", default_grain="month", default_start="2026-03", + default_periods=4) + m.y.debt = inst + kept, walked = _kept_and_walked(lambda: inst._y + m.t.fee) + assert kept == walked + + def test_a_window_slot_deleted(self): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.x = mo.MultiVariable("X", default_grain="month", default_start="2026-03", + default_periods=4) + m.x.a = mo.Variable([1.0, 2.0, 3.0, 4.0], display_name="a") + m.t = mo.MultiVariable("T", default_grain="month", default_start="2026-03", + default_periods=2) + m.t.fee = mo.Variable([10.0, 20.0], display_name="fee", extend=mo.zero()) + d = m.x.a + m.x.a + del m.x.default_grain + kept, walked = _kept_and_walked(lambda: d + m.t.fee) + fresh = _outcome(lambda: (m.x.a + m.x.a) + m.t.fee) + assert kept == walked == fresh + + @pytest.mark.parametrize("move", ["set", "del"]) + def test_track_slices_of_an_emptied_container_whose_window_changes(self, move): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4, + tracks=mo.Tracks("plan", "actual")) + with m: + if move == "del": + m.x = mo.MultiVariable("X", default_grain="month", default_start="2026-03", + default_periods=4) + else: + m.x = mo.MultiVariable("X") + m.x.trk = mo.Variable(plan=[10.0, 20.0, 30.0, 40.0], + actual=[11.0, 22.0, 33.0, 44.0], display_name="trk") + m.t = mo.MultiVariable("T", default_grain="month", default_start="2026-03", + default_periods=2) + m.t.fee = mo.Variable([1.0, 2.0], display_name="fee", extend=mo.zero()) + actual = m.x.trk.at(track="actual") + plan = m.x.trk.at(track="plan") + m.x.remove("trk") + variance = actual - plan + if move == "del": + del m.x.default_grain + else: + m.x.default_grain = "month" + m.x.default_start = "2026-03" + m.x.default_periods = 4 + kept, walked = _kept_and_walked(lambda: variance + m.t.fee) + assert kept == walked + + def test_a_time_line_read_again_for_the_container_it_joins(self): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.x = mo.MultiVariable("X") + m.t = mo.MultiVariable("T", default_grain="month", default_start="2026-03", + default_periods=2) + m.t.fee = mo.Variable([10.0, 20.0], display_name="fee", extend=mo.zero()) + side = mo.Model("side", default_grain="month", default_start="2026-03", + default_periods=4) + with side: + i = mo.time.index + f = i * 1.0 + i + m.x.i = i + kept, walked = _kept_and_walked(lambda: f + m.t.fee) + assert kept == walked == ([2.0, 4.0, 16.0, 28.0], "2026-01") + + def test_a_row_read_ahead_in_a_one_period_loop_container_is_no_constant(self): + # Its window comes from the container it is read from: one period + # of May beside four months is refused when the formula is built. + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.x = mo.MultiVariable("X") + m.x.rev = mo.Variable([1.0, 2.0, 3.0, 4.0], display_name="rev") + m.d = mo.MultiVariable("D", loop=True, default_grain="month", + default_start="2026-05", default_periods=1) + with pytest.raises(ValueError, match="does not cover the combined window"): + mo.lag(m.d.bal, fill=0.0) + m.x.rev + + def test_a_floating_loop_container_joining_with_a_row_still_read_ahead(self): + from modeleon.core import variable_ops + + def build(): + m = mo.Model("m", default_grain="month", default_start="2026-01", + default_periods=4) + with m: + m.x = mo.MultiVariable("X") + m.x.rev = mo.Variable([1.0, 2.0, 3.0, 4.0], display_name="rev") + m.t = mo.MultiVariable("T", default_grain="month", default_start="2026-03", + default_periods=2) + m.t.fee = mo.Variable([10.0, 20.0], display_name="fee", extend=mo.zero()) + m.y = mo.MultiVariable("Y", default_grain="month", default_start="2026-03", + default_periods=4) + d = mo.MultiVariable("D", loop=True) + y = mo.lag(d.bal, fill=0.0) + m.x.rev + m.y.d = d + return _outcome(lambda: y + m.t.fee) + + kept = build() + saved = variable_ops._kept_placement + variable_ops._kept_placement = lambda var: variable_ops._UNREAD + try: + fresh = build() + finally: + variable_ops._kept_placement = saved + assert kept == fresh + + + +def _tracked(): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4, + tracks=mo.Tracks("plan", "actual")) + with m: + m.x = mo.MultiVariable("X") + m.x.rev = mo.Variable([1.0, 2.0, 3.0, 4.0], display_name="rev") + m.x.trk = mo.Variable(plan=[10.0, 20.0, 30.0, 40.0], actual=[11.0, 21.0, 31.0, 41.0], + display_name="trk") + m.t = mo.MultiVariable("T", default_grain="month", default_start="2026-03", + default_periods=2) + m.t.fee = mo.Variable([10.0, 20.0], display_name="fee", extend=mo.zero()) + m.o = mo.MultiVariable("O", default_grain="month", default_start="2026-04", + default_periods=1) + m.o.once = mo.Variable([50.0], display_name="once", extend=mo.zero()) + m.u = mo.MultiVariable("U", default_grain="month", default_start="2025-11", + default_periods=2) + m.u.early = mo.Variable([100.0, 200.0], display_name="early", extend=mo.zero()) + return m + + +#: Calls over a tracked line and lines on other windows, per track: once +#: (April, zero) [0, 0, 0, 50], fee (March, zero) [0, 0, 10, 20]. +_TRACKED = { + "t_sum": (lambda m: mo.SUM(m.x.rev, m.o.once, m.x.trk), + [11.0, 22.0, 33.0, 94.0], [12.0, 23.0, 34.0, 95.0]), + "t_med": (lambda m: mo.MEDIAN(m.x.rev, m.o.once, m.x.trk), + [1.0, 2.0, 3.0, 40.0], [1.0, 2.0, 3.0, 41.0]), + "t_add": (lambda m: m.x.trk + m.o.once, [10.0, 20.0, 30.0, 90.0], [11.0, 21.0, 31.0, 91.0]), + "t_sub": (lambda m: m.x.trk - m.t.fee, [10.0, 20.0, 20.0, 20.0], [11.0, 21.0, 21.0, 21.0]), + "t_if": (lambda m: mo.IF(m.x.rev > 2, m.x.trk, m.o.once), + [0.0, 0.0, 30.0, 40.0], [0.0, 0.0, 31.0, 41.0]), + "t_cmp": (lambda m: mo.IF(m.x.trk - m.t.fee > 15, m.o.once, 0), + [0.0, 0.0, 0.0, 50.0], [0.0, 0.0, 0.0, 50.0]), +} + + +class TestTrackedLines: + @pytest.mark.parametrize("name", list(_TRACKED)) + def test_a_plain_line_is_read_on_the_tracked_line_s_window(self, name): + m = _tracked() + make, plan, actual = _TRACKED[name] + got = make(m) + assert got.value["plan"] == pytest.approx(plan) + assert got.value["actual"] == pytest.approx(actual) + assert got.time.start == "2026-01" + + def test_a_call_reads_what_the_operators_read(self): + m = _tracked() + x = m.x + avg = mo.AVERAGE(x.rev, m.t.fee, x.trk).value + ops = ((x.rev + m.t.fee + x.trk) / 3).value + for role in ("plan", "actual"): + assert avg[role] == pytest.approx(ops[role]) + assert mo.SUM(x.rev, m.o.once, x.trk).value["plan"] == ( + (x.rev + m.o.once) + x.trk).value["plan"] + + def test_a_tracked_line_is_not_continued_past_its_window(self): + m = _tracked() + for call in (lambda: m.x.trk + m.u.early, lambda: mo.SUM(m.u.early, m.x.trk)): + with pytest.raises(ValueError, match="'trk' holds tracks on 4 period"): + call() + + def test_a_list_typed_in_beside_tracks_keeps_its_reading(self): + m = _tracked() + got = mo.IF(m.x.rev > 2, m.x.trk, [7.0, 8.0]).value + assert got["plan"] == [7.0, 8.0, 30.0, 40.0] + + def test_the_workbook_reads_each_track_by_date(self, tmp_path): + m = _tracked() + with m: + m.x.y = (m.x.trk + m.o.once).set_display_name("y") + m.x.s = mo.SUM(m.x.rev, m.t.fee, m.x.trk).set_display_name("s") + path = tmp_path / "tracked.xlsx" + m.to_excel(path) + x = _rows(load_workbook(path)["X"]) + assert x["y"] == ["=B3 + 0", "=C3 + 0", "=D3 + 0", "=E3 + O!B2"] + assert x["y · actual"] == ["=B4 + 0", "=C4 + 0", "=D4 + 0", "=E4 + O!B2"] + assert x["s · actual"] == ["=SUM(B2, 0, B4)", "=SUM(C2, 0, C4)", + "=SUM(D2, T!B2, D4)", "=SUM(E2, T!C2, E4)"] + + +class TestRefusalsSayWhatToDo: + def test_zero_continues_numbers_not_dates_or_text(self): + m = _lines() + with m: + m.t.tdz = mo.Variable([date(2026, 6, 30), date(2026, 7, 31)], display_name="tdz", + extend=mo.zero()) + m.t.twz = mo.Variable(["p", "q"], display_name="twz", extend=mo.zero()) + for call in (lambda: mo.DAYS(m.t.tdz, m.x.dt), lambda: mo.MAX(m.t.tdz, m.x.dt), + lambda: m.t.tdz > m.x.dt): + with pytest.raises(ValueError, match="'tdz' holds dates, and extend=mo.zero"): + call() + with pytest.raises(ValueError, match="'twz' holds text") as refused: + mo.CONCAT(m.x.tx, m.t.twz) + assert "extend=mo.hold()" in str(refused.value) + # Held, the date stands where the line has none. + assert mo.DAYS(m.t.td, m.x.dt).value == [150, 122, 91, 92] + + def test_a_re_grained_line_says_its_rule_stays_behind(self): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=6) + with m: + m.x = mo.MultiVariable("X") + m.x.rev = mo.Variable([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], display_name="rev", + regrain=mo.up("sum")) + m.u = mo.MultiVariable("U", default_grain="month", default_start="2026-01", + default_periods=2) + m.u.early = mo.Variable([100.0, 200.0], display_name="early", extend=mo.zero(), + regrain=mo.up("sum")) + with pytest.raises(ValueError, match="A re-grained line keeps no rule") as refused: + m.u.early.at("quarter") + m.x.rev.at("quarter") + assert "'early' declares extend=mo.zero()" in str(refused.value) + both = (m.u.early + m.x.rev).set_regrain(mo.up("sum")).at("quarter") + assert both.value == [306.0, 15.0] + + +# ══════════════════════════════════════════════════════════════════════ +# A loop container reads a line on another window by date +# ══════════════════════════════════════════════════════════════════════ + + +def _loop(m, make): + """``make(m, d)`` run inside a loop container ``D`` of ``m``.""" + with m: + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + make(m, d) + return m.d + + +#: Rows of a loop over X's four months reading T, U and O by date: fee +#: [0, 0, 10, 20], rate [0.5, 0.5, 0.5, 0.25], cap [9, 8, 8, 8], once +#: [0, 0, 50, 0], level [2, 2, 2, 2]. +_LOOPS = { + "once": (lambda m, d: setattr(d, "bal", mo.lag(d.bal, fill=0.0) + m.o.once), + {"bal": [0.0, 0.0, 50.0, 50.0]}), + "fee": (lambda m, d: setattr(d, "bal", mo.lag(d.bal, fill=0.0) + m.t.fee), + {"bal": [0.0, 0.0, 10.0, 30.0]}), + "rate": (lambda m, d: setattr(d, "bal", mo.lag(d.bal, fill=100.0) * m.t.rate), + {"bal": [50.0, 25.0, 12.5, 3.125]}), + "cap": (lambda m, d: setattr(d, "bal", mo.lag(d.bal, fill=0.0) + m.u.cap), + {"bal": [9.0, 17.0, 25.0, 33.0]}), + "level": (lambda m, d: setattr(d, "bal", mo.lag(d.bal, fill=1.0) * m.o.level), + {"bal": [2.0, 4.0, 8.0, 16.0]}), + "mix": (lambda m, d: setattr(d, "bal", mo.lag(d.bal, fill=0.0) + m.x.rev + m.t.fee), + {"bal": [1.0, 3.0, 16.0, 40.0]}), + "max": (lambda m, d: setattr(d, "bal", mo.MAX(mo.lag(d.bal, fill=0.0), m.t.fee)), + {"bal": [0.0, 0.0, 10.0, 20.0]}), + "if": (lambda m, d: setattr(d, "bal", + mo.IF(m.x.rev > 2, mo.lag(d.bal, fill=0.0) + m.t.fee, 1.0)), + {"bal": [1.0, 1.0, 11.0, 31.0]}), + "two": (lambda m, d: (setattr(d, "a", mo.lag(d.b, fill=0.0) + m.o.once), + setattr(d, "b", d.a + m.t.fee)), + {"a": [0.0, 0.0, 50.0, 60.0], "b": [0.0, 0.0, 60.0, 80.0]}), +} + + +class TestALoopContainerReadsByDate: + @pytest.mark.parametrize("name", list(_LOOPS)) + def test_each_period_reads_the_line_s_value_of_its_date(self, name): + make, expected = _LOOPS[name] + d = _loop(_lines(), make) + for row, want in expected.items(): + assert getattr(d, row).value == pytest.approx(want), row + + def test_the_row_reads_what_the_operators_read(self): + # Outside a loop the same reads align by date: the loop agrees. + m = _lines() + fee_on_x = (m.x.rev + m.t.fee - m.x.rev).value + assert fee_on_x == [0.0, 0.0, 10.0, 20.0] + d = _loop(m, _LOOPS["fee"][0]) + bal, total = d.bal.value, 0.0 + for t, f in enumerate(fee_on_x): + total += f + assert bal[t] == total + + def test_a_line_without_a_rule_and_a_formula_are_refused(self): + for make, message in ( + (lambda m, d: setattr(d, "bal", mo.lag(d.bal, fill=0.0) + m.t.nofill), + "'nofill' does not cover the combined window"), + (lambda m, d: setattr(d, "bal", mo.lag(d.bal, fill=0.0) + m.o.bare), + "'bare' does not cover the combined window"), + (lambda m, d: setattr(d, "bal", mo.lag(d.bal, fill=0.0) + m.t.fee * 2), + r"the line fee \* 2 does not cover the combined window"), + (lambda m, d: setattr(d, "bal", mo.lag(d.bal, fill=0.0) * (1 + m.t.rate)), + r"the line rate \+ 1 does not cover the combined window"), + (lambda m, d: setattr(d, "bal", + mo.lag(d.bal, fill=0.0) + mo.lag(m.t.fee, fill=0.0)), + "does not cover the combined window")): + with pytest.raises(ValueError, match=message): + _loop(_lines(), make) + with pytest.raises(ValueError, match="different grains"): + _loop(_windows(), lambda m, d: setattr(d, "bal", mo.lag(d.bal, fill=0.0) + m.q.h)) + + def test_the_workbook_reads_the_cell_of_each_date(self, tmp_path): + m = _lines() + with m: + for name in ("once", "fee", "rate", "if", "two"): + mv = mo.MultiVariable("D_" + name, loop=True) + setattr(m, "d_" + name, mv) + with mv as d: + _LOOPS[name][0](m, d) + path = tmp_path / "loops.xlsx" + m.to_excel(path) + book = load_workbook(path) + rows = {name: _rows(book["D_" + name]) for name in ("once", "fee", "rate", "if", "two")} + # The row's own previous cell, then the line's cell of the same + # date — O's one cell in March only, T's from March, a rate held. + assert rows["once"]["Bal"] == ["=0 + 0", "=B2 + 0", "=C2 + O!B2", "=D2 + 0"] + assert rows["fee"]["Bal"] == ["=0 + 0", "=B2 + 0", "=C2 + T!B2", "=D2 + T!C2"] + assert rows["rate"]["Bal"] == ["=100.0 * T!B3", "=B2 * T!B3", "=C2 * T!B3", + "=D2 * T!C3"] + assert rows["if"]["Bal"] == ["=IF(X!B2 > 2, 0 + 0, 1.0)", "=IF(X!C2 > 2, B2 + 0, 1.0)", + "=IF(X!D2 > 2, C2 + T!B2, 1.0)", + "=IF(X!E2 > 2, D2 + T!C2, 1.0)"] + assert rows["two"]["A"] == ["=0 + 0", "=B3 + 0", "=C3 + O!B2", "=D3 + 0"] + assert rows["two"]["B"] == ["=B2 + 0", "=C2 + 0", "=D2 + T!B2", "=E2 + T!C2"] + + +# ══════════════════════════════════════════════════════════════════════ +# The workbook reads by date where Python does, by position where it does +# ══════════════════════════════════════════════════════════════════════ + + +class TestTheWorkbook: + def test_each_line_is_read_on_its_date_s_cell(self, tmp_path): + m = _model() + path = tmp_path / "book.xlsx" + m.to_excel(path) + x = _rows(load_workbook(path)["X"]) + # rev B2, flag B3, idx B4, big B5, tx B6, dt B7, lk B8; on T fee + # B2:C2, rate B3:C3, td B5:C5, tw B6:C6, rw B8:C8, cc B9:C9; on U + # early B2:C2, cap B3:C3; K's block B2:E3; I's cells B1..B5. + assert x["if_fee"] == ["=IF(B2 > 2, B2, 0)", "=IF(C2 > 2, C2, 0)", + "=IF(D2 > 2, D2, T!B2)", "=IF(E2 > 2, E2, T!C2)"] + assert x["if_rate"] == ["=IF(B2 > 2, T!B3, U!B3)", "=IF(C2 > 2, T!B3, U!C3)", + "=IF(D2 > 2, T!B3, U!C3)", "=IF(E2 > 2, T!C3, U!C3)"] + assert x["if_flag"] == ["=IF(B3, 0, U!B2)", "=IF(C3, 0, U!C2)", "=IF(D3, T!B2, 0)", + "=IF(E3, T!C2, 0)"] + assert x["and_"] == ["=AND(B2, 0)", "=AND(C2, 0)", "=AND(D2, T!B2)", "=AND(E2, T!C2)"] + assert x["or_"] == ["=OR(B3, U!B2)", "=OR(C3, U!C2)", "=OR(D3, 0)", "=OR(E3, 0)"] + assert x["ch_series"] == ["=CHOOSE(B4, 0, U!B3)", "=CHOOSE(C4, 0, U!C3)", + "=CHOOSE(D4, T!B2, U!C3)", "=CHOOSE(E4, T!C2, U!C3)"] + assert x["ch_scalar"] == ["=CHOOSE(2, B2, 0)", "=CHOOSE(2, C2, 0)", + "=CHOOSE(2, D2, T!B2)", "=CHOOSE(2, E2, T!C2)"] + assert x["md"] == ["=MOD(B5, U!B3)", "=MOD(C5, U!C3)", "=MOD(D5, U!C3)", + "=MOD(E5, U!C3)"] + assert x["gt"] == ["=B2 > 0", "=C2 > 0", "=D2 > T!B2", "=E2 > T!C2"] + assert x["ge"] == ["=U!B3 >= B5", "=U!C3 >= C5", "=U!C3 >= D5", "=U!C3 >= E5"] + assert x["cc_"] == ["=B6 & T!B6", "=C6 & T!B6", "=D6 & T!B6", "=E6 & T!C6"] + assert x["dd"] == ["=DATE(2026, B2, 0)", "=DATE(2026, C2, 0)", + "=DATE(2026, D2, T!B2)", "=DATE(2026, E2, T!C2)"] + assert x["d360"] == ["=DAYS360(B7, T!B5, FALSE)", "=DAYS360(C7, T!B5, FALSE)", + "=DAYS360(D7, T!B5, FALSE)", "=DAYS360(E7, T!C5, FALSE)"] + assert x["yf"][1] == "=YEARFRAC(C7, T!B5, 3)" and x["yf"][3] == "=YEARFRAC(E7, T!C5, 3)" + assert x["dy"] == ["=_xlfn.DAYS(T!B5, B7)", "=_xlfn.DAYS(T!B5, C7)", + "=_xlfn.DAYS(T!B5, D7)", "=_xlfn.DAYS(T!C5, E7)"] + assert x["nwd"][0] == "=NETWORKDAYS(B7, T!B5, I!B5)" + assert x["nwd"][3] == "=NETWORKDAYS(E7, T!C5, I!B5)" + assert x["xl"] == ["=_xlfn.XLOOKUP(B2, I!B1:B2, I!B3:B4, 0)", + "=_xlfn.XLOOKUP(C2, I!B1:B2, I!B3:B4, 0)", + "=_xlfn.XLOOKUP(D2, I!B1:B2, I!B3:B4, T!B2)", + "=_xlfn.XLOOKUP(E2, I!B1:B2, I!B3:B4, T!C2)"] + assert x["hl"] == ["=HLOOKUP(B2, K!B2:E3, T!B8, FALSE)", + "=HLOOKUP(C2, K!B2:E3, T!B8, FALSE)", + "=HLOOKUP(D2, K!B2:E3, T!B8, FALSE)", + "=HLOOKUP(E2, K!B2:E3, T!C8, FALSE)"] + assert x["ix"] == ["=INDEX(K!B2:E3, B4, T!B9)", "=INDEX(K!B2:E3, C4, T!B9)", + "=INDEX(K!B2:E3, D4, T!B9)", "=INDEX(K!B2:E3, E4, T!C9)"] + assert x["vl"][1] == "=VLOOKUP(C8, K!B2:E3, T!B9, FALSE)" + assert x["med"] == ["=MEDIAN(B2, 0, U!B3)", "=MEDIAN(C2, 0, U!C3)", + "=MEDIAN(D2, T!B2, U!C3)", "=MEDIAN(E2, T!C2, U!C3)"] + assert x["sdp"] == ["=_xlfn.STDEV.P(B2, U!B2)", "=_xlfn.STDEV.P(C2, U!C2)", + "=_xlfn.STDEV.P(D2, 0)", "=_xlfn.STDEV.P(E2, 0)"] + # The recurrence's previous cell, then this period's cells by date. + assert x["rc"][1:] == ["=B35 + C2 + 0", "=C35 + D2 + T!B2", "=D35 + E2 + T!C2"] + assert x["ie"] == ["=IFERROR(B2 / 0, 0)", "=IFERROR(C2 / 0, 0)", + "=IFERROR(D2 / T!B2, 0)", "=IFERROR(E2 / T!C2, 0)"] + # O's one cell: once (B2) in March only, 0 elsewhere; level (B3) held. + assert x["o_if"] == ["=IF(B2 > 3, B2, 0)", "=IF(C2 > 3, C2, 0)", + "=IF(D2 > 3, D2, O!B2)", "=IF(E2 > 3, E2, 0)"] + assert x["o_if2"] == ["=IF(B3, O!B3, 0)", "=IF(C3, O!B3, 0)", "=IF(D3, O!B3, O!B2)", + "=IF(E3, O!B3, 0)"] + assert x["o_med"][2:] == ["=MEDIAN(D2, O!B2, O!B3)", "=MEDIAN(E2, 0, O!B3)"] + assert x["o_cat"] == ["=B6 & 0", "=C6 & 0", "=D6 & O!B2", "=E6 & 0"] + assert x["o_rnd"] == ["=ROUND(K!B4, O!B3)", "=ROUND(K!C4, O!B3)", "=ROUND(K!D4, O!B3)", + "=ROUND(K!E4, O!B3)"] + assert x["o_ed"][0] == "=EDATE(B7, O!B3)" + assert x["o_pmt"][3] == "=PMT(0.05, O!B3, K!E3, 0, 0)" + # A formula over X's lines, written in X's column, beside the others' dates. + assert x["w_add"] == ["=B2 * 1 + 0", "=C2 * 1 + 0", "=D2 * 1 + T!B2", "=E2 * 1 + T!C2"] + assert x["w_if"][2:] == ["=IF(D2 * 1 > 2, D2 * 1, T!B2)", "=IF(E2 * 1 > 2, E2 * 1, T!C2)"] + assert x["w_once"] == ["=B2 * 1 + 0", "=C2 * 1 + 0", "=D2 * 1 + O!B2", "=E2 * 1 + 0"] + assert x["w_max"][0] == "=MAX(B2 * 1, 0, U!B3)" + assert x["w_cmp"] == ["=IF(B2 > 2, 0, U!B3)", "=IF(C2 > 2, 0, U!C3)", + "=IF(D2 > 2, T!B2, U!C3)", "=IF(E2 > 2, T!C2, U!C3)"] + # A list of lines on several sheets forms no run in one column: the + # cell carries the value Python read by date, and check_compat says so. + for name in ("mdl", "pct", "lg", "rk"): + assert x[name] == getattr(m.x, name).value, name + assert {i.qpath for i in mo.check_compat(m, "excel")} == { + "m.x.mdl", "m.x.pct", "m.x.lg", "m.x.rk"} + + def test_a_formula_over_one_other_window_is_read_by_position_as_python_reads_it( + self, tmp_path): + # T2 spans four months from November: nothing on X's window meets + # it in ``fee * 2``, so Python pairs January with November, and so + # does the workbook — the cell of the same position, never of the + # same date (which Python did not read). + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.x = mo.MultiVariable("X") + m.x.rev = mo.Variable([1.0, 2.0, 3.0, 4.0], display_name="rev") + m.t2 = mo.MultiVariable("T2", default_grain="month", default_start="2025-11", + default_periods=4) + m.t2.fee = mo.Variable([10.0, 20.0, 30.0, 40.0], display_name="fee", + extend=mo.zero()) + m.x.d = (m.t2.fee * 2).set_display_name("d") + m.x.e = mo.ABS(m.t2.fee).set_display_name("e") + m.x.c = (m.t2.fee > 15).set_display_name("c") + assert m.x.d.value == [20.0, 40.0, 60.0, 80.0] + path = tmp_path / "book.xlsx" + m.to_excel(path) + x = _rows(load_workbook(path)["X"]) + assert x["d"] == ["=T2!B2 * 2", "=T2!C2 * 2", "=T2!D2 * 2", "=T2!E2 * 2"] + assert x["e"] == ["=ABS(T2!B2)", "=ABS(T2!C2)", "=ABS(T2!D2)", "=ABS(T2!E2)"] + assert x["c"] == ["=T2!B2 > 15", "=T2!C2 > 15", "=T2!D2 > 15", "=T2!E2 > 15"] + + def test_constants_keys_and_lines_on_windows_of_their_own(self, tmp_path): + # A constant's one cell in every column; lines along keys in another + # order, on another sheet, key by key; a line on a window of its own + # (start=) that reads another line by position, as Python does. + m = _windows() + with m: + m.x.c_add = (m.x.rev + m.x.c).set_display_name("c_add") + m.x.c_if = mo.IF(m.x.rev > 3, m.x.c, m.t.fee).set_display_name("c_if") + m.x.k_max = mo.MAX(m.x.k, m.t.k2).set_display_name("k_max") + m.t2 = mo.MultiVariable("T2", default_grain="month", default_start="2026-03", + default_periods=4) + m.t2.y = mo.Variable(m.x.rev, start="2026-03", grain="month").set_display_name("y") + assert m.t2.y.value == [1.0, 2.0, 3.0, 4.0] + path = tmp_path / "book.xlsx" + m.to_excel(path) + book = load_workbook(path) + x = _rows(book["X"]) + assert x["c_add"] == ["=B2 + B9", "=C2 + B9", "=D2 + B9", "=E2 + B9"] + assert x["c_if"] == ["=IF(B2 > 3, B9, 0)", "=IF(C2 > 3, B9, 0)", + "=IF(D2 > 3, B9, T!B2)", "=IF(E2 > 3, B9, T!C2)"] + assert x["k_max"][:3] == ["=MAX(B11, T!D11)", "=MAX(C11, T!C11)", "=MAX(D11, T!B11)"] + assert _rows(book["T2"])["y"] == ["=X!B2", "=X!C2", "=X!D2", "=X!E2"] + + def test_a_copy_a_track_slice_and_a_lag_keep_their_positions(self, tmp_path): + # Placed on X by something other than alignment — a copy, a track + # slice, a lag of a line on another window — a value carries no + # mark and reads its line by position, as Python does, beside a + # line read by date. + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.x = mo.MultiVariable("X") + m.x.rev = mo.Variable([1.0, 2.0, 3.0, 4.0], display_name="rev") + m.t = mo.MultiVariable("T", default_grain="month", default_start="2026-03", + default_periods=2) + m.t.fee = mo.Variable([10.0, 20.0], display_name="fee", extend=mo.zero()) + m.t4 = mo.MultiVariable("T4", default_grain="month", default_start="2026-03", + default_periods=4) + m.t4.f4 = mo.Variable([10.0, 20.0, 30.0, 40.0], display_name="f4") + m.t4.tr = mo.Variable(tracks={"plan": [10.0, 20.0, 30.0, 40.0], + "actual": [11.0, 21.0, 31.0, 41.0]}, + display_name="tr") + m.x.cp = m.t4.f4.copy().set_display_name("cp") + m.x.sl = (m.t4.tr.at(track="plan") * 2).set_display_name("sl") + m.x.lg = mo.lag(m.t4.f4, fill=0.0).set_display_name("lg") + m.x.s = (m.x.rev + m.t.fee).set_display_name("s") + assert m.x.cp.value == [10.0, 20.0, 30.0, 40.0] + assert m.x.lg.value == [0.0, 10.0, 20.0, 30.0] + assert not any(getattr(m.x, n)._read_by_date for n in ("cp", "sl", "lg")) + path = tmp_path / "book.xlsx" + m.to_excel(path) + x = _rows(load_workbook(path)["X"]) + assert x["cp"] == ["=T4!B2", "=T4!C2", "=T4!D2", "=T4!E2"] + assert x["sl"] == ["=T4!B3 * 2", "=T4!C3 * 2", "=T4!D3 * 2", "=T4!E3 * 2"] + assert x["lg"] == ["=0", "=T4!B2", "=T4!C2", "=T4!D2"] + assert x["s"] == ["=B2 + 0", "=C2 + 0", "=D2 + T!B2", "=E2 + T!C2"] + + def test_an_aligned_intermediate_on_a_window_of_its_own_is_refused_by_the_export( + self, tmp_path): + # ``bz + cz`` met two windows and runs from November; a function or + # an operator over it alone keeps those dates, and X's columns + # start in January — refused as the bare sum is, naming the sum. + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=6) + with m: + m.x = mo.MultiVariable("X") + m.x.rev = mo.Variable([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], display_name="rev") + m.b = mo.MultiVariable("B", default_grain="month", default_start="2025-11", + default_periods=4) + m.b.bz = mo.Variable([1000.0, 2000.0, 3000.0, 4000.0], display_name="bz", + extend=mo.zero()) + m.c = mo.MultiVariable("C", default_grain="month", default_start="2026-01", + default_periods=4) + m.c.cz = mo.Variable([1.0, 2.0, 3.0, 4.0], display_name="cz", extend=mo.zero()) + m.x.y1 = mo.IFERROR(m.b.bz + m.c.cz, 0).set_display_name("y1") + assert m.x.y1.value == [1000.0, 2000.0, 3001.0, 4002.0, 3.0, 4.0] + with pytest.raises(ValueError, match="window of its own") as refused: + m.to_excel(tmp_path / "w1.xlsx") + assert "'y1'" in str(refused.value) and "the line bz + cz" in str(refused.value) + for name, make in (("y2", lambda: (m.b.bz + m.c.cz) * 1), + ("y3", lambda: mo.ROUND(m.b.bz + m.c.cz, 0))): + with m: + setattr(m.x, name, make().set_display_name(name)) + with pytest.raises(ValueError, match="window of its own"): + m.to_excel(tmp_path / f"{name}.xlsx") + delattr(m.x, name) + delattr(m.x, "y1") + # A total over it reads no dates; the same read inside the sheet's + # window is written by date. + with m: + m.i = mo.MultiVariable("I") + m.i.tot = mo.SUM(m.b.bz + m.c.cz).set_display_name("tot") + m.x.ok = mo.IFERROR(m.x.rev + m.c.cz, 0).set_display_name("ok") + m.to_excel(tmp_path / "ok.xlsx") + book = load_workbook(tmp_path / "ok.xlsx") + assert _rows(book["I"])["tot"][0] == 10010.0 + assert _rows(book["X"])["ok"] == ["=IFERROR(B2 + C!B2, 0)", "=IFERROR(C2 + C!C2, 0)", + "=IFERROR(D2 + C!D2, 0)", "=IFERROR(E2 + C!E2, 0)", + "=IFERROR(F2 + 0, 0)", "=IFERROR(G2 + 0, 0)"] + + def test_two_windows_reaching_beyond_the_sheet_s_are_refused_by_the_export( + self, tmp_path): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.x = mo.MultiVariable("X") + m.x.rev = mo.Variable([1.0, 2.0, 3.0, 4.0], display_name="rev", extend=mo.zero()) + m.x.on = mo.Variable([1.0, 0.0, 1.0, 0.0], display_name="on", extend=mo.zero()) + m.t2 = mo.MultiVariable("T2", default_grain="month", default_start="2025-11", + default_periods=4) + m.t2.fee = mo.Variable([10.0, 20.0, 30.0, 40.0], display_name="fee", + extend=mo.zero()) + m.x.i = mo.IF(m.x.on, m.x.rev, m.t2.fee).set_display_name("i") + m.x.g = (m.x.rev > m.t2.fee).set_display_name("g") + # November to April: on [0, 0, 1, 0, 1, 0]. + assert m.x.i.value == [10.0, 20.0, 1.0, 40.0, 3.0, 0.0] + assert m.x.i.time.start == "2025-11" and m.x.g.time.start == "2025-11" + with pytest.raises(ValueError, match="window of its own") as refused: + m.to_excel(tmp_path / "w.xlsx") + assert "'i'" in str(refused.value) and "'g'" in str(refused.value) + + def test_an_intermediate_aligned_inside_a_formula_is_read_on_its_dates(self, tmp_path): + # ``(rev + fee) * 2``: the sum is aligned by date, the product reads + # it by position — the cells of the sum's dates. + m = _lines() + with m: + m.x.tw2 = ((m.x.rev + m.t.fee) * 2).set_display_name("tw2") + m.x.ng = (-(m.x.rev + m.t.fee)).set_display_name("ng") + assert m.x.tw2.value == [2.0, 4.0, 26.0, 48.0] + path = tmp_path / "book.xlsx" + m.to_excel(path) + x = _rows(load_workbook(path)["X"]) + assert x["tw2"] == ["=(B2 + 0) * 2", "=(C2 + 0) * 2", "=(D2 + T!B2) * 2", + "=(E2 + T!C2) * 2"] + assert x["ng"][0] == "=-(B2 + 0)" and x["ng"][3] == "=-(E2 + T!C2)" + + +# ══════════════════════════════════════════════════════════════════════ +# Lines along keys combine key by key, as a + b combines them +# ══════════════════════════════════════════════════════════════════════ + + +def _keyed(): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with m: + m.s = mo.Variable([1.0, 2.0, 3.0], display_name="s") + m.k = mo.Variable({"Bear": 0.9, "Base": 1.0, "Bull": 1.1}, display_name="k") + m.k2 = mo.Variable({"Bear": 1.0, "Base": 2.0, "Bull": 3.0}, display_name="k2") + m.k3 = mo.Variable({"Bull": 1.0, "Base": 2.0, "Bear": 3.0}, display_name="k3") + m.k4 = mo.Variable({"Bear": 1.0, "Base": 2.0, "Mid": 3.0}, display_name="k4") + return m + + +#: Two lines along the same keys in another order: ``k3`` read in ``k``'s +#: order (Bear 3, Base 2, Bull 1). +_KEYED = { + "plus": (lambda a, b: a + b, [3.9, 3.0, 2.1]), + "SUM": (lambda a, b: mo.SUM(a, b), [3.9, 3.0, 2.1]), + "MAX": (lambda a, b: mo.MAX(a, b), [3.0, 2.0, 1.1]), + "COUNT": (lambda a, b: mo.COUNT(a, b), [2, 2, 2]), + "POWER": (lambda a, b: mo.POWER(a, b), [0.9 ** 3, 1.0, 1.1]), + "IFERROR": (lambda a, b: mo.IFERROR(a / (b - 2), 0), [0.9, 0.0, -1.1]), + "IF": (lambda a, b: mo.IF(a > 1, a, b), [3.0, 2.0, 1.1]), + "AND": (lambda a, b: mo.AND(a > 0.95, b > 1.5), [False, True, False]), + "MOD": (lambda a, b: mo.MOD(a, b), [0.9, 1.0, 0.1]), + "CONCAT": (lambda a, b: mo.CONCAT(a, b), ["0.93", "12", "1.11"]), + "MEDIAN": (lambda a, b: mo.MEDIAN(a, b), [1.95, 1.5, 1.05]), + "compare": (lambda a, b: a > b, [False, False, True]), +} + + +class TestLinesAlongKeys: + @pytest.mark.parametrize("name", list(_KEYED)) + def test_another_order_is_combined_key_by_key(self, name): + m = _keyed() + call, expected = _KEYED[name] + got = call(m.k, m.k3) + if isinstance(expected[0], float): + assert got.value == pytest.approx(expected) + else: + assert got.value == expected + assert got._keys == ["Bear", "Base", "Bull"] + + @pytest.mark.parametrize("name", list(_KEYED)) + def test_other_keys_are_refused(self, name): + m = _keyed() + with pytest.raises(ValueError, match="different key sets|other keys"): + m.k + m.k4 + with pytest.raises(ValueError, match="different key sets|other keys"): + _KEYED[name][0](m.k, m.k4) + + @pytest.mark.parametrize("name", list(_KEYED)) + def test_against_a_period_series_it_is_refused(self, name): + m = _keyed() + with pytest.raises(ValueError, match="runs along its keys"): + _KEYED[name][0](m.k, m.s) + + def test_a_line_along_keys_among_many_series_is_refused(self): + m = _keyed() + many = [m.s * (i + 1) for i in range(50)] + assert mo.SUM(*many).value == pytest.approx([1275.0, 2550.0, 3825.0]) + with pytest.raises(ValueError, match="runs along its keys"): + mo.SUM(*many, m.k) + with pytest.raises(ValueError, match="runs along its keys"): + mo.IF(m.s > 1, m.s, m.k) + + def test_one_line_along_keys_keeps_them(self): + m = _keyed() + for got in (mo.ABS(m.k), mo.ROUND(m.k, 1), mo.POWER(m.k, 2), mo.IF(m.k > 1, m.k, 0), + mo.NOT(m.k > 1), mo.CONCAT(m.k, "x")): + assert got._keys == ["Bear", "Base", "Bull"] + with pytest.raises(ValueError, match="runs along its keys"): + got + m.s + + def test_the_workbook_names_each_key_s_cell(self, tmp_path): + m = _keyed() + with m: + m.p = (m.k + m.k3).set_display_name("p") + m.pw = mo.POWER(m.k, m.k3).set_display_name("pw") + m.same = (m.k + m.k2).set_display_name("same") + path = tmp_path / "keys.xlsx" + m.to_excel(path) + rows = _rows(load_workbook(path)["M"]) + # k is B4:D4 (Bear, Base, Bull); k3 is B7:D7 (Bull, Base, Bear). + assert rows["p"] == ["=B4 + D7", "=C4 + C7", "=D4 + B7"] + assert rows["pw"] == ["=POWER(B4, D7)", "=POWER(C4, C7)", "=POWER(D4, B7)"] + assert rows["same"] == ["=B4 + B5", "=C4 + C5", "=D4 + D5"] + + +# ══════════════════════════════════════════════════════════════════════ +# A list of lines given as one argument +# ══════════════════════════════════════════════════════════════════════ + + +def _listed(): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with m: + m.x = mo.MultiVariable("X") + m.x.a = mo.Variable([1.0, 2.0, 3.0], display_name="a") + m.x.b = mo.Variable([10.0, 20.0, 30.0], display_name="b") + m.i = mo.MultiVariable("I") + m.i.p = mo.Variable(4.0, display_name="p") + m.i.q = mo.Variable(5.0, display_name="q") + return m + + +_AGGREGATES = ("SUM", "AVERAGE", "MIN", "MAX", "PRODUCT", "COUNT", "COUNTA") + + +class TestAListOfLinesAsOneArgument: + @pytest.mark.parametrize("name,expected", [ + ("SUM", 9.0), ("AVERAGE", 4.5), ("MIN", 4.0), ("MAX", 5.0), ("PRODUCT", 20.0), + ("COUNT", 2), ("COUNTA", 2), + ]) + def test_scalar_lines_are_one_number(self, name, expected): + m = _listed() + call = getattr(mo, name) + got = call([m.i.p, m.i.q]) + assert got.value == expected and got.var_type == "scalar" + assert call((m.i.p, m.i.q)).value == expected + # Beside a series, the same values in every period. + assert call([m.i.p, m.i.q], m.x.a).value == call(m.i.p, m.i.q, m.x.a).value + + @pytest.mark.parametrize("name", ("SUM", "AVERAGE", "MIN", "MAX", "PRODUCT")) + def test_the_five_read_the_lines_one_by_one(self, name): + m = _listed() + got = getattr(mo, name)([m.i.p, m.i.q]) + assert got.formula == f"{name}(p, q)" + with m: + m.i.p.set_unit("USD") + m.i.q.set_unit("EUR") + if name != "PRODUCT": + with pytest.raises(ValueError, match="one unit"): + getattr(mo, name)([m.i.p, m.i.q]) + + @pytest.mark.parametrize("name", _AGGREGATES) + def test_a_series_in_the_list_is_refused_naming_both_spellings(self, name): + m = _listed() + for arg in ([m.x.a, m.x.b], (m.x.a, m.x.b), [m.x.a], [m.x.a, m.i.p]): + with pytest.raises(ValueError) as refused: + getattr(mo, name)(arg) + text = str(refused.value) + assert f"mo.{name}([" in text and "per period" in text, text + assert "over all their periods" in text, text + with pytest.raises(ValueError, match="per period, the lines one by one, " + rf"mo\.{name}\(a, b\)"): + getattr(mo, name)([m.x.a, m.x.b], m.i.p) + + def test_the_sum_s_two_readings_are_spelled_out(self): + m = _listed() + with pytest.raises(ValueError) as refused: + mo.SUM([m.x.a, m.x.b]) + assert ("per period, the lines one by one, mo.SUM(a, b); the total over all their " + "periods, mo.SUM(a) + mo.SUM(b)") in str(refused.value) + + def test_the_statistics_read_a_list_of_series_one_sample_per_period(self): + m = _listed() + assert mo.MEDIAN([m.x.a, m.x.b]).value == [5.5, 11.0, 16.5] + assert mo.MEDIAN([m.i.p, m.i.q]).value == 4.5 + + def test_the_workbook_writes_each_line_s_cell(self, tmp_path): + m = _listed() + with m: + for name in _AGGREGATES: + setattr(m.i, name.lower(), getattr(mo, name)([m.i.p, m.i.q]) + .set_display_name(name.lower())) + path = tmp_path / "list.xlsx" + m.to_excel(path) + rows = _rows(load_workbook(path)["I"]) + for name in ("SUM", "AVERAGE", "MIN", "MAX", "PRODUCT"): + assert rows[name.lower()][0] == f"={name}(B1, B2)", name + # A count reads the list as the range it is, as before. + assert rows["count"][0] == "=COUNT(B1:B2)" + assert rows["counta"][0] == "=COUNTA(B1:B2)" + + +# ══════════════════════════════════════════════════════════════════════ +# The spreadsheet computes the same numbers +# ══════════════════════════════════════════════════════════════════════ + + +def _soffice(): + for candidate in ("soffice", "libreoffice"): + found = shutil.which(candidate) + if found: + return found + mac = Path("/Applications/LibreOffice.app/Contents/MacOS/soffice") + return str(mac) if mac.exists() else None + + +def _recalculated(m, tmp_path, name): + soffice = _soffice() + if soffice is None: + pytest.skip("LibreOffice not installed") + path = tmp_path / f"{name}.xlsx" + m.to_excel(path) + subprocess.run( + [soffice, f"-env:UserInstallation={(tmp_path / 'prof').as_uri()}", + "--headless", "--convert-to", "xlsx", "--outdir", + str(tmp_path / "calc"), str(path)], + check=True, capture_output=True, + ) + return load_workbook(tmp_path / "calc" / f"{name}.xlsx", data_only=True) + + +def _same(got, want): + if isinstance(want, bool): + return got == want + if isinstance(want, date): + return isinstance(got, datetime) and got.date() == want + if isinstance(want, str): + return got == want + return got == pytest.approx(want) + + +@pytest.mark.slow +def test_the_workbook_recalculates_every_call_by_date(tmp_path): + m = _model() + with m: + m.x.tw2 = ((m.x.rev + m.t.fee) * 2).set_display_name("tw2") + x = _rows(_recalculated(m, tmp_path, "dated")["X"]) + for name in [*_CALLS, "tw2"]: + want = getattr(m.x, name).value + got = x[name][:4] + assert len(got) == len(want) == 4, (name, got, want) + assert all(_same(g, w) for g, w in zip(got, want)), (name, got, want) + + +@pytest.mark.slow +def test_the_workbook_recalculates_constants_keys_and_windows_of_their_own(tmp_path): + m = _windows() + with m: + m.x.c_add = (m.x.rev + m.x.c).set_display_name("c_add") + m.x.c_max = mo.MAX(m.x.rev, m.x.c).set_display_name("c_max") + m.x.c_if = mo.IF(m.x.rev > 3, m.x.c, m.t.fee).set_display_name("c_if") + m.x.k_max = mo.MAX(m.x.k, m.t.k2).set_display_name("k_max") + m.x.k_add = (m.x.k + m.t.k2).set_display_name("k_add") + m.t2 = mo.MultiVariable("T2", default_grain="month", default_start="2026-03", + default_periods=4) + m.t2.y = mo.Variable(m.x.rev, start="2026-03", grain="month").set_display_name("y") + m.t2.z = mo.Variable(m.x.rev * 2, start="2026-03", grain="month").set_display_name("z") + book = _recalculated(m, tmp_path, "windows") + x, t2 = _rows(book["X"]), _rows(book["T2"]) + for name in ("c_add", "c_max", "c_if", "k_max", "k_add"): + want = getattr(m.x, name).value + assert x[name][:len(want)] == pytest.approx(want), name + for name in ("y", "z"): + assert t2[name][:4] == pytest.approx(getattr(m.t2, name).value), name + + +@pytest.mark.slow +def test_the_workbook_recalculates_each_track_by_date(tmp_path): + m = _tracked() + with m: + for name, (make, _, _) in _TRACKED.items(): + setattr(m.x, name, make(m).set_display_name(name)) + x = _rows(_recalculated(m, tmp_path, "tracked")["X"]) + for name, (_, plan, actual) in _TRACKED.items(): + assert x[name][:4] == pytest.approx(plan), name + assert x[f"{name} · actual"][:4] == pytest.approx(actual), name + + +@pytest.mark.slow +def test_the_workbook_recalculates_a_window_written_without_its_zero(tmp_path): + m = _unpadded() + with m: + m.x.a = (m.x.rev + m.t.fee).set_display_name("a") + m.x.s = mo.SUM(m.x.rev, m.t.fee).set_display_name("s") + m.x.i = mo.IF(m.x.rev > 2, m.t.fee, m.x.rev).set_display_name("i") + x = _rows(_recalculated(m, tmp_path, "unpadded")["X"]) + for name in ("a", "s", "i"): + assert x[name][:6] == pytest.approx(getattr(m.x, name).value), name + + +@pytest.mark.slow +def test_the_workbook_recalculates_lines_along_keys_key_by_key(tmp_path): + m = _keyed() + with m: + m.p = (m.k + m.k3).set_display_name("p") + m.pw = mo.POWER(m.k, m.k3).set_display_name("pw") + m.sm = mo.SUM(m.k, m.k3).set_display_name("sm") + m.mx = mo.MAX(m.k, m.k3).set_display_name("mx") + m.f = mo.IF(m.k > 1, m.k, m.k3).set_display_name("f") + rows = _rows(_recalculated(m, tmp_path, "keys")["M"]) + for name in ("p", "pw", "sm", "mx", "f"): + assert rows[name][:3] == pytest.approx(getattr(m, name).value), name + + +@pytest.mark.slow +def test_the_workbook_recalculates_a_list_of_scalar_lines(tmp_path): + m = _listed() + with m: + for name in _AGGREGATES: + setattr(m.i, name.lower(), getattr(mo, name)([m.i.p, m.i.q]) + .set_display_name(name.lower())) + rows = _rows(_recalculated(m, tmp_path, "listed")["I"]) + for name in _AGGREGATES: + assert rows[name.lower()][0] == pytest.approx(getattr(m.i, name.lower()).value), name + + +@pytest.mark.slow +def test_the_workbook_recalculates_a_loop_container_by_date(tmp_path): + m = _lines() + with m: + for name, (make, _) in _LOOPS.items(): + mv = mo.MultiVariable("D_" + name, loop=True) + setattr(m, "d_" + name, mv) + with mv as d: + make(m, d) + book = _recalculated(m, tmp_path, "loops") + for name, (_, expected) in _LOOPS.items(): + rows = _rows(book["D_" + name]) + for row, want in expected.items(): + assert rows[row.capitalize()][:4] == pytest.approx(want), (name, row) diff --git a/tests/test_logical.py b/tests/test_logical.py new file mode 100644 index 0000000..faca31a --- /dev/null +++ b/tests/test_logical.py @@ -0,0 +1,681 @@ +# SPDX-License-Identifier: Apache-2.0 +"""``IFERROR``, ``ISERROR``, ``ISNUMBER``, ``XOR``, ``IFS`` and ``SWITCH`` +compute what the spreadsheet computes, with the formula as the product. + +``IFERROR`` catches an error marker before it cascades; ``ISERROR`` and +``ISNUMBER`` name a cell's kind and are never an error themselves; +``XOR`` counts the conditions that hold; ``IFS`` picks the value of the +first condition that holds and ``SWITCH`` the value of the first case +equal to its expression — both lazily, a branch not taken never read. +Each is one call per period through the shared helper: scalars +broadcast, a one-value series broadcasts, other lengths are refused, +tracked operands lift per coordinate, the AST is a call over the +operands. An operand reads as its cell does: an empty text typed into a +line is the blank cell the workbook writes; a formula's is text. +""" + +from __future__ import annotations + +import datetime as dt + +import pytest + +import modeleon as mo +from modeleon.core.expr import FuncCall, Literal, VarRef +from modeleon.functions.logical import IFERROR, IFS, ISERROR, ISNUMBER, SWITCH, XOR +from modeleon.functions.registry import ( + admits_in_loop, + excel_name, + file_prefix, + is_elementwise, + is_passthrough, + is_range_taking, + spec, +) + +FUNCTIONS = { + "IFERROR": IFERROR, "ISERROR": ISERROR, "ISNUMBER": ISNUMBER, + "XOR": XOR, "IFS": IFS, "SWITCH": SWITCH, +} +MARKERS = ["#DIV/0!", "#VALUE!", "#N/A", "#NUM!", "#NAME?", "#REF!", "#NULL!"] + + +def _model(): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with m: + m.x = mo.MultiVariable("X") + m.x.a = mo.Variable([3.0, 8.0, 5.0], display_name="a") + m.x.z = mo.Variable([1.0, 0.0, 2.0], display_name="z") + m.x.q = (m.x.a / m.x.z).set_display_name("q") + return m + + +def _empty_text_formula(m): + """A formula line whose every value is the empty text.""" + out = IFS(m.x.a > 0.0, "", True, "x") + assert out.value == ["", "", ""] + return out + + +def _tracked(): + m = mo.Model("t", tracks=mo.Tracks("plan", "actual"), + default_grain="month", default_start="2026-01", default_periods=2) + with m: + m.x = mo.Variable(plan=[3.0, "#DIV/0!"], actual=[5.0, 8.0]) + return m + + +# ══════════════════════════════════════════════════════════════════════ +# IFERROR +# ══════════════════════════════════════════════════════════════════════ + + +class TestIfError: + def test_the_fallback_stands_in_for_an_error(self): + m = _model() + assert m.x.q.value == [3.0, "#DIV/0!", 2.5] + assert IFERROR(m.x.q, 0.0).value == [3.0, 0.0, 2.5] + assert IFERROR(m.x.q, "n/a").value == [3.0, "n/a", 2.5] + + def test_every_marker_is_an_error(self): + out = IFERROR(mo.Variable([*MARKERS, 1.0]), -1.0) + assert out.value == [-1.0] * len(MARKERS) + [1.0] + + def test_the_fallback_may_vary_by_period_or_be_an_error_itself(self): + m = _model() + assert IFERROR(m.x.q, m.x.a).value == [3.0, 8.0, 2.5] + assert IFERROR(m.x.q, [9.0, 8.0, 7.0]).value == [3.0, 8.0, 2.5] + assert IFERROR(m.x.q, mo.Variable(["#N/A"] * 3)).value == [3.0, "#N/A", 2.5] + + def test_a_blank_reads_as_zero(self): + # A formula that lands on a blank cell shows 0, whichever operand + # the blank is. + assert IFERROR(mo.Variable([None, 5.0]), "x").value == [0, 5.0] + assert IFERROR(mo.Variable(["#N/A", 2.0]), mo.Variable([None, None])).value == [0, 2.0] + + def test_an_empty_text_typed_into_a_line_is_a_blank_a_formula_s_is_text(self): + m = _model() + # The workbook writes a typed empty text as nothing: a blank cell. + assert IFERROR(mo.Variable(["", 5.0]), "x").value == [0, 5.0] + assert IFERROR(_empty_text_formula(m), "x").value == ["", "", ""] + assert IFERROR("", 1.0).value == "" + + def test_scalars_only_is_one_value(self): + assert IFERROR(5.0, 0).value == 5.0 and IFERROR(5.0, 0).var_type == "scalar" + assert IFERROR(mo.Variable(1.0) / 0, -1.0).value == -1.0 + + def test_both_operands_are_precedents_and_the_ast_is_a_call(self): + m = _model() + with m: + m.x.safe = IFERROR(m.x.q, m.x.a) + assert m.x.safe.dependencies == {m.x.q.id, m.x.a.id} + call = m.x.safe._expr + assert isinstance(call, FuncCall) and call.func == "IFERROR" + assert [type(arg) for arg in call.args] == [VarRef, VarRef] + lit = IFERROR(m.x.q, 0.0)._expr + assert isinstance(lit.args[1], Literal) and lit.args[1].value == 0.0 + + def test_the_formula_is_the_source(self): + m = _model() + out = IFERROR(m.x.q, 0.0) + assert out.formula == "IFERROR(q, 0.0)" + assert out._source_code == "IFERROR(q, 0.0)" + assert IFERROR(m.x.a / m.x.z, "n/a").formula == "IFERROR((a / z), 'n/a')" + + def test_typed_by_the_value_where_it_is_not_an_error(self): + cost = mo.Variable([3.0, "#DIV/0!"], unit="$") + assert IFERROR(cost, 0.0).value_type == "float" + # A text fallback does not make a row of numbers a row of text … + assert IFERROR(cost, "n/a").value_type == "float" + assert IFERROR(mo.Variable(["#DIV/0!", 3.0]), "n/a").value_type == "float" + # … and a row read in with markers among its numbers, typed as + # text, is cleaned into numbers. + read_in = mo.Variable(["#N/A", 1.0, "#VALUE!"]) + assert read_in.value_type == "string" + assert IFERROR(read_in, 0.0).value_type == "float" + assert IFERROR(mo.Variable(["x", "#N/A"]), "").value_type == "string" + assert IFERROR(mo.Variable([1, "#N/A"]), 0).value_type == "int" + assert IFERROR(mo.Variable([dt.date(2026, 1, 1), "#N/A"]), dt.date(2026, 1, 2)) \ + .value_type == "datetime" + # A value that is an error throughout is typed by the fallback. + assert IFERROR(mo.Variable(["#N/A", "#N/A"]), "none").value_type == "string" + assert IFERROR(mo.Variable(["#N/A", "#N/A"]), 0.5).value_type == "float" + + def test_the_unit_the_operands_share(self): + cost = mo.Variable([3.0, "#DIV/0!"], unit="$") + assert str(IFERROR(cost, 0.0)._unit) == "$" + assert str(IFERROR(cost, mo.Variable([1.0, 1.0], unit="$"))._unit) == "$" + with pytest.raises(ValueError, match="mo.IFERROR takes its values in one unit"): + IFERROR(cost, mo.Variable([1.0, 1.0], unit="hr")) + + def test_a_tracked_operand_lifts_per_track(self): + m = _tracked() + with m: + m.h = IFERROR(m.x, 0.0) + assert dict(m.h._value.items()) == {"plan": [3.0, 0.0], "actual": [5.0, 8.0]} + assert m.h.var_type == "list" + + +# ══════════════════════════════════════════════════════════════════════ +# ISERROR and ISNUMBER +# ══════════════════════════════════════════════════════════════════════ + + +class TestIsError: + def test_names_an_error_and_is_never_one(self): + m = _model() + assert ISERROR(m.x.q).value == [False, True, False] + assert ISERROR(mo.Variable(MARKERS)).value == [True] * len(MARKERS) + assert ISERROR(mo.Variable([1.0, "text", None, True, dt.date(2026, 1, 1)])).value == \ + [False, False, False, False, False] + + def test_any_text_starting_with_a_hash_is_a_marker_to_the_engine(self): + # The convention arithmetic carries: ``'#…'`` is an error cell. + assert ISERROR(mo.Variable(["#x", "x#"])).value == [True, False] + + def test_scalars_only_is_one_value(self): + assert ISERROR(1.0).value is False + assert ISERROR(mo.Variable(1.0) / 0).value is True + assert ISERROR(2.0).var_type == "scalar" + + def test_a_boolean_without_a_unit(self): + m = _model() + out = ISERROR(mo.Variable([3.0, "#DIV/0!"], unit="$")) + assert out.value_type == "bool" and out._unit is None + assert ISERROR(m.x.q).formula == "ISERROR(q)" == ISERROR(m.x.q)._source_code + + def test_a_tracked_operand_lifts_per_track(self): + m = _tracked() + with m: + m.e = ISERROR(m.x) + assert dict(m.e._value.items()) == {"plan": [False, True], "actual": [False, False]} + + +class TestIsNumber: + def test_numbers_dates_and_durations_are_numbers(self): + row = mo.Variable([1, 2.5, dt.date(2026, 1, 1), dt.datetime(2026, 1, 1, 12), + dt.timedelta(days=3)]) + assert ISNUMBER(row).value == [True] * 5 + + def test_booleans_text_blanks_and_errors_are_not(self): + # A number written as text is text; a boolean is not a number. + row = mo.Variable([True, False, "5", "text", "", None, "#DIV/0!", "#N/A"]) + assert ISNUMBER(row).value == [False] * 8 + + def test_scalars_only_is_one_value(self): + assert ISNUMBER(7).value is True and ISNUMBER("7").value is False + assert ISNUMBER(mo.Variable(1.0) / 0).value is False + assert ISNUMBER(7).var_type == "scalar" + + def test_a_boolean_without_a_unit(self): + m = _model() + out = ISNUMBER(mo.Variable([3.0, 8.0], unit="$")) + assert out.value_type == "bool" and out._unit is None + assert ISNUMBER(m.x.a).formula == "ISNUMBER(a)" == ISNUMBER(m.x.a)._source_code + + +class TestAValueGivenToTheCall: + """A Python value reaching one of these functions is read as the cell + it stands for — a loop runs the function again over its rows' plain + values, where ``None`` is a blank cell and ``'#DIV/0!'`` the error.""" + + def test_none_is_a_blank_cell(self): + assert ISNUMBER(None).value is False + assert XOR(None, True).value is True + assert SWITCH(None, "", "empty", "other").value == "empty" + assert IFS(None, 1.0, True, 2.0).value == 2.0 + + def test_a_marker_is_the_error(self): + assert ISERROR("#N/A").value is True + assert IFERROR("#DIV/0!", 0).value == 0 + assert IFS(True, "#N/A").value == "#N/A" + + +# ══════════════════════════════════════════════════════════════════════ +# XOR +# ══════════════════════════════════════════════════════════════════════ + + +class TestXor: + @pytest.mark.parametrize("conditions, expected", [ + ((True,), True), + ((False,), False), + ((True, True), False), + ((True, False), True), + ((True, True, True), True), + ((True, True, True, True), False), + ((1, 0), True), + ((2, -3), False), + ((0, 0), False), + ]) + def test_true_when_an_odd_number_of_conditions_hold(self, conditions, expected): + out = XOR(*conditions) + assert out.value is expected and out.var_type == "scalar" + + def test_element_wise_over_series(self): + m = _model() + assert XOR(m.x.a > 4.0, True).value == [True, False, False] + assert XOR(m.x.a > 4.0, m.x.z > 0.0, True).value == [False, False, True] + assert XOR(mo.Variable([0, 2, -1])).value == [False, True, True] + + def test_a_blank_is_skipped_and_nothing_left_is_value(self): + # As the spreadsheet skips a blank cell; an empty text typed into + # a line is written as one. + out = XOR(mo.Variable([None, 2.0, None]), mo.Variable([True, False, 0])) + assert out.value == [True, True, False] + assert XOR(mo.Variable(["", True]), True).value == [True, False] + assert XOR(mo.Variable([None, None])).value == ["#VALUE!"] * 2 + + def test_text_is_refused(self): + # A text cell the spreadsheet skips; text typed into the formula + # is #VALUE! — no value matches both, so none is given. + m = _model() + with m: + m.x.answer = mo.Variable(["yes", "no", "yes"]) + with pytest.raises(TypeError, match="XOR counts conditions"): + XOR(m.x.answer, True) + with pytest.raises(TypeError, match="XOR counts conditions"): + XOR(mo.Variable(["abc", True]), True) + with pytest.raises(TypeError, match="'TRUE'"): + XOR(True, "TRUE") + assert XOR(m.x.answer == "yes", True).value == [False, True, False] + + def test_an_empty_text_is_skipped_as_a_blank(self): + # Typed into a line it is the empty cell the workbook writes; a + # formula's, in the cell XOR reads, is text the spreadsheet skips. + m = _model() + with m: + m.x.label = _empty_text_formula(m) + assert XOR(m.x.label, True).value == [True] * 3 + assert XOR(m.x.label).value == ["#VALUE!"] * 3 + + def test_the_first_error_in_reading_order_is_the_result(self): + m = _model() + assert (m.x.q > 1.0).value == [True, "#DIV/0!", True] + out = XOR(m.x.q > 1.0, mo.Variable([True, "#N/A", True])) + assert out.value == [False, "#DIV/0!", False] + assert XOR(mo.Variable(["#N/A", "#DIV/0!", "#NUM!"]), m.x.q > 1.0).value == \ + ["#N/A", "#DIV/0!", "#NUM!"] + + def test_no_condition_is_refused(self): + with pytest.raises(TypeError, match="at least one condition"): + XOR() + + def test_a_boolean_without_a_unit_and_the_formula(self): + m = _model() + out = XOR(m.x.a > 4.0, True) + assert out.value_type == "bool" and out._unit is None + assert out.formula == "XOR((a > 4.0), True)" == out._source_code + assert XOR(mo.Variable([1.0, 0.0], unit="$"))._unit is None + + def test_a_tracked_operand_lifts_per_track(self): + m = _tracked() + with m: + m.o = XOR(m.x > 4.0, True) + assert dict(m.o._value.items()) == {"plan": [True, "#DIV/0!"], "actual": [False, False]} + + +# ══════════════════════════════════════════════════════════════════════ +# IFS +# ══════════════════════════════════════════════════════════════════════ + + +class TestIfs: + def test_the_value_of_the_first_condition_that_holds(self): + m = _model() + out = IFS(m.x.a > 7.0, "big", m.x.a > 4.0, "mid", True, "small") + assert out.value == ["small", "big", "mid"] and out.var_type == "list" + assert IFS(m.x.a > 4.0, m.x.a, True, 0.0).value == [0.0, 8.0, 5.0] + + def test_no_condition_holding_is_na(self): + m = _model() + assert IFS(m.x.a > 100.0, 1.0).value == ["#N/A"] * 3 + assert IFS(False, 1.0).value == "#N/A" + assert IFERROR(IFS(m.x.a > 100.0, 1.0), -1.0).value == [-1.0] * 3 + + def test_a_branch_not_taken_is_never_read(self): + m = _model() + # The error in q never reaches the result where z is 0 … + assert IFS(m.x.z == 0, 0.0, True, m.x.q).value == [3.0, 0.0, 2.5] + # … and a condition after the first that holds is not read either. + assert IFS(True, 1.0, m.x.q > 1.0, 2.0).value == [1.0, 1.0, 1.0] + + def test_a_condition_that_is_an_error_is_the_result(self): + m = _model() + assert IFS(m.x.q > 1.0, 1.0, True, 2.0).value == [1.0, "#DIV/0!", 1.0] + assert IFS(mo.Variable(["#N/A", False]), 1.0, True, 2.0).value == ["#N/A", 2.0] + + def test_how_a_condition_is_read(self): + # A number is TRUE when nonzero; a blank is FALSE; booleans are + # themselves. + assert IFS(mo.Variable([0, 2, -1]), "yes", True, "no").value == ["no", "yes", "yes"] + assert IFS(mo.Variable([None, "", 1]), "a", True, "b").value == ["b", "b", "a"] + assert IFS(mo.Variable([True, False]), 1.0, True, 2.0).value == [1.0, 2.0] + + def test_the_text_true_or_false_is_that_boolean_other_text_is_value(self): + words = mo.Variable(["TRUE", "FALSE", "true", "False"]) + assert IFS(words, 1.0, True, 2.0).value == [1.0, 2.0, 1.0, 2.0] + assert IFS("TRUE", 1.0, True, 2.0).value == 1.0 + # A number written as text is text, as Excel reads it. + assert IFS(mo.Variable(["abc", "1"]), 1.0, True, 2.0).value == ["#VALUE!"] * 2 + + def test_an_empty_text_is_text_unless_typed_into_a_line(self): + m = _model() + assert IFS(_empty_text_formula(m), 1.0, True, 2.0).value == ["#VALUE!"] * 3 + assert IFS("", 1.0, True, 2.0).value == "#VALUE!" + assert IFS(mo.Variable(["", True]), 1.0, True, 2.0).value == [2.0, 1.0] + + def test_a_picked_blank_shows_zero_and_does_not_type_the_result(self): + out = IFS(True, mo.Variable([None, 2.5])) + assert out.value == [0, 2.5] and out.value_type == "float" + assert IFS(True, mo.Variable(["", 2.5])).value == [0, 2.5] + out = IFS(mo.Variable([True, False]), mo.Variable([None, 9]), True, -1.0) + assert out.value == [0, -1.0] and out.value_type == "float" + assert IFS(False, 1.0, True, None).value == 0 + + def test_a_scalar_condition_over_a_series_value_gives_a_series(self): + m = _model() + out = IFS(True, m.x.a, False, 0.0) + assert out.value == [3.0, 8.0, 5.0] and out.var_type == "list" + assert IFS(m.x.a > 4.0, 1.0, True, 0.0).var_type == "list" + + def test_scalars_only_is_one_value(self): + out = IFS(7.0 > 5.0, "hi", True, "lo") + assert out.value == "hi" and out.var_type == "scalar" + assert IFS(mo.Variable(7.0) > 5.0, "hi", True, "lo").value == "hi" + + def test_pairs_only(self): + m = _model() + with pytest.raises(TypeError, match="condition and value pairs"): + IFS(m.x.a > 1.0) + with pytest.raises(TypeError, match="got 3 operands"): + IFS(m.x.a > 1.0, 1.0, True) + with pytest.raises(TypeError, match="got 0 operands"): + IFS() + + def test_typed_by_the_branch_taken(self): + m = _model() + assert IFS(m.x.a > 4.0, "big", True, "small").value_type == "string" + assert IFS(m.x.a > 4.0, 1, True, 0).value_type == "int" + assert IFS(m.x.a > 4.0, True, True, False).value_type == "bool" + assert IFS(m.x.a > 4.0, dt.date(2026, 1, 1), True, dt.date(2027, 1, 1)) \ + .value_type == "datetime" + # Typed by the first value that is not an error marker. + assert IFS(m.x.q > 1.0, "x", True, "y").value_type == "string" + + def test_the_unit_the_values_share(self): + m = _model() + cost = mo.Variable([3.0, 8.0, 5.0], unit="$") + out = IFS(m.x.a > 4.0, cost, True, 0.0) + assert out.value_type == "float" and str(out._unit) == "$" + # A condition is not a value the result takes: its unit is not + # the result's. + hours = mo.Variable([1.0, 0.0, 2.0], unit="hr") + out = IFS(hours, cost, True, 0.0) + assert out.value == [3.0, 0.0, 5.0] and str(out._unit) == "$" + assert IFS(hours, 1.0, True, 0.0)._unit is None + assert IFS(cost > 4.0, "x", True, "y")._unit is None + + def test_the_formula_is_the_source_and_the_ast_is_a_call(self): + m = _model() + out = IFS(m.x.a > 7.0, "big", m.x.a > 4.0, "mid", True, "small") + assert out.formula == "IFS((a > 7.0), 'big', (a > 4.0), 'mid', True, 'small')" + assert out._source_code == out.formula + call = out._expr + assert isinstance(call, FuncCall) and call.func == "IFS" and len(call.args) == 6 + assert isinstance(call.args[0], VarRef) and isinstance(call.args[1], Literal) + with m: + m.x.hot = m.x.a > 7.0 + m.x.band = IFS(m.x.hot, "big", True, m.x.z) + assert m.x.band.dependencies == {m.x.hot.id, m.x.z.id} + + def test_a_tracked_operand_lifts_per_track(self): + m = _tracked() + with m: + m.s = IFS(m.x > 4.0, "hi", True, "lo") + assert dict(m.s._value.items()) == {"plan": ["lo", "#DIV/0!"], "actual": ["hi", "hi"]} + assert m.s.var_type == "list" and m.s.value_type == "string" + + +# ══════════════════════════════════════════════════════════════════════ +# SWITCH +# ══════════════════════════════════════════════════════════════════════ + + +class TestSwitch: + def test_the_value_of_the_first_case_equal_to_the_expression(self): + m = _model() + out = SWITCH(m.x.a, 3, "three", 8, "eight", "other") + assert out.value == ["three", "eight", "other"] and out.var_type == "list" + assert SWITCH(2, 1, "a", 2, "b", "c").value == "b" + assert SWITCH(2, 1, "a", 2, "b", "c").var_type == "scalar" + + def test_no_case_matching_is_the_default_or_na(self): + m = _model() + assert SWITCH(m.x.a, 3, "three").value == ["three", "#N/A", "#N/A"] + assert SWITCH(9, 1, "one").value == "#N/A" + assert SWITCH(9, 1, "one", "none").value == "none" + + def test_numbers_are_equal_as_the_spreadsheet_compares_them(self): + # To within 2**-48 of either — about 15 significant digits — so a + # computed 0.1 * 3 is the 0.3 typed into a case. + f = mo.Variable([0.1, 0.2, 0.3]) + assert (f * 3).value != [0.3, 0.6, 0.9] + assert SWITCH(f * 3, 0.3, "p3", 0.6, "p6", 0.9, "p9", "other").value == \ + ["p3", "p6", "p9"] + assert SWITCH(f + 0.2, 0.3, "a", 0.4, "b", 0.5, "c", "other").value == ["a", "b", "c"] + assert SWITCH(1 + 2**-50, 1, "eq", "ne").value == "eq" + assert SWITCH(1 + 2**-48, 1, "eq", "ne").value == "ne" + assert SWITCH(123456789012345, 123456789012345.02, "eq", "ne").value == "eq" + assert SWITCH(-0.1 * 3, -0.3, "eq", "ne").value == "eq" + + def test_zero_equals_only_zero_and_whole_numbers_only_themselves(self): + assert SWITCH(1e-300, 0, "eq", "ne").value == "ne" + assert SWITCH(2**52, 2**52 + 1, "eq", "ne").value == "ne" + assert SWITCH(0.0, -0.0, "eq", "ne").value == "eq" + + def test_text_matches_regardless_of_case(self): + names = mo.Variable(["Base", "bull", "x"]) + assert SWITCH(names, "base", 0.05, "BULL", 0.08, 0.03).value == [0.05, 0.08, 0.03] + + def test_never_across_kinds(self): + m = _model() + # 3 is not "3"; TRUE is not 1; a text is not a number. + assert SWITCH(m.x.a, "3", "txt", "num").value == ["num"] * 3 + assert SWITCH(mo.Variable(["3", "8"]), 3, "num", "txt").value == ["txt"] * 2 + assert SWITCH(mo.Variable([True, 1]), 1, "one", True, "true").value == ["true", "one"] + assert SWITCH(mo.Variable([False, 0]), 0, "zero", False, "false").value == \ + ["false", "zero"] + + def test_a_date_by_its_serial_number(self): + d = mo.Variable(dt.date(2026, 1, 1)) + assert SWITCH(d, 46023, "yes", "no").value == "yes" + assert SWITCH(d, dt.date(2026, 1, 1), "same", "no").value == "same" + assert SWITCH(46023, dt.date(2026, 1, 1), "same", "no").value == "same" + assert SWITCH(d, dt.date(2026, 1, 2), "next", "no").value == "no" + + def test_a_blank_is_zero_empty_text_and_false(self): + # As the engine's ``==`` reads a blank cell against each kind; an + # empty text typed into a line is a blank cell. + blank = mo.Variable([None, 0, ""]) + assert SWITCH(blank, 0, "zero", "other").value == ["zero", "zero", "zero"] + assert SWITCH(blank, "", "empty", "other").value == ["empty", "other", "empty"] + assert SWITCH(blank, False, "false", "other").value == ["false", "other", "false"] + assert SWITCH(mo.Variable([0, 1]), None, "blank", "other").value == ["blank", "other"] + + def test_a_formula_s_empty_text_is_text(self): + m = _model() + label = _empty_text_formula(m) + assert SWITCH(label, 0, "zero", "other").value == ["other"] * 3 + assert SWITCH(label, "", "empty", "other").value == ["empty"] * 3 + + def test_an_expression_or_a_reached_case_that_is_an_error_is_the_result(self): + m = _model() + assert SWITCH(m.x.q, 3, "three", "other").value == ["three", "#DIV/0!", "other"] + # The case after the match is never read: period one matches 3 + # before q is compared; period two reaches q's error. + assert SWITCH(m.x.a, 3, "three", m.x.q, "q", "other").value == \ + ["three", "#DIV/0!", "other"] + assert SWITCH(5, 5, "#N/A", "other").value == "#N/A" + + def test_a_picked_blank_shows_zero_and_does_not_type_the_result(self): + m = _model() + blank = mo.Variable([None, 5.0, None]) + out = SWITCH(m.x.a, 3, blank, 8, blank, "other") + assert out.value == [0, 5.0, "other"] and out.value_type == "float" + assert SWITCH(9, 1, "one", None).value == 0 + + def test_a_scalar_expression_over_series_values_gives_a_series(self): + m = _model() + out = SWITCH(2, 1, m.x.a, 2, m.x.z) + assert out.value == [1.0, 0.0, 2.0] and out.var_type == "list" + assert SWITCH(mo.Variable(1), 1, m.x.a, 2, m.x.z).value == [3.0, 8.0, 5.0] + + def test_an_expression_and_at_least_one_pair(self): + m = _model() + with pytest.raises(TypeError, match="case and value pairs"): + SWITCH(m.x.a) + with pytest.raises(TypeError, match="got 1 operand after the expression"): + SWITCH(m.x.a, 1) + + def test_typed_by_the_branch_taken(self): + pick = mo.Variable([1, 2, 3]) + assert SWITCH(pick, 1, "one", 2, "two", "many").value_type == "string" + assert SWITCH(pick, 1, 10, 2, 20, 0).value_type == "int" + assert SWITCH(pick, 1, dt.date(2026, 1, 1), dt.date(2027, 1, 1)).value_type == \ + "datetime" + + def test_the_unit_the_values_share(self): + pick = mo.Variable([1, 2, 3]) + base = mo.Variable([3.0, 8.0, 5.0], unit="$") + bull = mo.Variable([4.0, 9.0, 6.0], unit="$") + out = SWITCH(pick, 1, base, 2, bull, 0.0) + assert out.value == [3.0, 9.0, 0.0] + assert out.value_type == "float" and str(out._unit) == "$" + assert SWITCH(pick, 1, "one", 2, "two", "many")._unit is None + assert SWITCH(base, 3.0, "x", "y")._unit is None + hours = mo.Variable([1.0, 1.0, 1.0], unit="hr") + with pytest.raises(ValueError, match="mo.SWITCH takes its values in one unit"): + SWITCH(pick, 1, base, 2, hours, 0.0) + # The expression and the cases are compared, never taken: their + # unit is not the result's. + shift = mo.Variable([8.0, 1.0, 8.0], unit="hr") + out = SWITCH(shift, 8.0, base, 0.0) + assert out.value == [3.0, 0.0, 5.0] and str(out._unit) == "$" + assert SWITCH(shift, 8.0, 1.0, 0.0)._unit is None + + def test_a_case_in_another_unit_than_the_expression_is_refused(self): + # Compared as == compares them: hours against a threshold in USD + # is refused, a plain number or a case in hours goes. + shift = mo.Variable([8.0, 1.0, 8.0], unit="hr", display_name="Shift") + full = mo.Variable(8.0, unit="hr", display_name="Full shift") + limit = mo.Variable(8.0, unit="USD", display_name="Limit") + assert SWITCH(shift, full, 1.0, 0.0).value == [1.0, 0.0, 1.0] + assert SWITCH(shift, 8.0, 1.0, 0.0).value == [1.0, 0.0, 1.0] + with pytest.raises(ValueError, match="mo.SWITCH compares 'Shift' with the case " + "'Limit'; they are in hr and USD"): + SWITCH(shift, limit, 1.0, 0.0) + with pytest.raises(ValueError, match="compares 'Shift' with the case 'Limit'"): + SWITCH(shift, full, 1.0, limit, 2.0, 0.0) # the second case too + + def test_the_formula_is_the_source_and_the_ast_is_a_call(self): + m = _model() + out = SWITCH(m.x.a, 3, "three", 8, "eight", "other") + assert out.formula == "SWITCH(a, 3, 'three', 8, 'eight', 'other')" + assert out._source_code == out.formula + call = out._expr + assert isinstance(call, FuncCall) and call.func == "SWITCH" and len(call.args) == 6 + assert isinstance(call.args[0], VarRef) and isinstance(call.args[1], Literal) + with m: + m.x.pick = SWITCH(m.x.z, 0, m.x.a, 1, m.x.q, -1.0) + assert m.x.pick.dependencies == {m.x.z.id, m.x.a.id, m.x.q.id} + + def test_a_tracked_operand_lifts_per_track(self): + m = _tracked() + with m: + m.w = SWITCH(m.x, 3.0, "three", 8.0, "eight", "other") + assert dict(m.w._value.items()) == {"plan": ["three", "#DIV/0!"], + "actual": ["other", "eight"]} + + +# ══════════════════════════════════════════════════════════════════════ +# One call per period — the shared helper's law +# ══════════════════════════════════════════════════════════════════════ + + +class TestOneCallPerPeriod: + def test_scalars_and_a_one_value_series_broadcast(self): + assert IFERROR(mo.Variable(["#N/A"]), mo.Variable([1.0, 2.0])).value == [1.0, 2.0] + assert IFS(mo.Variable([True]), mo.Variable([1.0, 2.0]), True, 0.0).value == [1.0, 2.0] + assert SWITCH(mo.Variable([2]), 1, "a", mo.Variable([2, 2]), "b").value == ["b", "b"] + assert XOR(mo.Variable([True]), mo.Variable([True, False])).value == [False, True] + + def test_series_of_different_lengths_are_refused(self): + a = mo.Variable([1.0, 2.0, 3.0]) + with pytest.raises(ValueError, match="different lengths"): + IFERROR(a, mo.Variable([1.0, 2.0])) + with pytest.raises(ValueError, match="different lengths"): + IFS(a > 1.0, mo.Variable([1.0, 2.0]), True, 0.0) + with pytest.raises(ValueError, match="different lengths"): + SWITCH(a, 1.0, mo.Variable([1.0, 2.0]), "other") + with pytest.raises(ValueError, match="different lengths"): + XOR(a > 1.0, mo.Variable([True, False])) + + @pytest.mark.parametrize("name", sorted(FUNCTIONS)) + def test_an_axised_operand_is_refused(self, name): + region = mo.Variable(["N", "S"], display_name="region") + by_region = mo.Variable([1.0, 2.0], indexed_by=[region]) + args = { + "IFERROR": (by_region, 0.0), "ISERROR": (by_region,), "ISNUMBER": (by_region,), + "XOR": (by_region, True), "IFS": (True, by_region), "SWITCH": (by_region, 1.0, "x"), + }[name] + with pytest.raises(ValueError, match=name): + FUNCTIONS[name](*args) + + +# ══════════════════════════════════════════════════════════════════════ +# The registry and the loop +# ══════════════════════════════════════════════════════════════════════ + + +class TestTheRegistry: + @pytest.mark.parametrize("name", sorted(FUNCTIONS)) + def test_each_is_registered_as_one_call_per_period(self, name): + assert spec(name) is not None and not spec(name).bare + assert is_elementwise(name) and is_passthrough(name) and not is_range_taking(name) + assert admits_in_loop(name, 2) and admits_in_loop(name, 6) + assert excel_name(name) == name + + def test_the_names_excel_added_after_2007_carry_the_file_prefix(self): + assert {n: file_prefix(n) for n in FUNCTIONS} == { + "IFERROR": "", "ISERROR": "", "ISNUMBER": "", + "XOR": "_xlfn.", "IFS": "_xlfn.", "SWITCH": "_xlfn.", + } + + +class TestInsideALoop: + def test_a_cap_written_as_ifs_runs_period_by_period(self): + # The loop re-runs a call through the engine's exported function + # of that name. + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with m: + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + d.bal = mo.lag(d.r, fill=10.0) * 1.37 + d.r = IFS(d.bal > 15.0, 15.0, True, d.bal) + assert d.bal.value == pytest.approx([13.7, 18.769, 20.55]) + assert d.r.value == pytest.approx([13.7, 15.0, 15.0]) + + def test_an_error_caught_inside_the_loop(self): + # Inside the loop the call is run again over plain values: the + # marker of a division by zero reaches IFERROR as text. + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with m: + m.i = mo.MultiVariable("I") + m.i.k = mo.Variable([2.0, 0.0, 4.0]) + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + d.bal = mo.lag(d.r, fill=8.0) + 1.0 + d.r = IFERROR(d.bal / m.i.k, 1.0) + assert d.bal.value == [9.0, 5.5, 2.0] + assert d.r.value == [4.5, 1.0, 0.5] diff --git a/tests/test_lookup.py b/tests/test_lookup.py new file mode 100644 index 0000000..cf4da01 --- /dev/null +++ b/tests/test_lookup.py @@ -0,0 +1,733 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Lookup and reference: a key found in a range or a block as the +spreadsheet finds it, one cell with its formula — a series when the +key is one.""" + +import datetime as dt + +import pytest + +import modeleon as mo +from modeleon.core.expr import FuncCall, ListExpr, Literal, VarRef +from modeleon.functions.lookup import ( + HLOOKUP, + INDEX, + MATCH, + VLOOKUP, + XLOOKUP, + _exact, + _nearest, + _position, + _sorted_match, +) +from modeleon.functions.registry import ( + admits_in_loop, + file_prefix, + is_passthrough, + is_range_taking, + range_positions, + spec, +) + +# ══════════════════════════════════════════════════════════════════════ +# One cell against the key +# ══════════════════════════════════════════════════════════════════════ + + +class TestTheMatchingCore: + @pytest.mark.parametrize("cells, key, wildcard, at", [ + ([10, 20, 30], 20, False, 1), + ([10, 20, 30], 25, False, None), + ([10, "20", 30], 20, False, None), # a number never matches text + (["10", 20], "10", False, 0), + (["North", "south"], "SOUTH", False, 1), # text, case-insensitively + (["Northern", "North"], "North", False, 1), # the whole cell + (["Northern", "North"], "North*", True, 0), # wildcards where Excel reads them + (["Northern", "North"], "North*", False, None), + (["a~b", "ab"], "a~b", True, 1), + (["a~b", "ab"], "a~b", False, 0), + ([True, 1], True, False, 0), # a boolean matches a boolean only + ([1, True], True, False, 1), + ([None, 0, ""], None, False, None), # a blank matches nothing + ([None, 0], 0, False, 1), + ([None, ""], "", False, 1), + (["#N/A", 5], 5, False, 1), # an error among the cells is skipped + (["#N/A", 5], "#N/A", False, None), + ([dt.date(2026, 1, 1), 46023], dt.date(2026, 1, 1), False, 0), + ([46023, dt.date(2026, 1, 1)], dt.date(2026, 1, 1), False, 0), # a date is its serial + ([dt.datetime(2026, 1, 1, 12)], dt.date(2026, 1, 1), False, None), + ]) + def test_an_exact_match(self, cells, key, wildcard, at): + assert _exact(cells, key, wildcard=wildcard) == at + + def test_the_first_or_the_last_among_equals(self): + assert _exact([2, 1, 2], 2, wildcard=False) == 0 + assert _exact([2, 1, 2], 2, wildcard=False, last=True) == 2 + + @pytest.mark.parametrize("cells, key, below, last, at", [ + ([10, 30, 20], 25, True, False, 2), # the largest not above, in any order + ([10, 30, 20], 25, False, False, 1), # the smallest not below + ([10, 30, 20], 20, True, False, 2), # an equal cell is nearest of all + ([10, 30, 20], 5, True, False, None), + ([10, 30, 20], 35, False, False, None), + ([20, 10, 20], 25, True, False, 0), # ties: the first, or the last + ([20, 10, 20], 25, True, True, 2), + (["b", "d"], "C", True, False, 0), # text orders case-insensitively + (["b", "d"], "C", False, False, 1), + ([10, "x", None, "#N/A", 20], 15, True, False, 0), # another kind, a blank, an error + ]) + def test_the_nearest_on_one_side(self, cells, key, below, last, at): + assert _nearest(cells, key, below=below, last=last) == at + + @pytest.mark.parametrize("cells, key, ascending, at", [ + ([10, 20, 30], 25, True, 1), + ([10, 20, 30], 30, True, 2), + ([10, 20, 20, 30], 20, True, 2), # the last among equals, as Excel finds it + ([10, 20, 30], 5, True, None), + ([30, 20, 10], 25, False, 0), + ([30, 20, 10], 15, False, 1), + ([30, 20, 10], 35, False, None), + ([10, "x", 30], 25, True, 0), # another kind is skipped + (["a", "c", "e"], "d", True, 1), + ]) + def test_over_sorted_cells(self, cells, key, ascending, at): + assert _sorted_match(cells, key, ascending=ascending) == at + + @pytest.mark.parametrize("n, size, at", [ + (1, 3, 0), (3, 3, 2), (2.9, 3, 1), + (0, 3, "#VALUE!"), (-1, 3, "#VALUE!"), (None, 3, "#VALUE!"), + ("2", 3, 1), ("2.9", 3, 1), ("two", 3, "#VALUE!"), ("", 3, "#VALUE!"), + (True, 3, 0), (False, 3, "#VALUE!"), # a boolean is 1 or 0 + (dt.date(2026, 1, 1), 3, "#REF!"), # a date is its serial + (float("inf"), 3, "#VALUE!"), ("1e999", 3, "#VALUE!"), ("1" * 400, 3, "#REF!"), + (4, 3, "#REF!"), ("#DIV/0!", 3, "#DIV/0!"), + ]) + def test_a_position(self, n, size, at): + assert _position(n, size) == at + + +# ══════════════════════════════════════════════════════════════════════ +# The functions +# ══════════════════════════════════════════════════════════════════════ + + +class Deal(mo.MultiVariableClass): + def compute(self, region, amount): + self.region = mo.Variable(region, display_name="Region") + self.amount = mo.Variable(amount, display_name="Amount", unit="USD") + self.flow = mo.Variable([amount * 0.1, amount * 0.2, amount * 0.3], display_name="Flow") + + +def _model(): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=3) + with m: + m.i = mo.MultiVariable("I") + m.i.years = mo.Variable([2026, 2027, 2028], display_name="years") + m.i.rates = mo.Variable([0.05, 0.04, 0.03], display_name="rates", unit="USD") + m.i.names = mo.Variable(["Alpha", "Beta", "Gamma"], display_name="names") + m.i.infl = mo.Variable({2026: 0.05, 2027: 0.04, 2028: 0.03}, display_name="infl") + m.i.year = mo.Variable([2026, 2028, 2030], display_name="year") + m.i.pick = mo.Variable(2027, display_name="pick") + m.deals = mo.MultiVariable("Deals", excel_props={"tab": True, "orient": "records"}) + with m.deals as d: + d.a = Deal(region="North", amount=100.0, display_name="Alpha") + d.b = Deal(region="South", amount=200.0, display_name="Beta") + d.c = Deal(region="North", amount=300.0, display_name="Gamma") + return m + + +class TestMATCH: + def test_exact_sorted_ascending_and_descending(self): + m = _model() + assert MATCH(2027, m.i.years, 0).value == 2 + assert MATCH(2027.5, m.i.years).value == 2 + assert MATCH(2027.5, m.i.years, 1).value == 2 + assert MATCH(2025, m.i.years).value == "#N/A" + desc = mo.Variable([30, 20, 10]) + assert MATCH(25, desc, -1).value == 1 + assert MATCH(15, desc, -1).value == 2 + assert MATCH(35, desc, -1).value == "#N/A" + + def test_text_case_insensitively_with_wildcards_and_never_a_number(self): + m = _model() + assert MATCH("beta", m.i.names, 0).value == 2 + assert MATCH("g*", m.i.names, 0).value == 3 + assert MATCH("2027", m.i.years, 0).value == "#N/A" + assert MATCH(2027, mo.Variable(["2026", "2027"]), 0).value == "#N/A" + + def test_a_blank_key_an_error_key_and_an_error_among_the_cells(self): + m = _model() + assert MATCH(mo.Variable(None), m.i.years, 0).value == "#N/A" + assert MATCH(mo.Variable([2027, None]), m.i.years, 0).value == [2, "#N/A"] + assert MATCH("#DIV/0!", m.i.years, 0).value == "#DIV/0!" + assert MATCH(2028, mo.Variable(["#N/A", 2027, 2028]), 0).value == 3 + + def test_over_a_list_of_lines_and_a_keyed_line_s_cells(self): + m = _model() + d = m.deals + assert MATCH("Beta", [d.a.region, d.b.region, d.c.region], 0).value == "#N/A" + assert MATCH("South", [d.a.region, d.b.region, d.c.region], 0).value == 2 + assert MATCH(0.04, m.i.infl, 0).value == 2 + + def test_a_series_key_is_one_lookup_per_period(self): + m = _model() + out = MATCH(m.i.year, m.i.years, 0) + assert out.value == [1, 3, "#N/A"] and out.var_type == "list" + assert MATCH([2028, 2026], m.i.years, 0).value == [3, 1] + + def test_a_position_is_an_int_without_a_unit(self): + m = _model() + out = MATCH(0.04, m.i.rates, 0) + assert out.value == 2 and out.value_type == "int" and out._unit is None + assert out.var_type == "scalar" + + def test_the_formula_and_the_source(self): + m = _model() + out = MATCH(m.i.pick, m.i.years, 0) + assert isinstance(out._expr, FuncCall) and out._expr.func == "MATCH" + assert [type(a) for a in out._expr.args] == [VarRef, VarRef, Literal] + assert out.formula == "MATCH(pick, years, 0)" + assert out._source_code == "MATCH(pick, years, 0)" + assert MATCH(m.i.pick, m.i.years).formula == "MATCH(pick, years)" + + def test_the_mode_is_typed_in(self): + m = _model() + with pytest.raises(ValueError, match="match_type is one of 1, 0, -1"): + MATCH(2027, m.i.years, 2) + with pytest.raises(ValueError, match="match_type"): + MATCH(2027, m.i.years, m.i.pick) + + def test_none_typed_as_the_key_is_refused(self): + # It has no cell: the formula would write it as 0 and look 0 up. + m = _model() + with pytest.raises(ValueError, match="the key is None, and None has no cell"): + MATCH(None, m.i.years, 0) + with pytest.raises(ValueError, match=r"the key \[2027, None\] holds None"): + MATCH([2027, None], m.i.years, 0) + for call in (lambda: XLOOKUP(None, m.i.years, m.i.rates), + lambda: XLOOKUP(None, m.i.infl), + lambda: VLOOKUP(None, m.deals, 3, False), + lambda: HLOOKUP(None, [m.i.years, m.i.rates], 2, False)): + with pytest.raises(ValueError, match="a blank cell finds nothing"): + call() + + +class TestINDEX: + def test_a_position_in_a_row_and_in_a_list_of_lines(self): + m = _model() + d = m.deals + assert INDEX(m.i.rates, 2).value == 0.04 + assert INDEX([d.a.amount, d.b.amount, d.c.amount], 3).value == 300.0 + assert INDEX(m.i.rates, 2.9).value == 0.04 + + @pytest.mark.parametrize("k, result", [ + (0, "#VALUE!"), (-1, "#VALUE!"), (None, "#VALUE!"), ("two", "#VALUE!"), + (False, "#VALUE!"), (4, "#REF!"), ("#N/A", "#N/A"), + ]) + def test_a_position_the_range_does_not_have(self, k, result): + assert INDEX(_model().i.rates, k).value == result + + def test_a_position_is_read_as_the_spreadsheet_reads_a_number(self): + from modeleon.core.unit import unit_key + m = _model() + assert INDEX(m.i.rates, "2").value == 0.04 + assert INDEX(m.i.rates, True).value == 0.05 + assert INDEX(m.i.rates, mo.Variable("3")).value == 0.03 + assert INDEX(m.deals, "2", "3").value == 200.0 + assert VLOOKUP("Beta", m.deals, "3", False).value == 200.0 + assert HLOOKUP(2027, [m.i.years, m.i.rates], "2", False).value == 0.04 + # Typed by the column the text names, and in that column's unit. + out = VLOOKUP("Beta", m.deals, "3", False) + assert out.value_type == "float" + assert unit_key(out._unit) == unit_key(m.deals.a.amount._unit) + + def test_the_cell_found_carries_its_own_content(self): + cells = mo.Variable([1.0, None, "#DIV/0!", "x"]) + assert INDEX(cells, 2).value == 0 + assert INDEX(cells, 3).value == "#DIV/0!" + assert INDEX(cells, 4).value == "x" + + def test_a_series_position_is_one_pick_per_period(self): + m = _model() + out = INDEX(m.i.rates, mo.Variable([1, 3, 5])) + assert out.value == [0.05, 0.03, "#REF!"] and out.var_type == "list" + + def test_type_and_unit_come_from_the_range(self): + from modeleon.core.unit import unit_key + m = _model() + out = INDEX(m.i.rates, 1) + assert out.value_type == "float" and unit_key(out._unit) == unit_key(m.i.rates._unit) + assert INDEX(m.i.years, 1).value_type == "int" + assert INDEX(m.i.names, 1).value_type == "string" + + def test_over_a_table_of_records_the_label_column_first(self): + from modeleon.core.unit import unit_key + m = _model() + assert INDEX(m.deals, 2, 1).value == "Beta" + assert INDEX(m.deals, 2, 2).value == "South" + assert INDEX(m.deals, 3, 3).value == 300.0 + assert INDEX(m.deals, 4, 3).value == "#REF!" + assert INDEX(m.deals, 1, 4).value == "#REF!" + assert INDEX(m.deals, 0, 1).value == "#VALUE!" + assert INDEX(m.deals, 2, 1).value_type == "string" + amount = INDEX(m.deals, 2, 3) + assert amount.value_type == "float" + assert unit_key(amount._unit) == unit_key(m.deals.a.amount._unit) + + def test_over_a_list_of_rows(self): + m = _model() + assert INDEX([m.i.years, m.i.rates], 2, 3).value == 0.03 + assert INDEX([m.i.years, m.i.rates], 1, 3).value == 2028 + out = INDEX([m.i.years, m.i.rates], 2, mo.Variable([1, 2, 3])) + assert out.value == [0.05, 0.04, 0.03] and out.var_type == "list" + assert INDEX([m.i.years, m.i.rates], 2, 1).value_type == "float" + assert INDEX([[1, 2], [3, 4]], 2, 1).value == 3 + + def test_the_formula_names_the_block(self): + m = _model() + out = INDEX(m.deals, 2, 3) + assert [type(a) for a in out._expr.args] == [ListExpr, Literal, Literal] + assert all(isinstance(row, ListExpr) for row in out._expr.args[0].items) + assert out._source_code == "INDEX(deals, 2, 3)" + rows = INDEX([m.i.years, m.i.rates], 2, 3) + assert [type(a) for a in rows._expr.args[0].items] == [VarRef, VarRef] + assert rows._source_code == "INDEX([i.years, i.rates], 2, 3)" + + def test_one_position_over_a_block_and_two_over_a_range_are_refused(self): + m = _model() + with pytest.raises(ValueError, match="a row and a column number"): + INDEX(m.deals, 2) + with pytest.raises(ValueError, match="one position"): + INDEX(m.i.rates, 1, 2) + + +class TestXLOOKUP: + def test_exact_by_default(self): + m = _model() + assert XLOOKUP(2027, m.i.years, m.i.rates).value == 0.04 + assert XLOOKUP("beta", m.i.names, m.i.rates).value == 0.04 + assert XLOOKUP(2030, m.i.years, m.i.rates).value == "#N/A" + assert XLOOKUP("B*", m.i.names, m.i.rates).value == "#N/A" + assert XLOOKUP("B*", m.i.names, m.i.rates, match_mode=2).value == 0.04 + assert XLOOKUP(dt.date(2026, 1, 1), mo.Variable([dt.date(2026, 1, 1)]), + mo.Variable([7.0])).value == 7.0 + + def test_if_not_found_a_value_or_a_cell(self): + m = _model() + assert XLOOKUP(2030, m.i.years, m.i.rates, 0).value == 0 + assert XLOOKUP(2030, m.i.years, m.i.rates, "none").value == "none" + assert XLOOKUP(2030, m.i.years, m.i.rates, m.i.pick).value == 2027 + assert XLOOKUP(2027, m.i.years, m.i.rates, 0).value == 0.04 + + def test_the_nearest_below_or_above_in_any_order(self): + keys = mo.Variable([10, 30, 20]) + vals = mo.Variable(["a", "b", "c"]) + assert XLOOKUP(25, keys, vals, match_mode=-1).value == "c" + assert XLOOKUP(25, keys, vals, match_mode=1).value == "b" + assert XLOOKUP(5, keys, vals, match_mode=-1).value == "#N/A" + assert XLOOKUP(5, keys, vals, 0, -1).value == 0 + dup = mo.Variable([20, 10, 20]) + assert XLOOKUP(20, dup, vals).value == "a" + assert XLOOKUP(20, dup, vals, search_mode=-1).value == "c" + assert XLOOKUP(25, dup, vals, match_mode=-1, search_mode=-1).value == "c" + + def test_a_series_key_is_one_lookup_per_period(self): + m = _model() + out = XLOOKUP(m.i.year, m.i.years, m.i.rates, 0) + assert out.value == [0.05, 0.03, 0] and out.var_type == "list" + assert XLOOKUP(m.i.year, m.i.years, m.i.rates).value == [0.05, 0.03, "#N/A"] + + def test_over_a_keyed_line_the_keys_are_searched(self): + m = _model() + assert XLOOKUP(2027, m.i.infl).value == 0.04 + assert XLOOKUP(2029, m.i.infl).value == "#N/A" + assert XLOOKUP(2029, m.i.infl, if_not_found=0).value == 0 + assert XLOOKUP(2029, m.i.infl, match_mode=-1).value == 0.03 + out = XLOOKUP(m.i.year, m.i.infl, if_not_found=0) + assert out.value == [0.05, 0.03, 0] + fy = mo.Variable({"FY2023": 400, "FY2024": 450}) + assert XLOOKUP("fy2024", fy).value == 450 + with pytest.raises(ValueError, match="runs along keys"): + XLOOKUP(2027, m.i.rates) + + def test_the_keyed_form_types_the_keys_into_the_call(self): + m = _model() + out = XLOOKUP(m.i.pick, m.i.infl) + assert [type(a) for a in out._expr.args] == [VarRef, Literal, VarRef] + assert out._expr.args[1].value == [2026, 2027, 2028] + assert out.formula == "XLOOKUP(pick, [2026, 2027, 2028], infl)" + assert out._source_code == "XLOOKUP(pick, [2026, 2027, 2028], infl)" + + def test_a_mode_without_if_not_found_is_written_after_na(self): + m = _model() + out = XLOOKUP(m.i.pick, m.i.years, m.i.rates, match_mode=-1) + assert out.formula == "XLOOKUP(pick, years, rates, NA(), -1)" + assert out._source_code == "XLOOKUP(pick, years, rates, match_mode=-1)" + out = XLOOKUP(m.i.pick, m.i.years, m.i.rates, search_mode=-1) + assert out.formula == "XLOOKUP(pick, years, rates, NA(), 0, -1)" + assert out._source_code == "XLOOKUP(pick, years, rates, match_mode=0, search_mode=-1)" + out = XLOOKUP(m.i.pick, m.i.years, m.i.rates, 0, 1, -1) + assert out.formula == "XLOOKUP(pick, years, rates, 0, 1, -1)" + assert XLOOKUP(m.i.pick, m.i.years, m.i.rates).formula == "XLOOKUP(pick, years, rates)" + + def test_type_and_unit_come_from_the_returned_range_never_the_keys(self): + from modeleon.core.unit import unit_key + m = _model() + out = XLOOKUP(2027, m.i.years, m.i.rates) + assert out.value_type == "float" and unit_key(out._unit) == unit_key(m.i.rates._unit) + back = XLOOKUP(0.04, m.i.rates, m.i.years) + assert back.value_type == "int" and back._unit is None + assert XLOOKUP(2027, m.i.infl).value_type == "float" + + def test_refusals(self): + m = _model() + with pytest.raises(ValueError, match="same number of cells"): + XLOOKUP(2027, m.i.years, mo.Variable([1, 2])) + with pytest.raises(ValueError, match="match_mode is one of"): + XLOOKUP(2027, m.i.years, m.i.rates, match_mode=3) + with pytest.raises(ValueError, match="search_mode is one of 1, -1"): + XLOOKUP(2027, m.i.years, m.i.rates, search_mode=0) + for binary in (2, -2): + with pytest.raises(ValueError, match="binary search .* Use 1 for the first"): + XLOOKUP(2027, m.i.years, m.i.rates, search_mode=binary) + with pytest.raises(ValueError, match="search_mode is one of 1, -1, typed in"): + XLOOKUP(2027, m.i.years, m.i.rates, search_mode=mo.Variable(2)) + with pytest.raises(TypeError, match="a key is one value"): + XLOOKUP({2027}, m.i.years, m.i.rates) + + +class TestVLOOKUP: + def test_over_a_table_of_records_by_label(self): + m = _model() + assert VLOOKUP("Beta", m.deals, 3, False).value == 200.0 + assert VLOOKUP("beta", m.deals, 2, False).value == "South" + assert VLOOKUP("Beta", m.deals, 1, False).value == "Beta" + assert VLOOKUP("G*", m.deals, 3, False).value == 300.0 + assert VLOOKUP("Delta", m.deals, 3, False).value == "#N/A" + + def test_approximate_by_default_over_a_sorted_first_column(self): + m = _model() + assert VLOOKUP("Bob", m.deals, 3).value == 200.0 + assert VLOOKUP("Bob", m.deals, 3, True).value == 200.0 + assert VLOOKUP("Aardvark", m.deals, 3).value == "#N/A" + assert VLOOKUP("Bob", m.deals, 3, False).value == "#N/A" + + def test_the_column_number(self): + m = _model() + assert VLOOKUP("Beta", m.deals, 0, False).value == "#VALUE!" + assert VLOOKUP("Beta", m.deals, 4, False).value == "#REF!" + assert VLOOKUP("Beta", m.deals, "#N/A", False).value == "#N/A" + # Checked before the key is searched: its error wins, found or not. + assert VLOOKUP("Delta", m.deals, 4, False).value == "#REF!" + assert VLOOKUP("Delta", m.deals, 0, False).value == "#VALUE!" + assert VLOOKUP("#DIV/0!", m.deals, 4, False).value == "#REF!" + assert VLOOKUP("#DIV/0!", m.deals, 0, False).value == "#VALUE!" + assert VLOOKUP("#DIV/0!", m.deals, 3, False).value == "#DIV/0!" + out = VLOOKUP("Beta", m.deals, mo.Variable([1, 2, 3]), False) + assert out.value == ["Beta", "South", 200.0] and out.var_type == "list" + + def test_a_series_key_a_blank_key_and_an_error_key(self): + m = _model() + keys = mo.Variable(["Alpha", "Delta", "Gamma"]) + assert VLOOKUP(keys, m.deals, 3, False).value == [100.0, "#N/A", 300.0] + assert VLOOKUP(mo.Variable(None), m.deals, 3, False).value == "#N/A" + assert VLOOKUP("#DIV/0!", m.deals, 3, False).value == "#DIV/0!" + + def test_over_a_list_of_rows(self): + m = _model() + # The first column of a list of rows is the rows' first cells. + assert VLOOKUP(2026, [m.i.years, m.i.rates], 3, False).value == 2028 + assert VLOOKUP(0.05, [m.i.years, m.i.rates], 2, False).value == 0.04 + assert VLOOKUP(2027, [m.i.years, m.i.rates], 1, False).value == "#N/A" + + def test_type_and_unit_come_from_the_column(self): + from modeleon.core.unit import unit_key + m = _model() + assert VLOOKUP("Beta", m.deals, 1, False).value_type == "string" + assert VLOOKUP("Beta", m.deals, 2, False).value_type == "string" + amount = VLOOKUP("Beta", m.deals, 3, False) + assert amount.value_type == "float" + assert unit_key(amount._unit) == unit_key(m.deals.a.amount._unit) + # A position read from a cell picks the column by the cell's value. + assert unit_key(VLOOKUP("Beta", m.deals, mo.Variable(3), False)._unit) == unit_key( + m.deals.a.amount._unit) + assert VLOOKUP("Beta", m.deals, mo.Variable(2), False).value_type == "string" + + def test_a_record_that_lacks_a_field_leaves_the_cell_blank(self): + class Custom(mo.MultiVariableClass): + def compute(self, salary, bonus=None): + self.salary = mo.Variable(salary, display_name="Salary") + if bonus is not None: + self.bonus = mo.Variable(bonus, display_name="Bonus") + + m = mo.Model("m") + with m: + m.staff = mo.MultiVariable("Staff", excel_props={"tab": True, "orient": "records"}) + with m.staff as s: + s.a = Custom(salary=1, display_name="A") + s.b = Custom(salary=2, bonus=9, display_name="B") + assert VLOOKUP("A", m.staff, 3, False).value == 0 + assert VLOOKUP("B", m.staff, 3, False).value == 9 + assert INDEX(m.staff, 1, 3).value == 0 + + def test_the_formula_and_the_flag(self): + m = _model() + out = VLOOKUP("Beta", m.deals, 3, False) + assert [type(a) for a in out._expr.args] == [Literal, ListExpr, Literal, Literal] + assert out._expr.args[3].value is False + assert out._source_code == "VLOOKUP('Beta', deals, 3, False)" + assert VLOOKUP("Beta", m.deals, 3)._source_code == "VLOOKUP('Beta', deals, 3)" + assert VLOOKUP("Bob", m.deals, 3, 0).value == "#N/A" + + def test_refusals(self): + m = _model() + with pytest.raises(ValueError, match="range_lookup is TRUE or FALSE"): + VLOOKUP("Beta", m.deals, 3, m.i.pick) + with pytest.raises(ValueError, match="holds no records"): + VLOOKUP("x", m.i, 1) + with pytest.raises(TypeError, match="a block is"): + VLOOKUP("x", m.i.rates, 1) + with pytest.raises(ValueError, match="a block is a list of rows"): + INDEX([m.i.years, m.i.pick], 1, 1) + with pytest.raises(ValueError, match="same number of cells"): + INDEX([m.i.years, mo.Variable([1, 2])], 1, 1) + with pytest.raises(ValueError, match="no scalar fields"): + class Only(mo.MultiVariableClass): + def compute(self): + self.flow = mo.Variable([1.0, 2.0, 3.0]) + t = mo.MultiVariable("T", excel_props={"orient": "records"}) + t.a = Only() + VLOOKUP("a", t, 1) + + +class TestAResultFromDifferentLines: + """A result that can come from cells of different lines is typed by them + when they agree, else by the value found; given a unit only when every one + of them carries the same unit.""" + + def test_the_cells_agree(self): + from modeleon.core.unit import unit_key + m = _model() + amount = INDEX(m.deals, mo.Variable([1, 2, 3]), 3) + assert amount.value_type == "float" + assert unit_key(amount._unit) == unit_key(m.deals.a.amount._unit) + assert INDEX(m.deals, mo.Variable([1, 2, 3]), 2).value_type == "string" + + def test_the_cells_disagree(self): + m = _model() + rows = [m.i.years, m.i.rates] + out = VLOOKUP(2027.5, rows, 2) # a year or a rate, by the key + assert out.value == 0.04 and out.value_type == "float" and out._unit is None + out = VLOOKUP(2026, rows, 2, False) + assert out.value == 2027 and out.value_type == "int" and out._unit is None + out = HLOOKUP("alpha", m.deals, 3, False) # a label, a region or an amount + assert out.value == "Gamma" and out.value_type == "string" and out._unit is None + out = INDEX(rows, mo.Variable([1, 2, 1]), 2) # a series: a number + assert out.value == [2027, 0.04, 2027] + assert out.value_type == "float" and out._unit is None + out = INDEX(m.deals, 2, mo.Variable([1, 2, 3])) + assert out.value == ["Beta", "South", 200.0] + assert out.value_type == "float" and out._unit is None + + +class TestHLOOKUP: + def test_over_a_list_of_rows(self): + m = _model() + assert HLOOKUP(2027, [m.i.years, m.i.rates], 2, False).value == 0.04 + assert HLOOKUP(2027.5, [m.i.years, m.i.rates], 2).value == 0.04 + assert HLOOKUP(2025, [m.i.years, m.i.rates], 2).value == "#N/A" + assert HLOOKUP(2027, [m.i.years, m.i.rates], 1, False).value == 2027 + assert HLOOKUP(2027, [m.i.years, m.i.rates], 3, False).value == "#REF!" + assert HLOOKUP(2027, [m.i.years, m.i.rates], 0, False).value == "#VALUE!" + assert HLOOKUP(2025, [m.i.years, m.i.rates], 3, False).value == "#REF!" + assert HLOOKUP("#N/A", [m.i.years, m.i.rates], 0, False).value == "#VALUE!" + assert HLOOKUP([2028, 2026], [m.i.years, m.i.rates], 2, False).value == [0.03, 0.05] + + def test_over_a_table_of_records_the_labels_are_the_first_row_s_cells(self): + m = _model() + assert HLOOKUP("North", m.deals, 2, False).value == "South" + assert HLOOKUP("Alpha", m.deals, 1, False).value == "Alpha" + + def test_type_and_unit_come_from_the_row(self): + from modeleon.core.unit import unit_key + m = _model() + rate = HLOOKUP(2027, [m.i.years, m.i.rates], 2, False) + assert rate.value_type == "float" + assert unit_key(rate._unit) == unit_key(m.i.rates._unit) + assert HLOOKUP(2027, [m.i.years, m.i.rates], 1, False).value_type == "int" + assert HLOOKUP(2027, [m.i.years, m.i.rates], 1, False)._unit is None + out = HLOOKUP(2027, [m.i.years, m.i.rates], 2, False) + assert out._source_code == "HLOOKUP(2027, [i.years, i.rates], 2, False)" + + +# ══════════════════════════════════════════════════════════════════════ +# The registry and the AST +# ══════════════════════════════════════════════════════════════════════ + + +class TestTheRegistry: + @pytest.mark.parametrize("name, positions, prefix", [ + ("MATCH", (1,), ""), ("INDEX", (0,), ""), ("XLOOKUP", (1, 2), "_xlfn."), + ("VLOOKUP", (1,), ""), ("HLOOKUP", (1,), ""), + ]) + def test_every_function_is_a_range_function(self, name, positions, prefix): + s = spec(name) + assert s is not None and s.shape == "range" and not s.bare + assert is_range_taking(name) and is_passthrough(name) + assert range_positions(name) == positions + assert file_prefix(name) == prefix + assert not admits_in_loop(name, 3) + + def test_na_is_written_as_given(self): + s = spec("NA") + assert s is not None and s.bare and s.shape == "scalar" and is_passthrough("NA") + + +class TestTheAST: + def test_every_function_carries_a_funccall(self): + m = _model() + for out in ( + MATCH(m.i.pick, m.i.years, 0), + INDEX(m.i.rates, m.i.pick), + INDEX(m.deals, m.i.pick, 1), + XLOOKUP(m.i.pick, m.i.years, m.i.rates), + XLOOKUP(m.i.pick, m.i.infl), + VLOOKUP(m.i.pick, m.deals, 2), + HLOOKUP(m.i.pick, [m.i.years, m.i.rates], 2), + ): + assert isinstance(out._expr, FuncCall) + assert {m.i.pick.id} <= {v.id for v in out._expr.iter_refs()} + + def test_the_block_s_lines_are_dependencies(self): + m = _model() + out = VLOOKUP("Beta", m.deals, 3, False) + refs = {v.id for v in out._expr.iter_refs()} + assert {m.deals.a.region.id, m.deals.c.amount.id} <= refs + assert m.deals.a.flow.id not in refs + + +def _unit_lines(): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with m: + m.i = mo.MultiVariable("I") + with m.i as i: + i.amount = mo.Variable(150.0, display_name="Amount", unit="USD") + i.amounts = mo.Variable([90.0, 150.0, 400.0], display_name="Amounts", unit="USD") + i.thresholds = mo.Variable([0.0, 100.0, 300.0], display_name="Thresholds", + unit="EUR") + i.fees = mo.Variable([1.0, 2.0, 3.0], display_name="Fees", unit="EUR") + i.grades = mo.Variable([1.0, 2.0, 3.0], display_name="Grades") + i.usd = mo.Variable(10.0, display_name="Usd", unit="USD") + i.eur = mo.Variable(20.0, display_name="Eur", unit="EUR") + i.k = mo.Variable([1, 2, 1], display_name="K") + return m.i + + +class TestUnits: + """A key is compared with the cells searched, as ``==`` compares two + lines; a cell is taken from the cells returned, as ``CHOOSE`` takes + one: two units are refused, a number or an unlabelled line goes with + any, and the result is in the returned cells' unit, never the key's.""" + + def test_a_key_in_another_unit_than_the_cells_searched_is_refused(self): + i = _unit_lines() + calls = { + "MATCH": lambda: MATCH(i.amount, i.thresholds, 1), + "XLOOKUP": lambda: XLOOKUP(i.amount, i.thresholds, i.fees, match_mode=-1), + "VLOOKUP": lambda: VLOOKUP(i.amount, [i.thresholds, i.fees], 2), + "HLOOKUP": lambda: HLOOKUP(i.amount, [i.thresholds, i.fees], 2), + } + for name, call in calls.items(): + with pytest.raises(ValueError, match=f"mo.{name} compares the key 'Amount' with " + f"'Thresholds'; they are in USD and EUR"): + call() + with pytest.raises(ValueError, match="compares the key 'Amount' with 'Eur'"): + XLOOKUP(i.amount, [i.usd, i.eur], [1.0, 2.0]) # a list of lines searched + # A typed key, an unlabelled line or a key in the cells' unit goes. + assert MATCH(150.0, i.thresholds, 1).value == 2 + assert MATCH(i.grades, i.thresholds, 1).value == [1, 1, 1] + out = XLOOKUP(i.amount, i.amounts, i.fees) + assert out.value == 2.0 and str(out.unit) == "EUR" # never the key's + + def test_an_if_not_found_in_another_unit_than_the_cells_returned_is_refused(self): + i = _unit_lines() + with pytest.raises(ValueError, match="mo.XLOOKUP takes its values in one unit; " + "'Fees' and 'Usd' are in EUR and USD"): + XLOOKUP(i.amount, i.amounts, i.fees, i.usd) + assert str(XLOOKUP(i.amount, i.amounts, i.fees, 0.0).unit) == "EUR" + assert str(XLOOKUP(i.amount, i.amounts, i.fees, i.eur).unit) == "EUR" + # The returned cells' unit, as before: a fallback cannot lend one. + assert XLOOKUP(i.amount, i.amounts, i.grades, i.eur).unit is None + + def test_cells_returned_in_two_units_are_refused(self): + i = _unit_lines() + calls = [ + lambda: INDEX([i.usd, i.eur], 2), + lambda: INDEX([i.usd, i.eur], i.k), + lambda: XLOOKUP(i.amount, i.amounts, [i.usd, i.eur, i.usd]), + lambda: INDEX([i.amounts, i.fees], i.k, 2), # a row position that moves + lambda: VLOOKUP(2.0, [i.grades, i.amounts, i.fees], 2, False), + lambda: HLOOKUP(2.0, [i.grades, i.amounts, i.fees], i.k, False), + ] + for call in calls: + with pytest.raises(ValueError, match="takes its values in one unit; 'Amounts' and " + "'Fees' are|takes its values in one unit; " + "'Usd' and 'Eur' are"): + call() + # One cell addressed: its own unit; a label or a line without a unit + # among the cells leaves none, as before. + assert str(INDEX([i.amounts, i.fees], 2, 2).unit) == "EUR" + assert INDEX([i.grades, i.fees], i.k, 2).unit is None + + def test_a_position_the_block_lacks_is_its_error_not_a_refusal(self): + # The call is #VALUE! / #REF! in every period: no cell is taken. + i = _unit_lines() + assert INDEX([i.amounts, i.fees], 0, 2).value == "#VALUE!" + assert INDEX([i.amounts, i.fees], 5, 2).value == "#REF!" + assert VLOOKUP(90.0, [i.amounts, i.fees], 9, False).value == "#REF!" + + def test_the_law_gives_the_unit_the_call_is_built_with(self): + # What a loop container derives the row by again (function_unit) + # agrees with the call: the returned cells' unit, never the key's. + from modeleon.core.unit import unit_key + from modeleon.functions._helpers import function_unit, operand_unit + i = _unit_lines() + for out in (MATCH(i.amount, i.amounts, 0), INDEX([i.usd, i.usd], 2), + INDEX([i.amounts, i.fees], 2, 2), XLOOKUP(i.amount, i.amounts, i.grades), + XLOOKUP(i.amount, i.amounts, i.fees, 0.0), + VLOOKUP(2.0, [i.grades, i.fees], 2, False), + HLOOKUP(2.0, [i.grades, i.fees], 2, False)): + law = function_unit(out._expr.func, out._expr.args, operand_unit) + assert (None if law is None else unit_key(law)) == ( + None if out.unit is None else unit_key(out.unit)), out.formula + + def test_a_key_read_before_its_line_is_compared_when_the_loop_closes(self): + def loop(body): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=3) + with m: + m.inp = mo.MultiVariable("Inputs") + with m.inp as i: + i.start = mo.Variable(100.0, unit="USD") + i.inflow = mo.Variable([100.0] * 3, unit="USD") + i.thresholds = mo.Variable([100.0, 200.0, 300.0], unit="EUR", + display_name="Thresholds") + i.fees = mo.Variable([1.0, 2.0, 3.0], unit="EUR") + m.loop = mo.MultiVariable("Loop", loop=True) + with m.loop as d: + body(d, m.inp) + return m.loop + + def mixed(d, i): + d.opening = mo.lag(d.closing, fill=i.start) + d.fee = mo.XLOOKUP(d.opening, i.thresholds, i.fees, 0.0) + d.closing = d.opening + i.inflow + with pytest.raises(ValueError, match="fee: mo.XLOOKUP compares the key 'opening' " + "with 'Thresholds'; they are in USD and EUR"): + loop(mixed) + + def same(d, i): + d.opening = mo.lag(d.closing, fill=i.start) + d.fee = mo.XLOOKUP(d.opening, i.inflow * 0 + [100.0, 200.0, 300.0], i.fees, 0.0) + d.closing = d.opening + i.inflow + out = loop(same) + assert out.fee.value == [1.0, 2.0, 3.0] and str(out.fee.unit) == "EUR" diff --git a/tests/test_loops.py b/tests/test_loops.py index 512d836..89bc97b 100644 --- a/tests/test_loops.py +++ b/tests/test_loops.py @@ -187,6 +187,26 @@ def test_rows_downstream_written_before_the_loop_closes(): assert series(m.x.ahead) == [3.0, 7.0, 15.0, 0.0] +def test_a_criteria_joined_to_a_cell_is_read_as_given_when_the_loop_closes(): + m = model(4) + with m: + m.cap = mo.Variable(500.0, unit="USD") + m.err = mo.Variable("#N/A") + m.repay = mo.Variable([200.0] * 4, unit="USD") + m.x = mo.MultiVariable("X", loop=True) + with m.x as x: + x.under = mo.COUNTIF(x.balance, ("<", m.cap)) + x.small = mo.SUMIF(x.balance, ("<", m.cap), x.balance) + x.broken = mo.COUNTIF(x.balance, ("<", m.err)) + x.balance = mo.lag(x.balance, fill=1000.0) - m.repay + assert m.x.under.value == 2 and m.x.under.unit is None + assert m.x.small.value == 600.0 and str(m.x.small.unit) == "USD" + # As outside a loop (and as the workbook computes ">" & a cell holding + # an error): the error is the result, never a count of error cells. + assert m.x.broken.value == "#N/A" + assert m.x.under.formula == "COUNTIF(balance, CONCAT('<', cap))" + + def test_a_raw_list_operand_inside_the_loop_is_a_series(): m = model(4) with m: @@ -331,6 +351,51 @@ def test_units_wait_for_the_loop_to_close(): assert m.x.share._unit is None +def _debt_inputs(m, n): + m.inp = mo.MultiVariable("Inputs") + with m.inp as i: + i.rate = mo.Variable(0.01) + i.opening = mo.Variable(1000.0, unit="EUR") + i.repay = mo.Variable([10.0] * n, unit="EUR") + i.fee = mo.Variable([1.0] * n, unit="USD") + + +def test_a_subscript_written_before_the_loop_closes_keeps_its_lines_unit(): + # ``line[0]`` is in the line's unit, built on the open loop as once it + # closed - and a ratio to it is a pure number, not the line's unit. + m = model(4) + with m: + _debt_inputs(m, 4) + m.debt = mo.MultiVariable("Debt", loop=True) + with m.debt as d: + d.first = d.balance[0] + d.last = d.balance[-1] + d.left = d.balance / d.balance[0] + d.balance = mo.lag(d.balance, fill=m.inp.opening) - m.inp.repay + assert str(m.debt.first._unit) == "EUR" + assert str(m.debt.last._unit) == "EUR" + assert m.debt.left._unit is None + assert m.debt.first.value == pytest.approx(990.0) + + +def test_a_recurrence_written_before_the_loop_closes_keeps_its_starts_unit(): + # A running total and a roll-forward are in their start's unit, as + # mo.recurrence builds them outside a loop; a bare number starts none. + m = model(4) + with m: + _debt_inputs(m, 4) + m.debt = mo.MultiVariable("Debt", loop=True) + with m.debt as d: + d.paid = mo.cumsum(d.interest) + d.roll = mo.recurrence(m.inp.opening, "{prev} - {x}", x=d.interest) + d.count = mo.recurrence(0.0, "{prev} + 1", x=d.interest) + d.interest = mo.lag(d.balance, fill=m.inp.opening) * m.inp.rate + d.balance = mo.lag(d.balance, fill=m.inp.opening) - m.inp.repay + assert str(m.debt.paid._unit) == "EUR" + assert str(m.debt.roll._unit) == "EUR" + assert m.debt.count._unit is None + + def test_a_thrown_away_expression_does_not_stop_the_loop(): m = model(4) with m: @@ -490,6 +555,73 @@ def test_a_unit_mix_found_when_the_loop_closes_names_the_row(): x.balance = mo.lag(x.balance, fill=100.0) - m.repay +def test_a_unit_mix_through_a_subscript_is_refused_when_the_loop_closes(): + m = model(4) + with m: + _debt_inputs(m, 4) + m.debt = mo.MultiVariable("Debt", loop=True) + with pytest.raises(ValueError, match=r"mix.*incompatible units: EUR and USD"): + with m.debt as d: + d.mix = d.balance[0] + m.inp.fee + d.balance = mo.lag(d.balance, fill=m.inp.opening) - m.inp.repay + + +def test_a_unit_mix_through_a_recurrence_is_refused_when_the_loop_closes(): + m = model(4) + with m: + _debt_inputs(m, 4) + m.debt = mo.MultiVariable("Debt", loop=True) + with pytest.raises(ValueError, match=r"mix.*incompatible units: EUR and USD"): + with m.debt as d: + d.mix = mo.cumsum(d.balance) + m.inp.fee + d.balance = mo.lag(d.balance, fill=m.inp.opening) - m.inp.repay + + +def _usd_eur_inputs(m): + m.inp = mo.MultiVariable("Inputs") + with m.inp as i: + i.opening = mo.Variable(100.0, unit="USD") + i.cash = mo.Variable([10.0] * 4, unit="USD") + i.fee = mo.Variable([1.0] * 4, unit="EUR") + + +@pytest.mark.parametrize("order", [ + ["prev", "balance", "flow"], ["balance", "flow", "prev"], ["flow", "prev", "balance"], +]) +def test_a_mix_is_reported_on_the_row_that_writes_it(order): + # ``balance`` adds an EUR fee to a USD lag; ``prev`` only reads it + # through mo.lag with a USD fill. The refusal names balance, in every + # order the rows are written, never the row whose lag reads it. + m = model(4) + with m: + _usd_eur_inputs(m) + m.x = mo.MultiVariable("X", loop=True) + with pytest.raises(ValueError, match=r"x\.balance: Cannot add/subtract values with " + r"incompatible units: USD and EUR"): + with m.x as d: + for name in order: + if name == "prev": + d.prev = mo.lag(d.balance, fill=m.inp.opening) + elif name == "balance": + d.balance = mo.lag(d.flow, fill=0.0) + m.inp.fee + else: + d.flow = d.prev + m.inp.cash + + +def test_a_mix_read_through_a_pass_through_row_is_reported_where_written(): + m = model(4) + with m: + _usd_eur_inputs(m) + m.x = mo.MultiVariable("X", loop=True) + with pytest.raises(ValueError, match=r"x\.balance: Cannot add/subtract values with " + r"incompatible units: USD and EUR"): + with m.x as d: + d.prev = mo.lag(d.mid, fill=m.inp.opening) + d.balance = mo.lag(d.flow, fill=0.0) + m.inp.fee + d.mid = d.balance * 1 + d.flow = d.prev + m.inp.cash + + def test_python_that_cannot_be_rerun_is_refused_over_an_open_loop(): m = model(4) with m: @@ -897,3 +1029,76 @@ def test_an_undeclared_loop_row_prints_the_label_its_formula_gives(): d.x = mo.lag(d.y, fill=0.0) + m.inp.sh d.y = (d.x * 0.5).set_unit(factor) assert str(m.x.x._unit) == "factor" + + +def test_a_list_holding_a_total_of_a_loop_row_waits_for_the_loop(): + # The list's lines are the call's dependencies: a total of a row read + # before its line is the closed loop's, never the placeholder's. + m = model(4) + with m: + m.inp = mo.MultiVariable("Inputs") + with m.inp as i: + i.rate = mo.Variable(0.01) + i.repay = mo.Variable([10.0] * 4, unit="EUR") + i.f = mo.Variable(6.0, unit="EUR") + m.x = mo.MultiVariable("X", loop=True) + with m.x as x: + x.npv = mo.NPV(m.inp.rate, [mo.SUM(x.balance), m.inp.f]) + x.balance = mo.lag(x.balance, fill=100.0) - m.inp.repay + assert m.x.npv.value == pytest.approx(300.0 / 1.01 + 6.0 / 1.01 ** 2) + + +def test_a_downstream_recurrence_refuses_two_units_when_the_loop_closes(): + # Run again from its formula when the loop closes, the template is read + # for units then, and a mix is reported under the row. + def build(sched_unit): + m = model(3) + with m: + m.inp = mo.MultiVariable("Inputs") + with m.inp as i: + i.opening = mo.Variable(100.0, unit="EUR") + i.sched = mo.Variable([1.0, 2.0, 3.0], unit=sched_unit) + m.debt = mo.MultiVariable("Debt", loop=True) + with m.debt as d: + d.cum = mo.recurrence(m.inp.opening, "{prev} + {r}", r=d.repay) + d.repay = m.inp.sched * 1.0 + return m + with pytest.raises(ValueError, match=r'debt\.cum: recurrence\("\{prev\} \+ \{r\}"\): ' + r'Cannot add/subtract'): + build("USD") + m = build("EUR") + eager = mo.recurrence(m.inp.opening, "{prev} + {r}", r=m.inp.sched * 1.0) + assert str(m.debt.cum.unit) == "EUR" + assert m.debt.cum.value == pytest.approx(eager.value) + + +def test_a_recurrence_in_a_loop_takes_the_unit_its_template_gives(): + # One rule inside and outside a loop: a start without a unit takes + # the template's, so the row is in USD as the same line built eagerly. + m = model(3) + with m: + m.inp = mo.MultiVariable("Inputs") + m.inp.sched = mo.Variable([1.0, 2.0, 3.0], unit="USD") + m.debt = mo.MultiVariable("Debt", loop=True) + with m.debt as d: + d.cum = mo.recurrence(0.0, "{prev} + {r}", r=d.repay) + d.repay = m.inp.sched * 1.0 + assert str(m.debt.cum.unit) == "USD" + assert str(mo.recurrence(0.0, "{prev} + {r}", r=m.inp.sched * 1.0).unit) == "USD" + + +def test_a_declared_recurrence_row_in_a_loop_keeps_its_unit_when_the_loop_closes(): + # A declared unit decides what the line prints, in a loop as outside: + # the re-run at close carries it, so the product is not refused. + m = model(3) + with m: + m.inp = mo.MultiVariable("Inputs") + with m.inp as i: + i.opening = mo.Variable(100.0, unit="EUR") + i.factor = mo.Variable([1.0, 2.0, 3.0], unit="USD") + m.debt = mo.MultiVariable("Debt", loop=True) + with m.debt as d: + d.cum = mo.recurrence(m.inp.opening, "{prev} * {x}", x=d.scale, unit="EUR") + d.scale = m.inp.factor * 1.0 + assert str(m.debt.cum.unit) == "EUR" + assert m.debt.cum.value == pytest.approx([100.0, 200.0, 600.0]) diff --git a/tests/test_mathfn.py b/tests/test_mathfn.py new file mode 100644 index 0000000..9b8f069 --- /dev/null +++ b/tests/test_mathfn.py @@ -0,0 +1,666 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The rounding family, the powers and logarithms, ``SIGN`` and +``PRODUCT`` compute what the spreadsheet computes, with the formula as +the product. + +``ROUNDUP`` / ``ROUNDDOWN`` / ``TRUNC`` move away from or toward zero on +the number as a cell shows it; ``CEILING`` / ``FLOOR`` keep the classic +function's sign rules; ``POWER`` / ``SQRT`` / ``EXP`` / ``LN`` / ``LOG`` +/ ``LOG10`` give the spreadsheet's errors outside their domain; ``SIGN`` +is 1, 0 or -1; ``PRODUCT`` reads a range as the sheet does. Each of the +first twelve is one call per period through the shared helper, so a +blank is 0, a date its serial number, numeric text its number and other +text ``#VALUE!``, an error cell passes through, scalars broadcast, tracks +lift, a rounded amount keeps its unit, and a number no cell can hold is +``#NUM!``. +""" + +from __future__ import annotations + +import datetime as dt +import math +import time + +import pytest + +import modeleon as mo +from modeleon.core.expr import FuncCall, Literal, VarRef +from modeleon.functions.mathfn import ( + CEILING, + EXP, + FLOOR, + LN, + LOG, + LOG10, + POWER, + PRODUCT, + ROUNDDOWN, + ROUNDUP, + SIGN, + SQRT, + TRUNC, +) +from modeleon.functions.registry import ( + admits_in_loop, + excel_name, + file_prefix, + is_elementwise, + is_range_taking, + spec, +) + +ELEMENTWISE = { + "ROUNDUP": ROUNDUP, "ROUNDDOWN": ROUNDDOWN, "TRUNC": TRUNC, "CEILING": CEILING, + "FLOOR": FLOOR, "POWER": POWER, "SQRT": SQRT, "EXP": EXP, "LN": LN, "LOG": LOG, + "LOG10": LOG10, "SIGN": SIGN, +} + + +# ══════════════════════════════════════════════════════════════════════ +# The rounding family +# ══════════════════════════════════════════════════════════════════════ + + +class TestRoundUp: + @pytest.mark.parametrize("value, digits, expected", [ + (3.2, 0, 4.0), + (76.9, 0, 77.0), + (3.14159, 3, 3.142), + (-3.14159, 1, -3.2), + (31415.92654, -2, 31500.0), + (-3.2, 0, -4.0), + (2.5, 0, 3.0), + (0.0, 0, 0.0), + (1.001, 2, 1.01), + (-1.001, 2, -1.01), + (1e-20, 2, 0.01), + ]) + def test_away_from_zero(self, value, digits, expected): + assert ROUNDUP(value, digits).value == expected + + def test_on_the_number_as_shown(self): + # 0.1 + 0.2 is held as 0.30000000000000004 and shown as 0.3. + assert ROUNDUP(0.1 + 0.2, 1).value == 0.3 + assert ROUNDUP(mo.Variable(0.1) * 3, 1).value == 0.3 + assert ROUNDUP(2.0000000000000004, 0).value == 2.0 + + def test_whole_numbers_stay_whole(self): + assert ROUNDUP(7, 2).value == 7 and isinstance(ROUNDUP(7, 2).value, int) + assert ROUNDUP(1234, -2).value == 1300 and isinstance(ROUNDUP(1234, -2).value, int) + + def test_a_list_and_digits_that_vary_by_period(self): + v = mo.Variable([2.345, -2.345, 1250.0]) + assert ROUNDUP(v, 1).value == [2.4, -2.4, 1250.0] + assert ROUNDUP(v, [2, 0, -2]).value == [2.35, -3.0, 1300.0] + assert ROUNDUP(v, mo.Variable(1)).value == [2.4, -2.4, 1250.0] + + +class TestRoundDown: + @pytest.mark.parametrize("value, digits, expected", [ + (3.2, 0, 3.0), + (76.9, 0, 76.0), + (3.14159, 3, 3.141), + (-3.14159, 1, -3.1), + (31415.92654, -2, 31400.0), + (-3.9, 0, -3.0), + (2.5, 0, 2.0), + (1.999, 2, 1.99), + (0.0, 0, 0.0), + ]) + def test_toward_zero(self, value, digits, expected): + assert ROUNDDOWN(value, digits).value == expected + + def test_on_the_number_as_shown(self): + # 4.35 * 100 is held as 434.99999999999994 and shown as 435. + assert ROUNDDOWN(4.35 * 100, 0).value == 435.0 + assert ROUNDDOWN(mo.Variable(4.35) * 100, 0).value == 435.0 + assert ROUNDDOWN(2.675 * 100, 0).value == 267.0 + + def test_whole_numbers_stay_whole(self): + assert ROUNDDOWN(1299, -2).value == 1200 + assert isinstance(ROUNDDOWN(1299, -2).value, int) + + +class TestTrunc: + @pytest.mark.parametrize("value, digits, expected", [ + (8.9, None, 8.0), + (-8.9, None, -8.0), + (0.45, None, 0.0), + (1234.5678, 2, 1234.56), + (-1234.5678, 2, -1234.56), + (1234.5678, -2, 1200.0), + (-1234.5678, -2, -1200.0), + (7, None, 7), + ]) + def test_drops_the_decimals_toward_zero(self, value, digits, expected): + out = TRUNC(value) if digits is None else TRUNC(value, digits) + assert out.value == expected + + def test_on_the_number_as_shown(self): + assert TRUNC(mo.Variable(4.35) * 100).value == 435.0 + + def test_the_second_argument_is_written_only_when_given(self): + x = mo.Variable(8.9, display_name="x") + assert TRUNC(x).formula == "TRUNC(x)" + assert TRUNC(x, 2).formula == "TRUNC(x, 2)" + assert len(TRUNC(x)._expr.args) == 1 and len(TRUNC(x, 2)._expr.args) == 2 + + +class TestCeiling: + @pytest.mark.parametrize("value, significance, expected", [ + (2.5, 1, 3.0), + (-2.5, -2, -4.0), + (-2.5, 2, -2.0), + (1.5, 0.1, 1.5), + (0.234, 0.01, 0.24), + (4.2, 2, 6.0), + (4.0, 2, 4.0), + (0.0, 0.5, 0.0), + (-0.5, 2, 0.0), + (7, 2, 8), + (6, 2, 6), + (-7, 2, -6), + (-7, -2, -8), + ]) + def test_away_from_zero_to_a_multiple(self, value, significance, expected): + assert CEILING(value, significance).value == expected + + def test_a_positive_number_with_a_negative_significance_is_num(self): + assert CEILING(2.5, -2).value == '#NUM!' + assert CEILING(mo.Variable([2.5, -2.5]), -2).value == ['#NUM!', -4.0] + + def test_a_zero_significance_gives_zero(self): + assert CEILING(5, 0).value == 0 + assert CEILING(mo.Variable([5.0, -5.0]), 0).value == [0, 0] + + def test_the_quotient_is_read_as_shown(self): + # 1.1 / 0.1 is held as 11.000000000000002 and shown as 11. + assert CEILING(1.1, 0.1).value == 1.1 + assert CEILING(mo.Variable(1.1), 0.1).value == 1.1 + assert CEILING(mo.Variable(0.7) * 3, 0.1).value == 2.1 + + def test_a_quotient_of_sixteen_whole_digits_is_taken_as_held(self): + # No decimals are left to show, so an exact multiple stays. + assert CEILING(1000000000000001, 1).value == 1000000000000001 + assert CEILING(1000000000000000.5, 1).value == 1000000000000001.0 + assert FLOOR(1000000000000000.5, 1).value == 1000000000000000.0 + assert CEILING(51234567890123.45, 0.01).value == 51234567890123.45 + assert FLOOR(51234567890123.45, 0.01).value == 51234567890123.45 + + +class TestFloor: + @pytest.mark.parametrize("value, significance, expected", [ + (3.7, 2, 2.0), + (-2.5, -2, -2.0), + (-2.5, 2, -4.0), + (1.58, 0.1, 1.5), + (0.234, 0.01, 0.23), + (4.0, 2, 4.0), + (0.0, 0.5, 0.0), + (7, 2, 6), + (6, 2, 6), + (-7, 2, -8), + (-7, -2, -6), + ]) + def test_toward_zero_to_a_multiple(self, value, significance, expected): + assert FLOOR(value, significance).value == expected + + def test_a_positive_number_with_a_negative_significance_is_num(self): + assert FLOOR(2.5, -2).value == '#NUM!' + + def test_a_zero_significance_divides_by_zero_unless_the_number_is_zero(self): + assert FLOOR(5, 0).value == '#DIV/0!' + assert FLOOR(mo.Variable([5.0, -5.0, 0.0]), 0).value == ['#DIV/0!', '#DIV/0!', 0.0] + assert FLOOR(0, 0).value == 0 and isinstance(FLOOR(0, 0).value, int) + assert FLOOR(mo.Variable([None]), mo.Variable([None])).value == [0] + + def test_the_quotient_is_read_as_shown(self): + # 0.3 / 0.1 is held as 2.9999999999999996 and shown as 3. + assert FLOOR(0.3, 0.1).value == 0.3 + assert FLOOR(mo.Variable(0.7) * 3, 0.1).value == 2.1 + + +class TestANegativeZeroIsZero: + @pytest.mark.parametrize("out", [ + ROUNDUP(-0.0, 0), ROUNDDOWN(-0.4, 0), TRUNC(-0.125), TRUNC(-0.004, 2), + CEILING(-0.5, 2), FLOOR(-0.0, 1), POWER(-0.0, 3), + ]) + def test_as_a_cell_shows_it(self, out): + assert out.value == 0 and math.copysign(1.0, out.value) == 1.0 + + +class TestANumberNoCellCanHoldIsNum: + def test_rounded_to_a_multiple_past_the_largest_number(self): + assert ROUNDUP(2.5, -320).value == '#NUM!' # 1e320 fits no cell + assert ROUNDUP(5, -320).value == '#NUM!' + assert ROUNDDOWN(2.5, -320).value == 0 and TRUNC(-2.5, -320).value == 0 + assert ROUNDDOWN(2.5, 400).value == 2.5 + assert CEILING(1.7e308, 1e308).value == '#NUM!' + assert ROUNDUP(math.inf, 0).value == '#NUM!' + assert CEILING(math.nan, 1).value == '#NUM!' + + def test_a_quotient_past_the_largest_number_or_below_the_smallest(self): + # The sheet divides in doubles: 1e300 / 1e-300 is past any cell, + # 1e-300 / 1e300 is 0. + assert CEILING(1e300, 1e-300).value == '#NUM!' + assert FLOOR(1e300, 1e-300).value == '#NUM!' + assert CEILING(-1e300, -1e-300).value == '#NUM!' + assert CEILING(1e-300, 1e300).value == 0 and FLOOR(-1e-300, 1e300).value == 0 + assert FLOOR(1.7e308, 1e308).value == 1e308 + + @pytest.mark.parametrize("fn", [SQRT, EXP, LN, LOG, LOG10, SIGN]) + def test_an_operand_no_cell_holds(self, fn): + assert fn(math.inf).value == '#NUM!' and fn(math.nan).value == '#NUM!' + + def test_a_power_or_a_product_past_it(self): + assert POWER(10, 400).value == '#NUM!' and POWER(2.0, 1024).value == '#NUM!' + assert PRODUCT(1e200, 1e200).value == '#NUM!' + assert PRODUCT(mo.Variable([10 ** 200, 10 ** 200])).value == '#NUM!' + + +# ══════════════════════════════════════════════════════════════════════ +# The powers and logarithms +# ══════════════════════════════════════════════════════════════════════ + + +class TestPower: + def test_the_values(self): + assert POWER(5, 2).value == 25 and isinstance(POWER(5, 2).value, int) + assert POWER(98.6, 3.2).value == pytest.approx(2401077.222, abs=1e-3) + assert POWER(4, 5 / 4).value == pytest.approx(5.656854, abs=1e-6) + assert POWER(2, -1).value == 0.5 + assert POWER(-2, 3).value == -8 + assert POWER(0, 5).value == 0 + assert POWER(mo.Variable([2, 3, 4]), 2).value == [4, 9, 16] + assert POWER(2, [1, 2, 3]).value == [2, 4, 8] + + def test_the_spreadsheet_s_errors(self): + assert POWER(0, 0).value == '#NUM!' + assert POWER(-8, 0.5).value == '#NUM!' + assert POWER(0, -1).value == '#DIV/0!' + assert POWER(10.0, 400).value == '#NUM!' + assert POWER(10, 400).value == '#NUM!' + + def test_a_negative_base_has_a_real_odd_root(self): + assert POWER(-8, 1 / 3).value == -2.0 + assert POWER(-32, 0.2).value == -2.0 and POWER(-8, -1 / 3).value == -0.5 + assert POWER(-27, 1 / 3).value == pytest.approx(-3.0) + assert POWER(mo.Variable([-8.0, 8.0]), 1 / 3).value == [-2.0, 2.0] + # The inverse of an odd whole number to within the 15th digit. + assert POWER(-8, 0.333333333333333).value == pytest.approx(-2.0) + assert POWER(-8, 0.33333333333333).value == '#NUM!' + + @pytest.mark.parametrize("exponent", [0.5, 0.4, 2 / 3, 1.5, -2.5, 1e-16, 5e-324]) + def test_any_other_fraction_of_a_negative_base_is_num(self, exponent): + assert POWER(-8, exponent).value == '#NUM!' + + def test_a_whole_power_past_any_cell_is_num_without_computing_it(self): + started = time.perf_counter() + assert POWER(3, 10 ** 7).value == '#NUM!' # some seconds to compute in full + assert POWER(-2, 10 ** 7 + 1).value == '#NUM!' + assert time.perf_counter() - started < 1.0 + assert POWER(2, 1023).value == 2 ** 1023 and POWER(2, 1024).value == '#NUM!' + assert POWER(-1, 10 ** 9 + 1).value == -1 and POWER(1, 10 ** 9).value == 1 + + def test_a_power_too_small_for_a_cell_is_zero(self): + assert POWER(8, -400).value == 0.0 and POWER(0.5, 1100).value == 0.0 + + def test_the_unit_follows_the_operator(self): + amount = mo.Variable([3.0, 8.0], unit="$") + assert str(POWER(amount, 2)._unit) == str((amount ** 2)._unit) + assert POWER(amount, 0.5)._unit is None + assert POWER(amount, mo.Variable(2))._unit is None + assert POWER(2, amount)._unit is None + + +class TestRootsExponentialsAndLogarithms: + def test_sqrt(self): + assert SQRT(16).value == 4.0 + assert SQRT(2).value == pytest.approx(1.414214, abs=1e-6) + assert SQRT(0).value == 0.0 + assert SQRT(-16).value == '#NUM!' + assert SQRT(mo.Variable([16.0, -16.0, 2.25])).value == [4.0, '#NUM!', 1.5] + + def test_exp(self): + assert EXP(0).value == 1.0 + assert EXP(1).value == pytest.approx(2.718282, abs=1e-6) + assert EXP(-1).value == pytest.approx(0.367879, abs=1e-6) + assert EXP(1000).value == '#NUM!' + + def test_ln(self): + assert LN(86).value == pytest.approx(4.454347, abs=1e-6) + assert LN(math.e).value == 1.0 + assert LN(1).value == 0.0 + assert LN(0).value == '#NUM!' and LN(-1).value == '#NUM!' + + def test_log(self): + assert LOG(10).value == 1.0 and LOG(100).value == 2.0 + assert LOG(8, 2).value == 3.0 + assert LOG(1000, 10).value == 3.0 # exact, as a cell shows it + assert LOG(81, 3).value == 4.0 + assert LOG(86, 2.7182818).value == pytest.approx(4.454347, abs=1e-6) + assert LOG(8, 1).value == '#DIV/0!' + assert LOG(-1).value == '#NUM!' and LOG(0).value == '#NUM!' + assert LOG(8, -2).value == '#NUM!' and LOG(8, 0).value == '#NUM!' + + def test_log10(self): + assert LOG10(86).value == pytest.approx(1.934498, abs=1e-6) + assert LOG10(10).value == 1.0 and LOG10(1e5).value == 5.0 + assert LOG10(0).value == '#NUM!' + + def test_the_base_is_written_only_when_given(self): + x = mo.Variable(8.0, display_name="x") + assert LOG(x).formula == "LOG(x)" and LOG(x, 2).formula == "LOG(x, 2)" + assert len(LOG(x)._expr.args) == 1 and len(LOG(x, 2)._expr.args) == 2 + + @pytest.mark.parametrize("fn", [SQRT, EXP, LN, LOG, LOG10]) + def test_a_float_without_the_operand_s_unit(self, fn): + amount = mo.Variable([3.0, 8.0], unit="$") + out = fn(amount) + assert out.value_type == 'float' and out._unit is None + assert fn(mo.Variable([3, 8])).value_type == 'float' + + +class TestSign: + def test_the_values(self): + assert SIGN(10).value == 1 and SIGN(0).value == 0 and SIGN(-0.00001).value == -1 + assert SIGN(mo.Variable(4) - 4).value == 0 + assert SIGN(mo.Variable([3.0, -8.0, 0.0])).value == [1, -1, 0] + + def test_a_whole_number_without_the_operand_s_unit(self): + out = SIGN(mo.Variable([3.0, -8.0], unit="$")) + assert out.value_type == 'int' and out._unit is None + + +# ══════════════════════════════════════════════════════════════════════ +# The shared law: one call per period +# ══════════════════════════════════════════════════════════════════════ + + +class TestOneCallPerPeriod: + def test_a_blank_is_zero(self): + blank = mo.Variable([None]) + assert ROUNDUP(blank, 0).value == [0] + assert TRUNC(blank).value == [0] + assert CEILING(5, blank).value == [0] + assert POWER(blank, 2).value == [0] and POWER(2, blank).value == [1] + assert SQRT(blank).value == [0.0] + assert EXP(blank).value == [1.0] + assert LN(blank).value == ['#NUM!'] + assert SIGN(blank).value == [0] + + def test_numeric_text_is_its_number_and_other_text_is_value(self): + words = mo.Variable(["abc", "4", "2.5", "50%"]) + assert ROUNDUP(words, 0).value == ['#VALUE!', 4, 3.0, 1.0] + assert CEILING(words, 1).value == ['#VALUE!', 4, 3.0, 1.0] + assert POWER(2, words).value == [ + '#VALUE!', 16, pytest.approx(5.656854, abs=1e-6), pytest.approx(1.414214, abs=1e-6), + ] + assert SIGN(words).value == ['#VALUE!', 1, 1, 1] + assert SQRT("16").value == 4.0 and SQRT("abc").value == '#VALUE!' + assert ROUNDUP("1,234.5", 0).value == 1235 # read as VALUE reads it + + def test_a_date_is_its_serial_number(self): + m = mo.Model("m", default_grain="month", default_start="2024-01", default_periods=3) + with m: + m.when = mo.EOMONTH(mo.time.start, 0) # 45322, 45351, 45382 + m.week = FLOOR(m.when, 7) + m.week_end = CEILING(mo.time.end, 7) + m.sign = SIGN(m.when) + assert m.week.value == [45318, 45346, 45381] + assert m.week_end.value == [45325, 45353, 45388] + assert m.sign.value == [1, 1, 1] + day = mo.Variable(dt.date(2024, 2, 29)) + noon = mo.Variable(dt.datetime(2024, 2, 29, 12, 0)) + assert ROUNDUP(day, -1).value == 45360 and TRUNC(day).value == 45351 + assert TRUNC(noon).value == 45351 and ROUNDDOWN(noon, 1).value == 45351.5 + assert SQRT(day).value == pytest.approx(212.957742287056) + assert FLOOR(day, 7).value_type == 'float' + + def test_a_boolean_is_one_or_zero(self): + assert ROUNDUP(True, 0).value == 1 and ROUNDUP(False, 0).value == 0 + assert SIGN(True).value == 1 and SQRT(True).value == 1.0 + assert POWER(True, 2).value == 1 and CEILING(True, 2).value == 2 + + def test_an_error_cell_is_the_period_s_result(self): + e = mo.Variable([2.0, 2.0]) / mo.Variable([1.0, 0.0]) + assert e.value == [2.0, '#DIV/0!'] + assert ROUNDUP(e, 0).value == [2.0, '#DIV/0!'] + assert SQRT(e).value == [pytest.approx(1.414214, abs=1e-6), '#DIV/0!'] + assert POWER(e, 2).value == [4.0, '#DIV/0!'] + assert POWER(2, e).value == [4.0, '#DIV/0!'] + assert CEILING(1.0, e).value == [2.0, '#DIV/0!'] + assert SIGN(e).value == [1, '#DIV/0!'] + + def test_scalars_only_is_one_number(self): + out = ROUNDUP(3.3, 0) + assert out.value == 4.0 and out.var_type == 'scalar' and out.value_type == 'float' + assert out.formula == "ROUNDUP(3.3, 0)" + assert CEILING(mo.Variable(2.5), mo.Variable(2)).var_type == 'scalar' + + def test_scalars_and_a_one_value_series_broadcast(self): + a = mo.Variable([2.345, -2.345, 1250.0]) + assert ROUNDUP(a, mo.Variable(1)).value == [2.4, -2.4, 1250.0] + assert ROUNDUP(a, mo.Variable([1])).value == [2.4, -2.4, 1250.0] + assert POWER(mo.Variable([2.0]), [1, 2, 3]).value == [2.0, 4.0, 8.0] + assert ROUNDUP(a, 0).var_type == 'list' + + def test_series_of_different_lengths_are_refused_as_the_operators_refuse_them(self): + with pytest.raises(ValueError, match="different lengths"): + ROUNDUP(mo.Variable([1.1, 2.2, 3.3]), mo.Variable([1, 2])) + + def test_tracked_operands_lift_per_track(self): + m = mo.Model("m", tracks=mo.Tracks("plan", "actual"), + default_grain="month", default_start="2026-01", default_periods=2) + with m: + m.x = mo.Variable(plan=[1.26, 2.0], actual=[1.0, None]) + m.r = ROUNDUP(m.x, 1) + m.s = SQRT(m.x) + assert dict(m.r._value.items()) == {"plan": [1.3, 2.0], "actual": [1.0, 0]} + assert dict(m.s._value.items()) == { + "plan": [pytest.approx(1.122497, abs=1e-6), pytest.approx(1.414214, abs=1e-6)], + "actual": [1.0, 0.0], + } + assert m.r.var_type == 'list' + + def test_value_type_is_int_only_when_the_result_is_whole_by_construction(self): + assert ROUNDUP(mo.Variable([3, 8]), -1).value_type == 'int' + assert ROUNDUP(7, 2).value_type == 'int' and TRUNC(7).value_type == 'int' + assert ROUNDUP(mo.Variable([3.0, 8.0]), 0).value_type == 'float' + assert ROUNDUP(mo.Variable(["4"]), 0).value_type == 'float' + assert CEILING(mo.Variable([3, 8]), 2).value_type == 'int' + assert CEILING(mo.Variable([3, 8]), 0.5).value_type == 'float' + assert CEILING(mo.Variable([3.0, 8.0]), mo.Variable(2)).value_type == 'float' + assert isinstance(CEILING(5.0, 0).value, float) and isinstance(CEILING(5, 0).value, int) + assert POWER(mo.Variable([3, 8]), 2).value_type == 'int' + assert POWER(mo.Variable([3, 8]), -1).value_type == 'float' + assert POWER(mo.Variable([3, 8]), mo.Variable(2)).value_type == 'float' + + def test_a_rounded_amount_keeps_its_unit(self): + a = mo.Variable([3.0, 8.0], unit="$") + step = mo.Variable(100.0, unit="$") + hours = mo.Variable(2.0, unit="hr") + for fn in (ROUNDUP, ROUNDDOWN, TRUNC, CEILING, FLOOR): + assert str(fn(a, 1)._unit) == "$", fn + assert str(CEILING(a, step)._unit) == "$" + # A multiple is in the unit of the step: a step in hours is refused. + with pytest.raises(ValueError, match="mo.CEILING takes its values in one unit"): + CEILING(a, hours) + + def test_the_formula_names_the_operands_and_the_ast_is_a_call(self): + a = mo.Variable([3.0, 8.0], display_name="a") + k = mo.Variable(2, display_name="k") + assert ROUNDUP(a, 1).formula == "ROUNDUP(a, 1)" + assert CEILING(a, k).formula == "CEILING(a, k)" + assert POWER(a, k).formula == "POWER(a, k)" + assert SIGN(a).formula == "SIGN(a)" + expr = ROUNDUP(a, 1)._expr + assert isinstance(expr, FuncCall) and expr.func == "ROUNDUP" + assert isinstance(expr.args[0], VarRef) and expr.args[0].var is a + assert isinstance(expr.args[1], Literal) and expr.args[1].value == 1 + + @pytest.mark.parametrize("name", sorted(ELEMENTWISE)) + def test_an_axised_operand_is_refused(self, name): + region = mo.Variable(["N", "S"], display_name="region") + by_region = mo.Variable([1.0, 2.0], indexed_by=[region]) + fn = ELEMENTWISE[name] + with pytest.raises(ValueError, match=name): + fn(by_region, 1) if name in ("ROUNDUP", "ROUNDDOWN", "CEILING", "FLOOR", "POWER") \ + else fn(by_region) + + +# ══════════════════════════════════════════════════════════════════════ +# PRODUCT, a reduction +# ══════════════════════════════════════════════════════════════════════ + + +class TestProduct: + def test_one_row_reduces_to_one_number(self): + out = PRODUCT(mo.Variable([5.0, 15.0, 30.0])) + assert out.value == 2250.0 and out.var_type == 'scalar' + assert PRODUCT([5, 15, 30]).value == 2250 + + def test_variadic_like_sum(self): + assert PRODUCT(5, 15, 30).value == 2250 + assert PRODUCT(5, 15, 30).formula == "PRODUCT(5, 15, 30)" + assert PRODUCT(mo.Variable(2.0), mo.Variable(3.0)).value == 6.0 + # A series beside a scalar: the scalar multiplies every period, as + # ``a * 2`` does; the row's product times 2 is + # ``PRODUCT(row) * 2``. + assert PRODUCT(mo.Variable([5.0, 15.0, 30.0]), 2).value == [10.0, 30.0, 60.0] + assert (PRODUCT(mo.Variable([5.0, 15.0, 30.0])) * 2).value == 4500.0 + + def test_two_series_are_multiplied_period_by_period(self): + # Several series are combined period by period, as ``a * b`` combines + # them; the product of all their cells is two products. + a = mo.Variable([2.0, 3.0]) + b = mo.Variable([4.0, 5.0]) + out = PRODUCT(a, b) + assert out.value == [8.0, 15.0] and out.var_type == 'list' + assert (PRODUCT(a) * PRODUCT(b)).value == 120.0 + + def test_the_range_is_read_as_the_sheet_reads_it(self): + assert PRODUCT(mo.Variable([2.0, None, 3.0])).value == 6.0 + assert PRODUCT(mo.Variable([None, None])).value == 0 + assert PRODUCT(mo.Variable(["#N/A", "#DIV/0!", 3.0])).value == "#N/A" + # Period by period: each period's first error, a blank skipped. + assert PRODUCT(mo.Variable([1.0, 2.0, None]), + mo.Variable(["#VALUE!", 1.0, None])).value == ["#VALUE!", 2.0, 0] + + def test_no_argument_is_refused(self): + with pytest.raises(ValueError, match=r"PRODUCT\(\) needs at least one argument"): + PRODUCT() + + def test_text_is_refused_at_build_time_as_sum_refuses_it(self): + with pytest.raises(TypeError, match="PRODUCT multiplies numbers; 'abc' is not one"): + PRODUCT(mo.Variable(["abc"])) + with pytest.raises(TypeError, match="give it the number"): + PRODUCT(mo.Variable([2.0, "abc"]), 3) + + def test_a_boolean_is_one_or_zero_as_libreoffice_reads_it(self): + assert PRODUCT(True, 2.5).value == 2.5 and PRODUCT(False, 2.5).value == 0 + assert PRODUCT(mo.Variable([True, False, True])).value == 0 + + def test_a_date_is_its_serial_number(self): + assert PRODUCT(mo.Variable(dt.date(2024, 2, 29)), 1).value == 45351 + assert PRODUCT(mo.Variable([dt.date(2024, 1, 31), 2])).value == 90644 + + def test_a_line_with_tracks_reduces_track_by_track(self): + m = mo.Model("m", tracks=mo.Tracks("plan", "actual"), + default_grain="month", default_start="2026-01", default_periods=2) + with m: + m.x = mo.Variable(plan=[1.5, 2.0], actual=[3.0, None]) + m.p = PRODUCT(m.x) + assert dict(m.p._value.items()) == {"plan": 3.0, "actual": 3.0} + assert m.p.formula == "PRODUCT(x)" + + def test_the_units_multiply_and_a_whole_number_only_over_whole_numbers(self): + a = mo.Variable([2.0, 3.0], unit="$") + price, volume = mo.Variable(2.0, unit="$/t"), mo.Variable(3.0, unit="t") + assert str(PRODUCT(price, volume)._unit) == "$" + # A row in a dimension is refused: its cells' product is $ to the + # power of their count; a row of pure numbers multiplies. + with pytest.raises(ValueError, match="a row in \\$"): + PRODUCT(a) + assert PRODUCT(mo.Variable([1.1, 1.2])).value == pytest.approx(1.32) + assert PRODUCT(mo.Variable([2.0, 3.0])).value_type == 'float' + assert PRODUCT(mo.Variable([2, 3])).value_type == 'int' + assert PRODUCT(mo.Variable([2, 3]), 3).value_type == 'int' + assert PRODUCT(mo.Variable([2, 3]), 2.5).value_type == 'float' + assert PRODUCT(mo.Variable(dt.date(2024, 2, 29)), 2).value_type == 'float' + + def test_an_axised_operand_is_refused(self): + region = mo.Variable(["N", "S"], display_name="region") + by_region = mo.Variable([1.0, 2.0], indexed_by=[region]) + with pytest.raises(ValueError, match="PRODUCT"): + PRODUCT(by_region) + + +# ══════════════════════════════════════════════════════════════════════ +# The registry, a recurrence template, a loop container +# ══════════════════════════════════════════════════════════════════════ + + +class TestTheRegistry: + @pytest.mark.parametrize("name", sorted(ELEMENTWISE)) + def test_each_is_element_wise_with_a_bare_classic_name(self, name): + assert is_elementwise(name) and not is_range_taking(name) + assert spec(name).passthrough and not spec(name).bare + assert file_prefix(name) == "" and excel_name(name) == name + assert admits_in_loop(name, 2) and admits_in_loop(name, 1) + + def test_product_is_a_reduction_and_one_call_per_period_over_two_values(self): + assert spec("PRODUCT").shape == "reduction" and is_range_taking("PRODUCT") + assert not is_elementwise("PRODUCT") and not admits_in_loop("PRODUCT", 1) + assert spec("PRODUCT").per_period_arity == 2 + assert admits_in_loop("PRODUCT", 2) and admits_in_loop("PRODUCT", 3) + + +class TestInARecurrenceTemplate: + def test_a_template_computes_what_the_cell_computes(self): + assert mo.recurrence(100.0, "ROUNDUP({prev} * 1.1, 0)", periods=4).value == \ + [100.0, 110.0, 121.0, 134.0] + assert mo.recurrence(100.0, "CEILING({prev} * 1.07, 5)", periods=3).value == \ + [100.0, 110.0, 120.0] + assert mo.recurrence(100.0, "FLOOR({prev} * 0.9, 1)", periods=3).value == \ + [100.0, 90.0, 81.0] + assert mo.recurrence(256.0, "SQRT({prev})", periods=3).value == [256.0, 16.0, 4.0] + assert mo.recurrence(2, "POWER({prev}, 2)", periods=3).value == [2, 4, 16] + assert mo.recurrence(5.5, "TRUNC({prev} * 1.5)", periods=3).value == [5.5, 8.0, 12.0] + assert mo.recurrence(1.0, "SIGN({prev}) * ROUNDDOWN({prev} * 2.5, 0)", periods=3).value == \ + [1.0, 2.0, 5.0] + grown = mo.recurrence(1.0, "EXP(LN({prev}) + LOG(100, 10) + LOG10(10))", periods=2) + assert grown.value == [1.0, pytest.approx(math.e ** 3)] + + def test_a_number_no_cell_holds_is_num_and_carries_on(self): + assert mo.recurrence(1.0, "EXP({prev} * 1000)", periods=3).value == [1.0, '#NUM!', '#NUM!'] + assert mo.recurrence(1e308, "ROUNDUP({prev} * 10, 0)", periods=3).value == \ + [1e308, '#NUM!', '#NUM!'] + assert mo.recurrence(-8.0, "POWER({prev}, 1 / 3)", periods=2).value == [-8.0, -2.0] + + +class TestInsideALoop: + def test_an_element_wise_function_runs_period_by_period(self): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with m: + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + d.bal = mo.lag(d.r, fill=10.0) * 1.37 + d.cap = mo.MIN(d.bal, 15.0) + d.r = ROUNDUP(d.cap, 0) + d.c = CEILING(d.bal, 5.0) + d.s = SQRT(d.c) + assert d.bal.value == pytest.approx([13.7, 19.18, 20.55]) + assert d.r.value == pytest.approx([14.0, 15.0, 15.0]) + assert d.c.value == pytest.approx([15.0, 20.0, 25.0]) + assert d.s.value == pytest.approx([15.0 ** 0.5, 20.0 ** 0.5, 5.0]) + + def test_the_reduction_is_refused_and_points_past_the_loop(self): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with pytest.raises(ValueError, match="PRODUCT reads a whole series"): + with m: + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + d.bal = mo.lag(d.x, fill=10.0) * 1.37 + d.total = PRODUCT(d.bal) + d.x = d.bal - d.total diff --git a/tests/test_minmax_elementwise.py b/tests/test_minmax_elementwise.py index c54ced3..80a32c3 100644 --- a/tests/test_minmax_elementwise.py +++ b/tests/test_minmax_elementwise.py @@ -127,13 +127,14 @@ def test_a_reduction_as_an_operand_is_a_scalar(self): class TestUnitsAndErrors: - def test_a_shared_unit_is_kept_and_a_mixed_one_dropped(self): + def test_a_shared_unit_is_kept_and_a_mixed_one_refused(self): a = mo.Variable([3.0, 8.0, 5.0], unit="$") b = mo.Variable([4.0, 4.0, 4.0], unit="$") c = mo.Variable([4.0, 4.0, 4.0], unit="hr") assert str(mo.MIN(a, b)._unit) == "$" assert str(mo.MAX(a, 0.0)._unit) == "$" - assert mo.MIN(a, c)._unit is None + with pytest.raises(ValueError, match="mo.MIN takes its values in one unit"): + mo.MIN(a, c) def test_an_error_cell_is_the_period_s_result(self): e = mo.Variable([2.0, 2.0, 2.0]) / mo.Variable([1.0, 0.0, 2.0]) diff --git a/tests/test_multi_variable.py b/tests/test_multi_variable.py index 3cef460..2fbb6d2 100644 --- a/tests/test_multi_variable.py +++ b/tests/test_multi_variable.py @@ -570,3 +570,175 @@ def compute(self, code=7): n = Named.__shell__(code=99, description="d") assert n.component_names == [] assert n.description == "d" + + +class TestAdoptionRefusals: + """A container assigned under a second name inside its own tree — + ``model.m = model`` — used to recurse until Python gave up. + Refused with the alias to write, spelled in the author's own names; + the container is untouched.""" + + def test_a_container_cannot_hold_itself(self): + m = Model("m", default_grain="year", default_start="2026", default_periods=2) + with pytest.raises(ValueError, match=r"cannot hold itself.*`with m as alias:`"): + m.alias = m + assert "alias" not in m._components + assert type(m) is Model # not demoted on the way to the refusal + + def test_a_container_cannot_hold_its_ancestor(self): + m = Model("m", default_grain="year", default_start="2026", default_periods=2) + m.sub = MultiVariable("sub") + with pytest.raises( + ValueError, match=r"`m\.sub\.back = m`: .*one of its ancestors.*`with m as back:`" + ): + m.sub.back = m + assert "back" not in m.sub._components + + def test_the_hint_spells_the_names_the_author_bound(self): + """``mo.Model('m')`` bound as ``model``: the hint says ``model``; + a holder is spelled from the highest bound ancestor + (``model.sub``, not the block's alias ``s``), an unattached + container by its own name.""" + model = Model("m", default_grain="year", default_start="2026", default_periods=2) + with model: + model.sub = MultiVariable("sub") + with model.sub as s: + with pytest.raises( + ValueError, match=r"`model\.sub\.back = model`: .*`with model as back:`" + ): + s.back = model + with MultiVariable("Debt", loop=True) as debt: + with pytest.raises(ValueError, match=r"`debt\.me = debt`: .*`with debt as me:`"): + debt.me = debt + + def test_the_same_name_points_at_a_plain_block(self): + m = Model("m") + with pytest.raises(ValueError, match=r"`with m:`"): + m.m = m + + def test_a_missing_component_says_how_to_assign_it(self): + m = Model("m") + m.sub = MultiVariable("sub") + with pytest.raises( + AttributeError, + match=r"has no component 'rate'.*`m\.sub\.rate = mo\.Variable\(\.\.\.\)`.*loop=True", + ): + m.sub.rate + + def test_the_missing_component_hint_is_python(self): + """The display name ('Income Statement') and the declared name + ('m' bound as ``model``) are never the hint's spelling.""" + with MultiVariable("Income Statement") as income: + with pytest.raises(AttributeError, match=r"`income\.revenue = mo\.Variable"): + income.revenue + model = Model("m") + with pytest.raises(AttributeError, match=r"`model\.revenue = mo\.Variable"): + model.revenue + # A probe with a default still answers the default. + assert getattr(model, "revenue", None) is None + + def test_the_hint_survives_the_line_it_points_at(self): + """A comprehension's variable is gone before the line the hint + says to write, so the hint is spelled from the model (``model.b``); + a global the frame's own parameter shadows is not the model's + name there; a setting of the container is never offered as a row.""" + model = Model("m", default_grain="year", default_start="2026", default_periods=2) + model.a = MultiVariable("a") + model.b = MultiVariable("b") + model.a.revenue = Variable([1.0, 2.0]) + with pytest.raises(AttributeError, match=r"`model\.b\.revenue = mo\.Variable"): + [seg.revenue for seg in (model.a, model.b)] + with pytest.raises(AttributeError, match=r"`model\.b\.revenue = mo\.Variable"): + (lambda q: q.revenue)(model.b) + + base = model + other = Model("other") + other.b = MultiVariable("b") + + def read(model): # the parameter shadows the name ``model`` + return base.b.revenue + + with pytest.raises(AttributeError, match=r"`base\.b\.revenue = mo\.Variable"): + read(other) + with pytest.raises(AttributeError, match=r"has no component 'default_periods'$"): + Model("bare").default_periods + with pytest.raises(AttributeError, match=r"has no component 'default_zone'$"): + Model("bare").default_zone + + +class TestAHorizonByAssignment: + @pytest.fixture(autouse=True) + def _no_model_noted(self): + # The model last given a horizon is process-wide state: start from + # none, so a model an earlier test built cannot answer for this one. + from dataclasses import replace + + from modeleon.core import mv_context + + ctx = mv_context._loop_context + token = ctx.set(replace(ctx.get(), windowed=None)) + yield + ctx.reset(token) + + def test_a_model_given_its_horizon_by_assignment_is_the_window_time_finds(self): + """``m.default_periods = 3`` marks the model last given a horizon, + as the constructor does: a line reading ``mo.time`` outside the + model's ``with`` block (its lines written without the block) finds + the window there (it used to raise "there is no model window here").""" + import modeleon as mo + + m = Model("m") + m.default_grain = "quarter" + m.default_periods = 3 + m.default_start = "2031-Q2" + m.pnl = MultiVariable("pnl") + m.pnl.days = mo.time.days + assert list(m.pnl.days.value) == [91, 92, 92] + + def test_a_refused_change_of_horizon_notes_nothing(self): + """A model whose lines already read the window refuses a new + horizon; the refused model is not the one ``mo.time`` finds next.""" + import modeleon as mo + + m = Model("m", default_grain="month", default_start="2026-01", default_periods=3) + m.pnl = MultiVariable("pnl") + m.pnl.days = mo.time.days + other = Model("other", default_grain="month", default_start="2031-05", default_periods=2) + with pytest.raises(ValueError): + m.default_periods = 99 + x = MultiVariable("x") + x.d = mo.time.days + assert list(x.d.value) == [31, 30] # other's window, not m's + + +class TestAPlainValueUnderAComponentName: + """A line overwritten by a plain number keeps its name: removing it, + deleting it or moving its container works as for a line.""" + + def _model(self): + m = Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.inputs = MultiVariable("Inputs") + m.inputs.rate = Variable(0.05, display_name="rate") + m.inputs.a = Variable([1.0, 2.0, 3.0, 4.0], display_name="a") + m.y = MultiVariable("Y", default_grain="month", default_start="2026-03", + default_periods=4) + m.inputs.rate = 0.05 + return m + + def test_remove(self): + m = self._model() + assert m.inputs.remove("rate") == 0.05 + assert "rate" not in m.inputs._component_names + + def test_del(self): + m = self._model() + del m.inputs.rate + assert "rate" not in m.inputs._component_names + + def test_its_container_moved(self): + m = self._model() + inputs = m.remove("inputs") + m.y.inputs = inputs + assert m.y.inputs.a.time.start == "2026-03" + assert m.y.inputs.a.id == "m.y.inputs.a" diff --git a/tests/test_operator_fast_paths.py b/tests/test_operator_fast_paths.py new file mode 100644 index 0000000..e2354e8 --- /dev/null +++ b/tests/test_operator_fast_paths.py @@ -0,0 +1,209 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The operator fast paths agree with the general rules they shortcut. + +Every operator builds its result by writing fields straight into the +result's ``__dict__``, takes two plain numbers through a short cell rule, +and asks for the right operand's location only when the left one has one. +Each shortcut is exact only while an assumption holds; these tests pin +the assumptions and compare each shortcut with the long way round. +""" + +from __future__ import annotations + +import enum +import inspect +import math +import operator +from datetime import date, datetime, timedelta +from decimal import Decimal +from fractions import Fraction + +import pytest + +import modeleon as mo +from modeleon.core.expr import BinOp, VarRef +from modeleon.core.variable import Variable +from modeleon.core.variable_ops import ( + _COMPARISONS, + _VariableArithmetic, + _date_serial_op, +) + + +def _general_cell_rule(a, b, op, safe_divide, op_symbol): + """One cell, the long way: an error marker propagates (the left one + first), a blank reads as 0 (or "" against text in a comparison), a zero + divisor is #DIV/0!, a date computes on its serial number, and anything + that fails is #VALUE!.""" + if isinstance(a, str) and a.startswith('#'): + return a + if isinstance(b, str) and b.startswith('#'): + return b + if a is None: + a = '' if isinstance(b, str) and op_symbol in _COMPARISONS else 0 + if b is None: + b = '' if isinstance(a, str) and op_symbol in _COMPARISONS else 0 + if safe_divide and b == 0: + return '#DIV/0!' + if isinstance(a, (date, timedelta)) or isinstance(b, (date, timedelta)): + return _date_serial_op(a, b, op, op_symbol) + try: + return op(a, b) + except Exception: + return '#VALUE!' + + +class _Small(enum.IntEnum): + ZERO = 0 + ONE = 1 + + +class _Float(float): + pass + + +class _Int(int): + pass + + +_OPS = { + '+': operator.add, '-': operator.sub, '*': operator.mul, '/': operator.truediv, + '//': operator.floordiv, '%': operator.mod, '**': operator.pow, + '==': operator.eq, '!=': operator.ne, '<': operator.lt, '<=': operator.le, + '>': operator.gt, '>=': operator.ge, +} + +# Plain numbers (the fast path) next to everything that must NOT take it: +# bools, int and float subclasses, other numeric types, blanks, text, error +# markers and dates. +_CELLS = [ + 0, 1, -1, 7, 2**63, 0.0, -0.0, 1.5, -2.25, math.inf, -math.inf, math.nan, + True, False, _Small.ZERO, _Small.ONE, _Float(1.5), _Int(3), + Decimal('1.5'), Decimal('0'), Fraction(1, 3), complex(1, 1), + None, '', 'abc', '#DIV/0!', '#N/A', + date(2026, 1, 31), datetime(2026, 1, 1, 12), timedelta(days=2), timedelta(0), +] + + +def _huge_power(a, b) -> bool: + """Whether ``a ** b`` would build an enormous exact number.""" + exact = (int, Decimal, Fraction) + if isinstance(b, exact) and not isinstance(b, bool) and abs(b) > 64: + return True + return isinstance(a, exact) and not isinstance(a, bool) and abs(a) > 10**6 + + +def _same(x, y) -> bool: + if isinstance(x, float) and isinstance(y, float) and math.isnan(x) and math.isnan(y): + return True + if type(x) is int and type(y) is int: + return x == y # 7 ** a date's serial has more digits than repr allows + return type(x) is type(y) and repr(x) == repr(y) + + +@pytest.mark.parametrize('op_symbol', sorted(_OPS)) +@pytest.mark.parametrize('safe_divide', [False, True]) +def test_every_cell_matches_the_general_rule(op_symbol, safe_divide): + op = _OPS[op_symbol] + for a in _CELLS: + for b in _CELLS: + if op_symbol == '**' and _huge_power(a, b): + continue # 7 ** 2**63 is a big number, not a test + got = _VariableArithmetic._broadcast_operation( + a, b, op, safe_divide=safe_divide, op_symbol=op_symbol) + want = _general_cell_rule(a, b, op, safe_divide, op_symbol) + assert _same(got, want), (op_symbol, safe_divide, a, b, got, want) + + +def test_lists_broadcast_through_the_same_rule(): + left = [1, 2.0, None, '#N/A', 0, True] + right = [0, 0.0, 3, 4, -0.0, 2] + got = _VariableArithmetic._broadcast_operation( + left, right, operator.truediv, safe_divide=True, op_symbol='/') + want = [_general_cell_rule(a, b, operator.truediv, True, '/') + for a, b in zip(left, right)] + assert all(_same(g, w) for g, w in zip(got, want)), (got, want) + + +def _all_subclasses(cls): + for sub in cls.__subclasses__(): + yield sub + yield from _all_subclasses(sub) + + +def _data_descriptor(cls, name) -> bool: + try: + attr = inspect.getattr_static(cls, name) + except AttributeError: + return False + return hasattr(type(attr), '__set__') or hasattr(type(attr), '__delete__') + + +def test_no_operator_field_is_a_descriptor(): + """A field written straight into ``__dict__`` must be a plain field: a + property of that name would be read instead of what was written.""" + m = mo.Model('m', default_grain='month', default_start='2026-01', default_periods=3) + m.s = mo.MultiVariable('S') + m.s.a = mo.Variable([1.0, 2.0, 3.0]) + m.s.part = mo.Variable([5.0], start='2026-02', grain='month', extend=mo.zero()) + m.s.when = mo.time.start + results = [m.s.a + m.s.part, m.s.a * 2, 2 - m.s.a, m.s.a > 1, -m.s.a, m.s.a ** 2, + m.s.when + 3] + fields = {name for r in results for name in vars(r)} + assert '_first_day' in fields # the date-aligned result was built + for cls in [Variable, *_all_subclasses(Variable)]: + shadowed = sorted(n for n in fields if _data_descriptor(cls, n)) + assert not shadowed, (cls.__qualname__, shadowed) + + +def test_no_variable_class_intercepts_attribute_writes(): + """The ``expr`` setter writes its fields straight into ``__dict__`` for + every Variable class, so none may observe writes on its own.""" + for cls in _all_subclasses(Variable): + assert cls.__setattr__ is Variable.__setattr__, cls.__qualname__ + + +def test_the_expr_setter_fills_a_subclass_like_a_variable(): + class Line(mo.Variable): + pass + + m = mo.Model('m', default_grain='month', default_start='2026-01', default_periods=3) + m.s = mo.MultiVariable('S') + m.s.a = mo.Variable([1.0, 2.0, 3.0]) + m.s.days = mo.time.days + node = BinOp('*', VarRef(m.s.a), VarRef(m.s.days)) + plain, sub = mo.Variable(), Line() + plain._set_expr(node) + sub.expr = node + for v in (plain, sub): + assert v._expr is node + assert v._dependency_refs == [m.s.a, m.s.days] + assert v._reads_time is True + assert v._awaits is None + sub.expr = None + assert sub._dependency_refs == [] and sub._reads_time is False + + +def test_an_unlocated_left_side_never_asks_the_right_side(): + """``_align_by_date`` answers ``None`` for an unlocated left side before + it looks the right side up — the same answer it gave when it looked.""" + stub = mo.Model('stub', default_grain='quarter', default_start=date(2026, 2, 1), + default_periods=2) + stub.s = mo.MultiVariable('S') + stub.s.days = mo.time.days + whole = mo.Model('whole', default_grain='quarter', default_start='2026-Q1', + default_periods=2) + whole.s = mo.MultiVariable('S') + whole.s.days = mo.time.days + floating = mo.Variable([1.0, 2.0]) + align = _VariableArithmetic._align_by_date + assert stub.s.days.time.first_day is not None + assert floating.time is None + assert align(floating, stub.s.days, floating._value, stub.s.days._value) is None + assert align(stub.s.days, floating, stub.s.days._value, floating._value) is None + assert (floating + stub.s.days).value == [a + b for a, b in + zip([1.0, 2.0], stub.s.days.value)] + # Both located, first periods on different days: refused either way round. + for left, right in ((stub.s.days, whole.s.days), (whole.s.days, stub.s.days)): + with pytest.raises(ValueError, match='do not line up'): + left + right diff --git a/tests/test_passthrough_hygiene.py b/tests/test_passthrough_hygiene.py new file mode 100644 index 0000000..f92008f --- /dev/null +++ b/tests/test_passthrough_hygiene.py @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Passthrough hygiene: the Excel renderer writes ``=NAME(args)`` for every +name the function registry passes through, so every such name is a +function an author can call — exported from ``modeleon`` and +``modeleon.functions``, the same object in both — unless it is registered +``bare``: written as given, with no helper of its own (``SUMPRODUCT``, +what ``SUM(a * b)`` over two rows is written as; ``NA()``, the not-found +value an ``XLOOKUP`` is written with). A bare name that gained a helper +is a stale flag, and a registered name without a helper is a formula the +renderer can write but an author cannot reach. +""" + +from __future__ import annotations + +import pytest + +import modeleon as mo +from modeleon import functions +from modeleon.functions.registry import names, passthrough_names, spec + + +def _excel_cased(exported): + return {n for n in exported if n.isupper()} + + +@pytest.mark.parametrize("name", sorted(passthrough_names())) +def test_every_passthrough_name_has_an_exported_helper_unless_bare(name): + if spec(name).bare: + assert not hasattr(functions, name) and not hasattr(mo, name), ( + f"{name} has a helper now: drop bare=True from its register() line") + return + assert name in functions.__all__ and name in mo.__all__, name + assert callable(getattr(functions, name)), name + assert getattr(mo, name) is getattr(functions, name), name + + +def test_a_name_written_some_other_way_has_a_helper_too(): + # CONCAT is written as the ``&`` operator. + others = names() - passthrough_names() + assert others == {"CONCAT"} + for name in others: + assert not spec(name).bare and name in mo.__all__, name + + +def test_the_package_exports_exactly_the_registered_helpers(): + registered = {n for n in names() if not spec(n).bare} + assert _excel_cased(functions.__all__) == registered + assert _excel_cased(mo.__all__) == registered + + +def test_the_bare_names_are_the_ones_without_a_helper(): + assert {n for n in names() if spec(n).bare} == {"SUMPRODUCT", "NA"} diff --git a/tests/test_range_node.py b/tests/test_range_node.py new file mode 100644 index 0000000..9a7d16b --- /dev/null +++ b/tests/test_range_node.py @@ -0,0 +1,628 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A range is read once. + +Every call over the same lines — the rank of each record among its peers, +a ``SUMIFS`` per record over the records' fields, a ``MATCH`` per record — +reads the range once and shares it: one AST node that every call's formula +holds, the cells, their names in the source, the unit they share and what +is derived from them (the numbers sorted, the cells indexed by value). The +workbook writes the range once per sheet. A call reads the lines again when +one holds another value — replaced, or a list changed in place — or +another unit or path than it held when read. + +The scale tests count the work, never the clock: a table twice as long +costs twice the work, not four times. +""" + +import copy +import datetime as dt +import gc +import pickle + +import pytest +from openpyxl import load_workbook + +import modeleon as mo +from modeleon.compile.excel.renderer import ExcelRenderer +from modeleon.core.expr import ListExpr +from modeleon.functions import criteria, lookup, statistics + + +class Comp(mo.MultiVariableClass): + def compute(self, multiple, rev=None): + self.multiple = mo.Variable(multiple, display_name="Multiple", unit="x") + if rev is not None: + self.rev = mo.Variable(rev, display_name="Rev", unit="USD") + + +class Tx(mo.MultiVariableClass): + def compute(self, region, amount, day): + self.region = mo.Variable(region, display_name="Region") + self.amount = mo.Variable(amount, display_name="Amount", unit="USD") + self.day = mo.Variable(day, display_name="Day") + + +def _model(): + return mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + + +def _comps(m, n): + m.comps = mo.MultiVariable("Comps", excel_props={"tab": True, "orient": "records"}) + with m.comps as c: + for k in range(n): + setattr(c, f"r{k}", Comp(multiple=float(k * 7 % n), rev=[k, 2.0 * k, n - k], + display_name=f"R{k}")) + return [m.comps._component(f"r{k}") for k in range(n)] + + +def _txs(m, n, groups): + m.tx = mo.MultiVariable("Tx", excel_props={"tab": True, "orient": "records"}) + with m.tx as t: + for k in range(n): + setattr(t, f"x{k}", Tx(region=f"G{k % groups}", amount=float(k % 13), + day=dt.date(2026, 1, 1 + k % 28), display_name=f"x{k}")) + return [m.tx._component(f"x{k}") for k in range(n)] + + +def _read_by_each_call(monkeypatch): + """From here on every call reads its range itself, as before a range + was read once: the answers to compare with.""" + monkeypatch.setattr(criteria, "_shared", lambda arg: None) + monkeypatch.setattr(statistics, "_shared", lambda arg: None) + + +@pytest.fixture +def counted(monkeypatch): + """The work, by kind: the cells read off a line, the cells a statistic + counts, the cells a criteria or a lookup tests, the cells a renderer + looks up for a range, the lists of lines walked for their lines.""" + work = {"cells": 0, "counted": 0, "tested": 0, "rendered": 0, "walked": 0} + + real_cells = criteria._cells + + def cells(var, raw): + work["cells"] += 1 + return real_cells(var, raw) + + real_counted = statistics._counted + + def counted_(cells_): + work["counted"] += len(cells_) + return real_counted(cells_) + + def testing(real): + def make(*args, **kwargs): + test = real(*args, **kwargs) + + def tested(v): + work["tested"] += 1 + return test(v) + return tested + return make + + real_cell_of = ExcelRenderer._cell_of + + def cell_of(self, *args, **kwargs): + work["rendered"] += 1 + return real_cell_of(self, *args, **kwargs) + + monkeypatch.setattr(criteria, "_cells", cells) + monkeypatch.setattr(statistics, "_counted", counted_) + monkeypatch.setattr(criteria, "_matcher", testing(criteria._matcher)) + monkeypatch.setattr(lookup, "_exact_test", testing(lookup._exact_test)) + real_walk = ListExpr.iter_refs + + def walked(self): + work["walked"] += 1 + return real_walk(self) + + monkeypatch.setattr(ExcelRenderer, "_cell_of", cell_of) + monkeypatch.setattr(ListExpr, "iter_refs", walked) + return work + + +def _rows(path, sheet): + return [[c.value for c in row] for row in load_workbook(path)[sheet].iter_rows()] + + +# ══════════════════════════════════════════════════════════════════════ +# The work grows with the table, not with its square +# ══════════════════════════════════════════════════════════════════════ + + +class TestScale: + def _ranked(self, counted, n, tmp_path): + m = _model() + with m: + recs = _comps(m, n) + peers = [r.multiple for r in recs] + for k in counted: + counted[k] = 0 + with m.comps: + for r in recs: + r.rank = mo.RANK_EQ(r.multiple, peers).set_display_name("Rank") + built = dict(counted) + counted["rendered"] = counted["walked"] = 0 + m.to_excel(tmp_path / f"rank{n}.xlsx") + return built, (counted["rendered"], counted["walked"]) + + def test_a_rank_per_record_reads_and_writes_its_range_once(self, counted, tmp_path): + (small, written_small), (large, written_large) = ( + self._ranked(counted, 60, tmp_path), self._ranked(counted, 120, tmp_path)) + # The range is read once: its cells once, counted and sorted once. + assert (small["cells"], large["cells"]) == (60, 120) + assert (small["counted"], large["counted"]) == (60, 120) + # Written once per sheet: each peer's cell looked up once, and the + # list walked once for what its lines read. + assert (written_small, written_large) == ((60, 1), (120, 1)) + + def test_a_rank_per_record_and_year_sorts_each_year_once(self, counted): + def ranked(n): + m = _model() + with m: + recs = _comps(m, n) + revs = [r.rev for r in recs] + counted["counted"] = 0 + with m.comps: + for r in recs: + r.rank = mo.RANK_EQ(r.rev, revs).set_display_name("Rank") + return counted["counted"] + + assert (ranked(40), ranked(80)) == (3 * 40, 3 * 80) + + def test_a_sumifs_per_record_tests_only_the_cells_of_its_key(self, counted, tmp_path): + def summed(n): + m = _model() + with m: + recs = _txs(m, n, groups=n // 4) + amounts, regions = [r.amount for r in recs], [r.region for r in recs] + counted["tested"] = counted["cells"] = 0 + with m.tx: + for r in recs: + r.total = mo.SUMIFS(amounts, regions, r.region).set_display_name("Tot") + built = dict(counted) + counted["rendered"] = 0 + m.to_excel(tmp_path / f"sumifs{n}.xlsx") + return built, counted["rendered"] + + (small, written_small), (large, written_large) = summed(60), summed(120) + # The first call tests every cell, as an equality alone does; each + # after tests the four cells of its key. Each line is read once (its + # region once more, as the criteria of its own call). + assert (small["tested"], large["tested"]) == (60 + 4 * 59, 120 + 4 * 119) + assert (small["cells"], large["cells"]) == (3 * 60, 3 * 120) + assert (written_small, written_large) == (2 * 60, 2 * 120) + + def test_a_match_per_record_tests_only_the_cells_of_its_key(self, counted): + def matched(n): + m = _model() + with m: + recs = _txs(m, n, groups=n) + regions = [r.region for r in recs] + counted["tested"] = 0 + with m.tx: + for r in recs: + r.at = mo.MATCH(r.region, regions, 0).set_display_name("At") + r.back = mo.XLOOKUP(r.region, regions, regions).set_display_name("Back") + assert [r.at.value for r in recs] == list(range(1, n + 1)) + return counted["tested"] + + # The first call tests every cell; each after, the one of its key. + assert (matched(60), matched(120)) == (60 + 2 * 60 - 1, 120 + 2 * 120 - 1) + + def test_an_equality_alone_reads_every_cell_and_a_second_indexes_them(self, counted): + lines = [mo.Variable(f"t{k}") for k in range(40)] + counted["tested"] = 0 + first = mo.COUNTIF(lines, "t7") + assert counted["tested"] == 40 + shared = criteria._shared(lines) + assert "index" not in shared.kept + pattern = mo.COUNTIF(lines, "t1*") # a pattern, which no index answers + found = mo.MATCH("t1?", lines, 0) + assert "index" not in shared.kept + counted["tested"] = 0 + second = mo.MATCH("T9", lines, 0) + assert "index" in shared.kept + assert counted["tested"] == 1 + assert (first.value, pattern.value, found.value, second.value) == (1, 11, 11, 10) + + def test_a_median_per_record_is_worked_out_once(self, counted): + def mid(n): + m = _model() + with m: + recs = _comps(m, n) + peers = [r.multiple for r in recs] + counted["counted"] = 0 + with m.comps: + for r in recs: + r.gap = (r.multiple - mo.MEDIAN(peers)).set_display_name("Gap") + return counted["counted"] + + assert (mid(60), mid(120)) == (60, 120) + + +# ══════════════════════════════════════════════════════════════════════ +# One node, shared by every call over the same lines +# ══════════════════════════════════════════════════════════════════════ + + +class TestOneNode: + def test_the_calls_hold_one_node_and_a_new_list_of_the_same_lines_finds_it(self): + m = _model() + with m: + recs = _comps(m, 5) + peers = [r.multiple for r in recs] + with m.comps: + for r in recs: + r.rank = mo.RANK_EQ(r.multiple, list(peers)) + nodes = {id(r.rank._expr.args[1]) for r in recs} + assert len(nodes) == 1 + again = mo.PERCENTILE_INC(tuple(peers), 0.5) + assert again._expr.args[0] is recs[0].rank._expr.args[1] + other = mo.RANK_EQ(recs[0].multiple, peers[::-1]) + assert other._expr.args[1] is not again._expr.args[0] + + def test_a_copy_or_a_pickle_takes_the_formula_and_reads_the_lines_again(self): + m = _model() + with m: + recs = _comps(m, 4) + peers = [r.multiple for r in recs] + with m.comps: + for r in recs: + r.rank = mo.RANK_EQ(r.multiple, peers) + for twin in (copy.deepcopy(recs[1].rank), pickle.loads(pickle.dumps(recs[1].rank))): + node = twin._expr.args[1] + assert node.__dict__.get("_lines") is None + assert [item.var.value for item in node.items] == [0.0, 3.0, 2.0, 1.0] + assert twin.value == recs[1].rank.value == 1 + + def test_a_range_no_formula_holds_any_more_is_forgotten(self): + m = _model() + with m: + recs = _comps(m, 4) + key = tuple(id(r.multiple) for r in recs) + rank = mo.RANK_EQ(recs[0].multiple, [r.multiple for r in recs]) + assert key in criteria._LISTS + del rank + gc.collect() + assert key not in criteria._LISTS + + +class TestTheWriterWalksAListOnce: + def test_a_line_off_the_window_in_a_shared_list_is_named_for_each_formula( + self, tmp_path): + m = _model() + # Read by date, as ``a + b`` reads it: a year later, no rule to + # continue it — refused when the call is built. + late = mo.Variable([5.0, 6.0, 7.0], start="2027", grain="year") + # Two years from the sheet's first, continued by zero: read by date, + # but a line the book does not write, whose values would stand in + # the sheet's columns — refused by the export, for each formula. + odd = mo.Variable([5.0, 6.0], start="2026", grain="year", extend=mo.zero()) + with m: + m.s = mo.MultiVariable("S") + m.s.a = mo.Variable([1.0, 2.0, 3.0]) + m.s.b = mo.Variable([3.0, 1.0, 2.0]) + with pytest.raises(ValueError, match="does not cover the combined window"): + mo.RANK_EQ(m.s.a, [m.s.a, m.s.b, late]) + lines = [m.s.a, m.s.b, odd] + m.s.r1 = mo.RANK_EQ(m.s.a, lines) + m.s.r2 = mo.RANK_EQ(m.s.b, lines) + with pytest.raises(ValueError, match="time window of its own") as refused: + m.to_excel(tmp_path / "odd.xlsx") + assert "(m.s.r1) reads" in str(refused.value) + assert "(m.s.r2) reads" in str(refused.value) + + def test_a_list_met_again_on_its_own_way_down_is_not_taken_as_walked(self): + from modeleon.compile.excel.writer import _reads_unwritten_off_window + from modeleon.core.expr import FuncCall, VarRef + + a = mo.Variable() + inner = ListExpr([VarRef(a)]) + a._set_expr(FuncCall("MEDIAN", [inner])) + outer = ListExpr([VarRef(a)]) + clean = {} + assert _reads_unwritten_off_window(FuncCall("F", [outer]), None, {}, set(), + clean) is None + assert clean == {} + b = mo.Variable() + b._set_expr(FuncCall("MEDIAN", [ListExpr([VarRef(mo.Variable(1.0))])])) + shared = ListExpr([VarRef(b)]) + assert _reads_unwritten_off_window(FuncCall("F", [shared]), None, {}, set(), + clean) is None + assert shared in clean.values() + + +# ══════════════════════════════════════════════════════════════════════ +# A line that changed is read again +# ══════════════════════════════════════════════════════════════════════ + + +class TestReadAgain: + def test_a_value_replaced_is_read_again(self): + m = _model() + with m: + recs = _comps(m, 4) # 0, 3, 2, 1 + peers = [r.multiple for r in recs] + assert mo.RANK_EQ(recs[1].multiple, peers).value == 1 + # As a loop closing, or a schedule laid out on adoption, writes it. + recs[0].multiple._value = 9.0 + assert mo.RANK_EQ(recs[1].multiple, peers).value == 2 + assert mo.LARGE(peers, 1).value == 9.0 + assert mo.SUMIF(peers, 9.0).value == 9.0 + assert mo.MATCH(9.0, peers, 0).value == 1 + + def test_a_row_changed_in_place_is_read_again(self): + m = _model() + with m: + m.s = mo.MultiVariable("S") + data = [1, 8.0, 9.0] + m.s.u = mo.Variable(data) # the line holds the caller's list + m.s.k = mo.Variable(8.0) + assert mo.MEDIAN(m.s.u).value == 8.0 + assert mo.COUNTIF(m.s.u, 8.0).value == 1 + assert mo.RANK_EQ(m.s.k, m.s.u).value == 2 + data[1] = 100.0 + assert mo.MEDIAN(m.s.u).value == 9.0 + assert mo.COUNTIF(m.s.u, 8.0).value == 0 + assert mo.RANK_EQ(m.s.k, m.s.u).value == "#N/A" + assert mo.MATCH(100.0, m.s.u, 0).value == 2 + assert mo.COUNTIF(m.s.u, 1).value == 1 + m.s.u.value[0] = True # equal to 1, yet a boolean + assert mo.COUNTIF(m.s.u, 1).value == 0 + assert mo.MEDIAN(m.s.u).value == 54.5 + assert mo.LARGE(m.s.u, 3).value == "#NUM!" + + def test_a_series_changed_in_place_is_read_again(self): + m = _model() + with m: + m.s = mo.MultiVariable("S") + m.s.a = mo.Variable([1.0, 2.0, 3.0]) + m.s.b = mo.Variable([4.0, 5.0, 6.0]) + m.s.c = mo.Variable([7.0, 0.0, 9.0]) + lines = [m.s.a, m.s.b, m.s.c] + assert mo.RANK_EQ(m.s.a, lines).value == [3, 2, 3] + assert mo.MEDIAN(lines).value == [4.0, 2.0, 6.0] + m.s.a.value[0] = 100.0 + assert mo.RANK_EQ(m.s.a, lines).value == [1, 2, 3] + assert mo.MEDIAN(lines).value == [7.0, 2.0, 6.0] + # A cell moved from one line to the next: the same cells in the + # same order, in lines of other lengths — refused as they now + # are, as ``a + b`` refuses them. + m.s.b.value.insert(0, m.s.a.value.pop()) + with pytest.raises(ValueError, match="does not cover the combined window"): + m.s.a + m.s.b + with pytest.raises(ValueError, match="does not cover the combined window"): + mo.MEDIAN(lines) + + def test_a_unit_set_is_read_again(self): + m = _model() + with m: + recs = _comps(m, 3) + peers = [r.multiple for r in recs] + assert str(mo.MEDIAN(peers)._unit) == "x" + recs[2].multiple.unit = "USD" + with pytest.raises(ValueError, match="one unit"): + mo.MEDIAN(peers) + recs[2].multiple.unit = "x" + assert str(mo.LARGE(peers, 1)._unit) == "x" + + def test_a_line_adopted_or_moved_is_named_again(self): + a, b = mo.Variable(1.0), mo.Variable(2.0) + first = mo.RANK_EQ(a, [a, b]) + assert first.source_code == f"RANK_EQ({a.path.leaf}, [{a.path.leaf}, {b.path.leaf}])" + m = _model() + with m: + m.s = mo.MultiVariable("S") + m.s.a, m.s.b = a, b + second = mo.RANK_EQ(m.s.a, [a, b]) + assert second.source_code == "RANK_EQ(a, [s.a, s.b])" + + def test_a_line_moved_to_another_container_is_named_again(self): + m = _model() + with m: + m.s = mo.MultiVariable("S") + m.t = mo.MultiVariable("T") + m.s.a, m.s.b = mo.Variable(1.0), mo.Variable(2.0) + lines = [m.s.a, m.s.b] + assert mo.LARGE(lines, 1).source_code == "LARGE([s.a, s.b], 1)" + line = m.s.remove("a") + m.t.a = line + assert mo.LARGE(lines, 1).source_code == "LARGE([t.a, s.b], 1)" + + def test_a_list_changed_is_another_range(self): + m = _model() + with m: + recs = _comps(m, 4) + peers = [r.multiple for r in recs] + assert mo.LARGE(peers, 1).value == 3.0 + peers.append(mo.Variable(7.0)) + assert mo.LARGE(peers, 1).value == 7.0 + peers[-1] = mo.Variable(-1.0) + assert mo.SMALL(peers, 1).value == -1.0 + + def test_empty_text_typed_in_then_given_a_formula(self): + from modeleon.core.expr import Literal + a, blank = mo.Variable(1.0), mo.Variable("") + line = [a, blank] + assert mo.COUNTIF(line, "=").value == 1 # typed in: a blank cell + blank.expr = Literal("") + assert mo.COUNTIF(line, "=").value == 0 # a formula's empty text + + def test_the_first_line_typed_again(self): + a, b = mo.Variable(1), mo.Variable(2) + assert mo.LARGE([a, b], 1).value_type == "int" + a.value_type = "float" + assert mo.LARGE([a, b], 1).value_type == "float" + + +class TestALoopClosing: + """A call made while a loop is open reads provisional rows; when the + loop closes the call runs again over the same lines, now holding their + values — the range is read again, as a range read by each call is.""" + + @staticmethod + def _model(): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=4) + with m: + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + d.a = mo.Variable([1.0, 5.0, 2.0, 4.0]) + d.c = mo.lag(d.b, fill=0.0) + 1.0 + d.mid = mo.MEDIAN([d.a, d.c]) + d.rank = mo.RANK_EQ(d.a, [d.a, d.c]) + d.top = mo.LARGE([d.a, d.c], 1) + d.b = d.a + d.c + return m + + def test_the_rows_are_read_again(self, monkeypatch): + shared = self._model() + _read_by_each_call(monkeypatch) + plain = self._model() + assert shared.d.c.value == plain.d.c.value == [1.0, 3.0, 9.0, 12.0] + for name in ("mid", "rank", "top"): + assert getattr(shared.d, name).value == getattr(plain.d, name).value, name + assert shared.d.mid.value == [1.0, 4.0, 5.5, 8.0] + assert shared.d.rank.value == [1, 1, 2, 2] + assert shared.d.top.value == [1.0, 5.0, 9.0, 12.0] + + +# ══════════════════════════════════════════════════════════════════════ +# An index answers as reading every cell does +# ══════════════════════════════════════════════════════════════════════ + +_CELLS = [ + 10, 10.0, 10 + 1e-15, 10 + 1e-13, -0.0, 0, 0.1 + 0.2, 0.3, 1e300, 1e-300, 3, 2.5, + dt.date(2026, 1, 31), dt.datetime(2026, 1, 31, 12, 0), dt.timedelta(days=3), + True, False, "10", "1e1", "10%", " 10 ", "North", "north", "NORTH", "nörth", + "ı", "I", "i", "İ", "ß", "SS", "ss", "ſt", "st", "\u212a", "k", "Σ", "σ", "ς", + "\u0432", "\u1c80", "\u0412", "TRUE", "true", "x*", "x?y", "~", "#N/A", "#text", "", None, +] +_CRITERIA = [ + 10, "10", "=10", 10.0, "1e1", True, "TRUE", "true", False, "north", "NORTH", "i", + "I", "ı", "İ", "ss", "ß", "st", "k", "σ", "ς", "\u0432", "\u0412", dt.date(2026, 1, 31), + ("=", dt.date(2026, 1, 31)), "#N/A", "", "=", "<>", ">5", "<>north", "n*", "x~*", + 0.3, 0.1 + 0.2, -0.0, 3, "3", None, 1e300, +] + + +def _answers(lines, amounts): + out = [] + for crit in _CRITERIA: + out.append(("SUMIF", crit, mo.SUMIF(lines, crit, amounts).value)) + out.append(("COUNTIF", crit, mo.COUNTIF(lines, crit).value)) + out.append(("AVERAGEIF", crit, mo.AVERAGEIF(lines, crit, amounts).value)) + out.append(("SUMIFS", crit, mo.SUMIFS(amounts, lines, crit, amounts, ">0").value)) + out.append(("COUNTIFS", crit, mo.COUNTIFS(amounts, ">0", lines, crit).value)) + out.append(("MAXIFS", crit, mo.MAXIFS(amounts, lines, crit).value)) + for key in _CELLS + ["x", "nörth", "NöRTH"]: + if key is None: + continue + line = mo.Variable(key) + out.append(("MATCH", key, mo.MATCH(line, lines, 0).value)) + for mode in (0, 2): + for search in (1, -1): + out.append(("XLOOKUP", (key, mode, search), + mo.XLOOKUP(line, lines, amounts, "none", + match_mode=mode, search_mode=search).value)) + return out + + +def _same(a, b): + return a == b or (isinstance(a, float) and isinstance(b, float) and a != a and b != b) + + +class TestTheIndexAnswersAsEveryCell: + def test_criteria_and_lookups_over_awkward_cells(self, monkeypatch): + lines = [mo.Variable(v) for v in _CELLS] + amounts = [mo.Variable(float(k + 1)) for k in range(len(_CELLS))] + indexed = _answers(lines, amounts) + _read_by_each_call(monkeypatch) + read = _answers(lines, amounts) + assert len(indexed) == len(read) + wrong = [(i, r) for i, r in zip(indexed, read) if not _same(i[2], r[2])] + assert not wrong + + def test_a_letter_folds_as_the_text_pattern_compares_it(self): + """Two letters a pattern of one matches fold alike — every letter + the pattern engine compares without regard to case.""" + _sre = pytest.importorskip("_sre") + casefix = pytest.importorskip("re._casefix") + import sys + lower = {} + groups = {} + for i in range(sys.maxunicode + 1): + if 0xD800 <= i <= 0xDFFF: + continue + low = _sre.unicode_tolower(i) + lower[i] = low + groups.setdefault(low, []).append(i) + for p, low in lower.items(): + if not _sre.unicode_iscased(p): + continue + key = criteria._fold(chr(p)) + if key is None: + continue + for same in (low, *casefix._EXTRA_CASES.get(low, ())): + for c in groups.get(same, ()): + other = criteria._fold(chr(c)) + assert other is None or other == key, (chr(p), chr(c)) + + +# ══════════════════════════════════════════════════════════════════════ +# The workbook is the one a range read by each call writes +# ══════════════════════════════════════════════════════════════════════ + + +def _records_model(): + m = _model() + with m: + recs = _comps(m, 8) + peers = [r.multiple for r in recs] + revs = [r.rev for r in recs] + with m.comps: + for r in recs: + r.rank = mo.RANK_EQ(r.multiple, peers).set_display_name("Rank") + r.avg = mo.RANK_AVG(r.multiple, peers, 1).set_display_name("Avg") + r.p = mo.PERCENTILE_INC(peers, 0.25).set_display_name("P25") + r.gap = (r.multiple - mo.MEDIAN(peers)).set_display_name("Gap") + r.year_rank = mo.RANK_EQ(r.rev, revs).set_display_name("YearRank") + r.same = mo.COUNTIF(peers, r.multiple).set_display_name("Same") + r.over = mo.SUMIFS(peers, peers, (">", r.multiple)).set_display_name("Over") + r.at = mo.MATCH(r.multiple, peers, 0).set_display_name("At") + r.back = mo.INDEX(peers, r.at).set_display_name("Back") + r.look = mo.XLOOKUP(r.multiple, peers, peers).set_display_name("Look") + m.s = mo.MultiVariable("Summary", excel_props={"tab": True}) + m.s.top = mo.LARGE(peers, 2) + m.s.mid = mo.MEDIAN(revs) + return m + + +class TestTheSameWorkbook: + def test_every_cell(self, monkeypatch, tmp_path): + _records_model().to_excel(tmp_path / "shared.xlsx") + _read_by_each_call(monkeypatch) + _records_model().to_excel(tmp_path / "each.xlsx") + shared = load_workbook(tmp_path / "shared.xlsx") + each = load_workbook(tmp_path / "each.xlsx") + assert shared.sheetnames == each.sheetnames + for name in shared.sheetnames: + assert _rows(tmp_path / "shared.xlsx", name) == _rows(tmp_path / "each.xlsx", name) + rows = _rows(tmp_path / "shared.xlsx", "Comps") + header, first, last = rows[1], rows[2], rows[9] + assert first[header.index("Rank")] == "=_xlfn.RANK.EQ(B3, B3:B10)" + assert last[header.index("Rank")] == "=_xlfn.RANK.EQ(B10, B3:B10)" + assert first[header.index("Over (x)")] == '=SUMIFS(B3:B10, B3:B10, ">" & B3)' + assert first[header.index("At")] == "=MATCH(B3, B3:B10, 0)" + + def test_the_source_and_the_units(self, monkeypatch): + shared = _records_model() + _read_by_each_call(monkeypatch) + each = _records_model() + for a, b in zip(shared.walk(), each.walk()): + va, vb = a[1], b[1] + if isinstance(va, mo.Variable): + assert va.source_code == vb.source_code, a[0] + assert va.value == vb.value, a[0] + assert str(va._unit) == str(vb._unit), a[0] + assert va.value_type == vb.value_type, a[0] diff --git a/tests/test_records_scale.py b/tests/test_records_scale.py new file mode 100644 index 0000000..0f21a1b --- /dev/null +++ b/tests/test_records_scale.py @@ -0,0 +1,293 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A table of many records is built and written in time linear in its size. + +A container's children live in its ``__dict__``; ``_components`` is a +dict snapshot of them — a copy of every child, fresh on every read. The +adoption check, the path crystallization and every walk of the layout +and the writer read one child at a time through that snapshot, so a +records table of n rows copied the table n times: 10,000 records took +minutes to write. A child is now read by name (``_component``), its +presence asked of a name list whose ``in`` is a set lookup +(``_has_component``), and the snapshot stays what it was for every +other reader — a fresh dict. + +The scale tests count the entries the snapshot copies, never the +clock: a table twice as long must cost twice the work, not four times. +""" + +import copy +import pickle + +import pytest + +import modeleon as mo +from modeleon.core.multi_variable import MultiVariableBase +from modeleon.core.mv_context import ModelStructureWarning + + +class Tx(mo.MultiVariableClass): + def compute(self, day, amount): + self.day = mo.Variable(day, display_name="Day") + self.amount = mo.Variable(amount, display_name="Amount") + + +def _model() -> mo.Model: + return mo.Model("m", default_grain="day", default_start="2026-01-01", default_periods=30) + + +def _fill(m: mo.Model, n: int) -> None: + with m: + m.t = mo.MultiVariable("Tx", excel_props={"orient": "records"}) + with m.t as t: + for k in range(n): + setattr(t, f"x{k}", Tx(day=k % 30 + 1, amount=float(k % 13), + display_name=f"x{k}")) + + +def _table(n: int) -> mo.Model: + m = _model() + _fill(m, n) + return m + + +@pytest.fixture +def copied(monkeypatch): + """How many children the ``_components`` snapshot has copied, summed + over every read.""" + count = [0] + real = MultiVariableBase.__dict__["_components"] + + def read(self): + snapshot = real.fget(self) + count[0] += len(snapshot) + return snapshot + + monkeypatch.setattr(MultiVariableBase, "_components", property(read, real.fset)) + return count + + +class TestScale: + @pytest.mark.parametrize("n", [200, 400]) + def test_filling_a_table_copies_each_node_at_most_once(self, copied, n): + m = _model() + copied[0] = 0 + _fill(m, n) + # A record is three nodes: itself and its two fields. A snapshot + # per child adopted copied the table once per record: n² / 2. + assert copied[0] <= 3 * n, copied[0] + + def test_writing_a_table_twice_as_long_copies_twice_as_much(self, copied, tmp_path): + def write(n): + m = _table(n) + copied[0] = 0 + m.to_excel(tmp_path / f"t{n}.xlsx") + return copied[0] + + small, large = write(200), write(400) + # Linear: about 2. A snapshot per child read: about 4. + assert large <= 2.5 * small, (small, large) + + def test_a_sheet_of_lines_twice_as_long_with_totals_copies_twice_as_much( + self, copied, tmp_path + ): + def write(n): + m = mo.Model("m", default_grain="day", default_start="2026-01-01", + default_periods=10, + default_excel_view=mo.ExcelView(timeline={"totals": ["month"]})) + with m: + m.s = mo.MultiVariable("S") + with m.s as s: + for r in range(n): + setattr(s, f"r{r}", mo.Variable([float(r + k) for k in range(10)], + regrain=mo.up("sum"))) + copied[0] = 0 + m.to_excel(tmp_path / f"s{n}.xlsx") + return copied[0] + + small, large = write(100), write(200) + assert large <= 2.5 * small, (small, large) + + def test_a_walk_copies_each_child_at_most_once(self, copied): + m = _table(300) + copied[0] = 0 + walked = list(m.walk()) + assert len(walked) == 1 + 300 * 3 # the table, its records, their fields + assert copied[0] <= len(walked) + + def test_the_table_is_written_whole(self, tmp_path): + import openpyxl + + path = tmp_path / "t.xlsx" + _table(50).to_excel(path) + ws = openpyxl.load_workbook(path)["Tx"] + rows = [[c.value for c in r][:3] for r in ws.iter_rows()] + assert rows[0] == [None, "Day", "Amount"] + assert rows[1:] == [[f"x{k}", k % 30 + 1, k % 13] for k in range(50)] + + +class TestTheSnapshotStaysASnapshot: + def test_each_read_is_a_fresh_dict_the_container_does_not_share(self): + mv = mo.MultiVariable(a=mo.Variable(1), b=mo.Variable(2)) + snapshot = mv._components + assert snapshot is not mv._components + snapshot["c"] = mo.Variable(3) + del snapshot["a"] + assert list(mv._components) == ["a", "b"] + assert mv._has_component("a") and not mv._has_component("c") + + def test_a_snapshot_iterates_while_the_container_changes(self): + mv = mo.MultiVariable(**{f"v{i}": mo.Variable(i) for i in range(20)}) + seen = [] + for name, var in mv._components.items(): + seen.append(name) + mv.remove(name) + setattr(mv, f"w{name}", mo.Variable(var._value)) + assert seen == [f"v{i}" for i in range(20)] + assert mv._component_order == [f"wv{i}" for i in range(20)] + + def test_a_walk_reads_each_child_when_its_turn_comes(self): + mv = mo.MultiVariable(a=mo.Variable(1), b=mo.Variable(2)) + seen = [] + for name, comp in mv.walk(): + seen.append((name, comp._value)) + if name == "a": + with pytest.warns(ModelStructureWarning): + mv.b = mo.Variable(20) # replaced before its turn + mv.c = mo.Variable(3) # joined after the walk began + assert seen == [("a", 1), ("b", 20)] + + def test_the_helpers_answer_what_the_snapshot_answers(self): + m = _table(40) + m.t.note = 5 # a plain attribute, not a child + for mv in (m, m.t, m.t.x0): + snapshot = mv._components + assert list(snapshot) == mv._component_order + assert all(mv._component(name) is child for name, child in snapshot.items()) + for name in (*snapshot, "missing", "note", "default_grain", "_parent"): + assert mv._has_component(name) == (name in snapshot) + with pytest.raises(KeyError): + m.t._component("note") + + +class TestChildrenComeAndGo: + def test_a_replaced_child_moves_to_the_end_once(self): + mv = mo.MultiVariable(**{f"v{i}": mo.Variable(i) for i in range(20)}) + with pytest.warns(ModelStructureWarning): + mv.v3 = mo.Variable(33) + assert mv._component_order == [f"v{i}" for i in range(20) if i != 3] + ["v3"] + assert mv._component("v3")._value == 33 + + def test_a_removed_child_is_gone_and_its_name_free_again(self): + mv = mo.MultiVariable(**{f"v{i}": mo.Variable(i) for i in range(20)}) + del mv.v5 + gone = mv.remove("v6") + assert gone._owner is None + for name in ("v5", "v6"): + assert not mv._has_component(name) and name not in mv._component_names + with pytest.raises(KeyError): + mv._component(name) + mv.v5 = mo.Variable(55) + assert mv._component_order.count("v5") == 1 and mv._component_order[-1] == "v5" + + def test_a_child_owned_elsewhere_joins_as_a_copy_an_orphan_as_itself(self): + a, b, c = mo.MultiVariable("A"), mo.MultiVariable("B"), mo.MultiVariable("C") + a.rec = Tx(day=1, amount=2.0) + b.rec = a.rec + assert b._component("rec") is not a._component("rec") + assert b._component("rec")._parent is b + orphan = a.remove("rec") + assert not a._has_component("rec") + c.rec = orphan + assert c._component("rec") is orphan and orphan._parent is c + + def test_a_cloned_table_keeps_its_records(self): + m = _table(30) + other = _model() + with other: + other.t = m.t # owned by m: adopted as a clone + assert other.t is not m.t + assert other.t._component_order == m.t._component_order + assert all(other.t._component(n)._parent is other.t for n in other.t._component_order) + assert other.t.x7.day.id == "m.t.x7.day" + + def test_the_bulk_setter_replaces_the_children(self): + # Virtual sheets are filled this way: the old set out, the new in. + mv = mo.MultiVariable(**{f"v{i}": mo.Variable(i) for i in range(20)}) + x, y = mo.Variable(1), mo.Variable(2) + mv._components = {"x": x, "y": y} + assert mv._component_order == ["x", "y"] + assert mv._component("x") is x and mv._has_component("y") + assert not mv._has_component("v0") and "v0" not in mv.__dict__ + assert [name for name, _ in mv.walk()] == ["x", "y"] + + def test_an_order_without_a_name_drops_it_from_the_children(self): + mv = mo.MultiVariable(**{f"v{i}": mo.Variable(i) for i in range(20)}) + order = [f"v{i}" for i in reversed(range(1, 20))] + mv._component_order = order + assert "v0" not in mv._components and not mv._has_component("v0") + with pytest.raises(KeyError): + mv._component("v0") + assert [name for name, _ in mv.walk()] == order + + +class TestTheNameList: + @pytest.fixture + def names_type(self): + from modeleon.core.multi_variable import _ComponentNames + + return _ComponentNames + + def test_every_way_a_container_gets_its_names_keeps_the_lookup(self, names_type): + m = _table(20) + assert type(m.t._component_names) is names_type + m.t._component_order = list(reversed(m.t._component_order)) + assert type(m.t._component_names) is names_type + other = _model() + with other: + other.t = m.t + assert type(other.t._component_names) is names_type + m.t._components = {"x": mo.Variable(1)} + assert type(m.t._component_names) is names_type + + @pytest.mark.parametrize("size", [3, 40]) # scanned, looked up + def test_in_follows_every_change(self, names_type, size): + names = names_type(f"n{i}" for i in range(size)) + same = list(names) + + def agree(): + for probe in (*same, "absent", "n999"): + assert (probe in names) == (probe in same), probe + assert names == same + + changes = [ + lambda s: s.append("a"), + lambda s: s.extend(["b", "c"]), + lambda s: s.insert(0, "d"), + lambda s: s.remove("b"), + lambda s: s.pop(), + lambda s: s.pop(0), + lambda s: s.__setitem__(0, "e"), + lambda s: s.__setitem__(slice(0, 2), ["f", "g", "h"]), + lambda s: s.__delitem__(1), + lambda s: s.__delitem__(slice(0, 2)), + lambda s: s.__iadd__(["i"]), + lambda s: s.__imul__(2), + lambda s: s.sort(), + lambda s: s.reverse(), + lambda s: s.clear(), + lambda s: s.append("j"), + ] + agree() + for change in changes: + change(names) + change(same) + agree() + + def test_a_copy_is_a_list_of_its_own(self, names_type): + names = names_type(f"n{i}" for i in range(40)) + assert "n0" in names and [1] not in names + for other in (copy.copy(names), copy.deepcopy(names), + pickle.loads(pickle.dumps(names))): + assert other == names and type(other) is names_type + other.append("only_here") + assert "only_here" in other and "only_here" not in names diff --git a/tests/test_series_arguments.py b/tests/test_series_arguments.py new file mode 100644 index 0000000..0aad070 --- /dev/null +++ b/tests/test_series_arguments.py @@ -0,0 +1,270 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A series where a function used to take one value is one call per period. + +``ROUND``'s digits, ``EDATE`` / ``EOMONTH``'s months, ``DAYS360``'s +method and every argument of ``PMT``, ``FV`` and ``PV`` may be a line: +the call is one per period, as the spreadsheet writes it in every column +(``=ROUND(C5, C9)``, ``=EDATE(C5, C9)``, ``=PMT(C9 / 100, 12, 1000, 0, +0)``) — a scalar broadcast, lines on other windows read by date and +lines along keys key by key, as ``a + b`` reads them. A rate ``NPV`` +discounts a whole row by, and the guess ``IRR`` / ``XIRR`` start from, +stay one value: a series there is refused by name. +""" + +import shutil +import subprocess +from datetime import date, datetime +from pathlib import Path + +import pytest +from openpyxl import load_workbook + +import modeleon as mo + +pytestmark = pytest.mark.filterwarnings("ignore::UserWarning") + + +def _rows(ws): + out = {} + for row in ws.iter_rows(): + label = row[0].value + if label: + out[str(label)] = [c.value for c in row[1:]] + return out + + +def _row_of(ws): + return {str(row[0].value): row[0].row for row in ws.iter_rows() if row[0].value} + + +def _lines(): + """Three years of inputs on X, a line from 2027 on T (held), keyed + lines on K.""" + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with m: + m.x = mo.MultiVariable("X") + m.x.v = mo.Variable([1.2345, 2.3456, -3.4567], display_name="v") + m.x.d = mo.Variable([0.0, 1.0, 2.0], display_name="d") + m.x.dt = mo.Variable([date(2026, 1, 31), date(2026, 2, 28), date(2026, 3, 31)], + display_name="dt") + m.x.k = mo.Variable([1.0, 2.0, -1.0], display_name="k") + m.x.pct = mo.Variable([5.0, 6.0, 7.0], display_name="pct") + m.x.rt = mo.Variable([0.05, 0.06, 0.07], display_name="rt") + m.x.n = mo.Variable([12.0, 24.0, 36.0], display_name="n") + m.x.amt = mo.Variable([1000.0, 2000.0, 3000.0], display_name="amt") + m.x.eu = mo.Variable([True, False, True], display_name="eu") + m.x.end = mo.Variable([date(2026, 3, 31), date(2026, 3, 31), date(2026, 5, 31)], + display_name="end") + m.t = mo.MultiVariable("T", default_grain="year", default_start="2027", + default_periods=2) + m.t.dg = mo.Variable([3.0, 1.0], display_name="dg", extend=mo.hold()) + m.t.nofill = mo.Variable([3.0, 1.0], display_name="nofill") + return m + + +def _one_by_one(make, *series): + """The scalar call made in each period — what a period's cell computes.""" + return [make(*(s[i] if isinstance(s, list) else s for s in series)).value + for i in range(max(len(s) for s in series if isinstance(s, list)))] + + +#: Each call over X's lines, with the values the scalar call gives period by +#: period. +_CALLS = { + "rnd": (lambda m: mo.ROUND(m.x.v, m.x.d), + lambda m: _one_by_one(mo.ROUND, m.x.v.value, m.x.d.value)), + "rnd2": (lambda m: mo.ROUND(2.71828, m.x.d), + lambda m: _one_by_one(mo.ROUND, 2.71828, m.x.d.value)), + "ed": (lambda m: mo.EDATE(m.x.dt, m.x.k), + lambda m: _one_by_one(mo.EDATE, m.x.dt.value, m.x.k.value)), + "ed2": (lambda m: mo.EDATE(date(2026, 1, 31), m.x.k), + lambda m: _one_by_one(mo.EDATE, date(2026, 1, 31), m.x.k.value)), + "eo": (lambda m: mo.EOMONTH(m.x.dt, m.x.k), + lambda m: _one_by_one(mo.EOMONTH, m.x.dt.value, m.x.k.value)), + "pm": (lambda m: mo.PMT(m.x.pct / 100, 12, 1000), + lambda m: _one_by_one(mo.PMT, [p / 100 for p in m.x.pct.value], 12, 1000)), + "pm2": (lambda m: mo.PMT(0.05, m.x.n, m.x.amt, 0, 1), + lambda m: _one_by_one(mo.PMT, 0.05, m.x.n.value, m.x.amt.value, 0, 1)), + "fv": (lambda m: mo.FV(m.x.pct / 100, 12, -100), + lambda m: _one_by_one(mo.FV, [p / 100 for p in m.x.pct.value], 12, -100)), + "pv": (lambda m: mo.PV(0.05, m.x.n, -100, m.x.amt), + lambda m: _one_by_one(mo.PV, 0.05, m.x.n.value, -100, m.x.amt.value)), + "d360": (lambda m: mo.DAYS360(m.x.dt, m.x.end, m.x.eu), + lambda m: _one_by_one(mo.DAYS360, m.x.dt.value, m.x.end.value, m.x.eu.value)), +} + + +def _model(): + m = _lines() + with m: + for name, (make, _) in _CALLS.items(): + setattr(m.x, name, make(m).set_display_name(name)) + return m + + +class TestEachPeriodIsTheScalarCall: + @pytest.mark.parametrize("name", list(_CALLS)) + def test_the_value_is_the_scalar_call_in_each_period(self, name): + m = _lines() + make, expected = _CALLS[name] + got = make(m) + want = expected(m) + assert got.var_type == "list" and len(got.value) == 3 + if isinstance(want[0], float): + assert got.value == pytest.approx(want) + else: + assert got.value == want + + def test_the_digits_of_each_period(self): + m = _lines() + assert mo.ROUND(m.x.v, m.x.d).value == [1.0, 2.3, -3.46] + assert mo.ROUND(2.71828, m.x.d).value == [3.0, 2.7, 2.72] + + def test_the_months_of_each_period(self): + m = _lines() + assert mo.EDATE(m.x.dt, m.x.k).value == [ + date(2026, 2, 28), date(2026, 4, 28), date(2026, 2, 28)] + assert mo.EOMONTH(m.x.dt, m.x.k).value == [ + date(2026, 2, 28), date(2026, 4, 30), date(2026, 2, 28)] + + def test_the_method_of_each_period(self): + # 28 Feb to 31 Mar: the U.S. method counts 30, the European 32. + m = _lines() + assert mo.DAYS360(m.x.dt, m.x.end, m.x.eu).value == [60, 30, 60] + assert mo.DAYS360(m.x.dt, m.x.end, True).value == [60, 32, 60] + + def test_one_value_keeps_the_call_as_it_was(self): + m = _lines() + assert mo.ROUND(m.x.v, 1).value == [1.2, 2.3, -3.5] + assert mo.ROUND(m.x.v, 1).formula == "ROUND(v, 1)" + assert mo.EDATE(m.x.dt, 1).value[0] == date(2026, 2, 28) + assert mo.PMT(0.05, 12, 1000).value == pytest.approx(-112.82541002081534) + assert mo.PMT(0.05, 12, 1000).var_type == "scalar" + assert mo.DAYS360(m.x.dt, m.x.end, True).value == [60, 32, 60] + + def test_a_blank_month_is_refused_by_name(self): + m = _lines() + with m: + m.x.gap = mo.Variable([1.0, None, 2.0], display_name="gap") + with pytest.raises(TypeError, match="'gap' is blank in"): + mo.EDATE(m.x.dt, m.x.gap) + + +class TestAsTheOperatorsReadTheirOperands: + def test_a_line_on_another_window_is_read_by_date(self): + # dg runs from 2027 and is held back to 2026: digits [3, 3, 1]. + m = _lines() + got = mo.ROUND(m.x.v, m.t.dg) + assert got.value == [1.235, 2.346, -3.5] + assert got.time is not None and got.time.start == "2026" + assert mo.EDATE(m.x.dt, m.t.dg).value == [ + date(2026, 4, 30), date(2026, 5, 28), date(2026, 4, 30)] + assert mo.PMT(m.x.rt, m.t.dg, 1000).value == pytest.approx( + [mo.PMT(r, n, 1000).value for r, n in zip([0.05, 0.06, 0.07], [3, 3, 1])]) + assert mo.DAYS360(m.x.dt, m.x.end, m.t.dg).value == [60, 32, 60] + + def test_a_line_without_a_rule_is_refused(self): + m = _lines() + for call in (lambda: mo.ROUND(m.x.v, m.t.nofill), + lambda: mo.EDATE(m.x.dt, m.t.nofill), + lambda: mo.EOMONTH(m.x.dt, m.t.nofill), + lambda: mo.PMT(0.05, m.t.nofill, m.x.amt), + lambda: mo.FV(m.x.rt, m.t.nofill, -100), + lambda: mo.PV(0.05, m.t.nofill, -100, m.x.amt), + lambda: mo.DAYS360(m.x.dt, m.x.end, m.t.nofill)): + with pytest.raises(ValueError, match="'nofill' does not cover the combined window"): + call() + + def test_lines_along_keys_combine_key_by_key(self): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with m: + m.r = mo.Variable({"Bear": 0.04, "Base": 0.05, "Bull": 0.06}, display_name="r") + m.n = mo.Variable({"Bull": 10.0, "Base": 20.0, "Bear": 30.0}, display_name="n") + m.s = mo.Variable([1.0, 2.0, 3.0], display_name="s") + got = mo.PMT(m.r, m.n, 1000) + assert got._keys == ["Bear", "Base", "Bull"] + assert got.value == pytest.approx([mo.PMT(0.04, 30, 1000).value, + mo.PMT(0.05, 20, 1000).value, + mo.PMT(0.06, 10, 1000).value]) + assert mo.ROUND(1.23456, m.n)._keys == ["Bull", "Base", "Bear"] + with pytest.raises(ValueError, match="runs along its keys"): + mo.PMT(m.r, m.s, 1000) + + +class TestOneValueStaysOneValue: + def test_a_series_rate_for_npv_is_refused_by_name(self): + m = _lines() + with pytest.raises(TypeError, match="NPV takes a single discount rate"): + mo.NPV(m.x.pct / 100, m.x.amt) + with pytest.raises(TypeError, match="NPV takes a single discount rate"): + mo.NPV([0.05, 0.06, 0.07], [100.0, 200.0, 300.0]) + + @pytest.mark.parametrize("call", ["IRR", "XIRR"]) + def test_a_series_guess_is_refused_by_name(self, call): + m = _lines() + with m: + m.x.cf = mo.Variable([-100.0, 60.0, 70.0], display_name="cf") + m.x.on = mo.Variable([date(2026, 1, 1), date(2027, 1, 1), date(2028, 1, 1)], + display_name="on") + args = (m.x.cf,) if call == "IRR" else (m.x.cf, m.x.on) + with pytest.raises(TypeError, match=f"{call}'s guess is one number"): + getattr(mo, call)(*args, m.x.pct / 100) + + +class TestTheWorkbook: + def test_each_column_writes_its_period_s_cells(self, tmp_path): + m = _model() + path = tmp_path / "book.xlsx" + m.to_excel(path) + ws = load_workbook(path)["X"] + x, at = _rows(ws), _row_of(ws) + + def each(text): + return [text.format(c=c, **{k: f"{c}{r}" for k, r in at.items()}) + for c in "BCD"] + + assert x["rnd"] == each("=ROUND({v}, {d})") + assert x["rnd2"] == each("=ROUND(2.71828, {d})") + assert x["ed"] == each("=EDATE({dt}, {k})") + assert x["ed2"][1] == f"=EDATE(DATE(2026, 1, 31), C{at['k']})" + assert x["eo"] == each("=EOMONTH({dt}, {k})") + assert x["pm"] == each("=PMT({pct} / 100, 12, 1000, 0, 0)") + assert x["pm2"] == each("=PMT(0.05, {n}, {amt}, 0, 1)") + assert x["fv"] == each("=FV({pct} / 100, 12, -100, 0, 0)") + assert x["pv"] == each("=PV(0.05, {n}, -100, {amt}, 0)") + assert x["d360"] == each("=DAYS360({dt}, {end}, {eu})") + + +def _soffice(): + for candidate in ("soffice", "libreoffice"): + found = shutil.which(candidate) + if found: + return found + mac = Path("/Applications/LibreOffice.app/Contents/MacOS/soffice") + return str(mac) if mac.exists() else None + + +@pytest.mark.slow +def test_the_workbook_recalculates_each_period_s_call(tmp_path): + soffice = _soffice() + if soffice is None: + pytest.skip("LibreOffice not installed") + m = _model() + with m: + m.x.rdg = mo.ROUND(m.x.v, m.t.dg).set_display_name("rdg") + path = tmp_path / "series.xlsx" + m.to_excel(path) + subprocess.run( + [soffice, f"-env:UserInstallation={(tmp_path / 'prof').as_uri()}", + "--headless", "--convert-to", "xlsx", "--outdir", str(tmp_path / "calc"), str(path)], + check=True, capture_output=True, + ) + x = _rows(load_workbook(tmp_path / "calc" / "series.xlsx", data_only=True)["X"]) + for name in [*_CALLS, "rdg"]: + want = getattr(m.x, name).value + got = x[name][:3] + for g, w in zip(got, want): + if isinstance(w, date): + assert isinstance(g, datetime) and g.date() == w, (name, got, want) + else: + assert g == pytest.approx(w), (name, got, want) diff --git a/tests/test_source_code_lazy.py b/tests/test_source_code_lazy.py new file mode 100644 index 0000000..c0d481f --- /dev/null +++ b/tests/test_source_code_lazy.py @@ -0,0 +1,158 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The source text of a condition is its formula, rendered when someone +reads it — not at every call. + +``IF``, ``AND``, ``OR``, ``NOT``, ``CHOOSE`` and the logical helpers say +"the readable source is the formula"; rendering that text at construction +inlines every anonymous operand, and nested conditions render the inner +text twice, for a value almost nobody reads. ``Variable._source_code`` is +therefore derived on read: a text written on the instance wins, else the +formula (the condition family), else the one text the Variables of a list +share (a ``Variable([a, b, ...])`` compound), else None. +""" + +import copy +import pickle + +import pytest + +import modeleon as mo +from modeleon.core.expr import FuncCall +from modeleon.functions.logical import IFERROR, IFS, ISERROR, ISNUMBER, SWITCH, XOR + + +def _model(): + m = mo.Model('m', default_grain='month', default_start='2026-01', default_periods=3) + m.x = mo.MultiVariable('X') + m.x.a = mo.Variable([1.0, 5.0, 9.0]) + m.x.b = mo.Variable([2.0, 2.0, 2.0]) + return m + + +@pytest.fixture +def renders(monkeypatch): + """Count the renders of a call node — the whole-expression render a + condition's source text costs.""" + count = {'n': 0} + original = FuncCall.to_string + + def counting(self): + count['n'] += 1 + return original(self) + + monkeypatch.setattr(FuncCall, 'to_string', counting) + return count + + +CONDITIONS = { + 'IF': lambda m: mo.IF(m.x.a > 4.0, m.x.a, 0.0), + 'AND': lambda m: mo.AND(m.x.a > 4.0, m.x.b > 1.0), + 'OR': lambda m: mo.OR(m.x.a > 4.0, False), + 'NOT': lambda m: mo.NOT(m.x.a > 4.0), + 'CHOOSE': lambda m: mo.CHOOSE(2, m.x.a, m.x.b), + 'IFERROR': lambda m: IFERROR(m.x.a / m.x.b, 0.0), + 'ISERROR': lambda m: ISERROR(m.x.a), + 'ISNUMBER': lambda m: ISNUMBER(m.x.a), + 'XOR': lambda m: XOR(m.x.a > 4.0, True), + 'IFS': lambda m: IFS(m.x.a > 7.0, 'big', True, 'small'), + 'SWITCH': lambda m: SWITCH(m.x.a, 1.0, 'one', 'other'), +} + + +class TestTheSourceIsTheFormulaRenderedOnRead: + def test_an_if_renders_nothing_until_read(self, renders): + m = _model() + out = mo.IF(m.x.a > 4.0, m.x.a, 0.0) + assert renders['n'] == 0 + assert '_source_code' not in vars(out) + assert out.source_code == "IF((a > 4.0), a, 0.0)" + assert renders['n'] >= 1 + + @pytest.mark.parametrize('name', sorted(CONDITIONS)) + def test_every_reader_gets_the_formula(self, name, renders): + m = _model() + out = CONDITIONS[name](m) + assert renders['n'] == 0, name + text = out.formula + assert text and text.startswith(f"{name}(") + # The public property, the internal attribute and the guarded read + # the renderers use all return the same string. + assert out.source_code == text + assert out._source_code == text + assert getattr(out, '_source_code', None) == text + assert hasattr(out, '_source_code') + + def test_a_nested_condition_renders_the_inner_text_once_per_read(self, renders): + m = _model() + inner = mo.IF(m.x.a > 4.0, 1.0, 0.0) + outer = mo.IF(m.x.b > 1.0, inner, 2.0) + assert renders['n'] == 0 + assert outer.source_code == "IF((b > 1.0), (IF((a > 4.0), 1.0, 0.0)), 2.0)" + assert renders['n'] == 2 # the outer call and the inlined inner one + + def test_a_written_text_wins_over_the_formula(self): + m = _model() + out = mo.IF(m.x.a > 4.0, m.x.a, 0.0) + out.source_code = "mine(a)" + assert out.source_code == out._source_code == "mine(a)" + # The ``None`` an adopted typed-dates line writes clears the text. + out._source_code = None + assert out.source_code is None and out._source_code is None + + def test_a_resolved_call_text_stays_eager(self): + # Helpers that resolve scalars into the text (``PMT(0.005, ...)``) + # still write it at the call; a plain input has none. + r = mo.PMT(0.005, 360, 300000) + assert '_source_code' in vars(r) + assert r.source_code == "PMT(0.005, 360, 300000, 0, 0)" + assert mo.Variable(1.0).source_code is None + + def test_copies_carry_the_text(self): + m = _model() + out = mo.IF(m.x.a > 4.0, m.x.a, 0.0) + assert mo.Variable(out).source_code == out.formula + assert mo.Variable(formula=out).source_code == out.formula + assert pickle.loads(pickle.dumps(out)).source_code == out.formula + assert copy.deepcopy(out).source_code == out.formula + + +class TestACompoundSharesItsItemsText: + def test_the_one_text_the_items_share_decided_on_read(self, renders): + m = _model() + out = mo.IF(m.x.a > 4.0, m.x.a, 0.0) + same = mo.Variable([out, out]) + with_scalars = mo.Variable([out, 1.0, out]) + assert renders['n'] == 0 + assert same.source_code == out.formula + assert with_scalars.source_code == out.formula + assert '_source_code' not in vars(same) + + def test_different_or_missing_texts_give_none(self): + m = _model() + out = mo.IF(m.x.a > 4.0, m.x.a, 0.0) + other = mo.IF(m.x.a > 1.0, 1.0, 0.0) + assert mo.Variable([out, other]).source_code is None + assert mo.Variable([out, m.x.a]).source_code is None + assert mo.Variable([m.x.a, m.x.b]).source_code is None + assert mo.Variable([1.0, 2.0]).source_code is None + + def test_eager_texts_share_too(self): + a = mo.PMT(0.005, 360, 300000) + b = mo.PMT(0.005, 360, 300000) + assert mo.Variable([a, b]).source_code == "PMT(0.005, 360, 300000, 0, 0)" + assert mo.Variable([a, mo.PMT(0.01, 360, 300000)]).source_code is None + + +def test_a_loop_row_listing_itself_shares_no_text(): + """A row of a loop container whose line lists the row itself is + rewritten in place, so its items reach the row again. The model is + refused; reading the row's text afterwards must answer None, not + recurse.""" + m = mo.Model('m', default_grain='month', default_start='2026-01', default_periods=3) + m.d = mo.MultiVariable('D', loop=True) + with pytest.raises(Exception): + with m.d: + placeholder = m.d.row + m.d.row = mo.Variable([placeholder, 2.0, 3.0]) + assert m.d.row._source_code is None + assert mo.Variable(m.d.row).source_code is None diff --git a/tests/test_statistics.py b/tests/test_statistics.py new file mode 100644 index 0000000..0e34a50 --- /dev/null +++ b/tests/test_statistics.py @@ -0,0 +1,623 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Statistics: the middle, the spread, percentiles, quartiles, the k-th +largest and smallest, and ranks — read as the spreadsheet reads them, over +a row or one field down the records of a table, one cell with its +formula. Several series, one call per period: test_statistics_per_period.py.""" + +import datetime as dt +import math +import statistics + +import pytest + +import modeleon as mo +from modeleon.core.expr import FuncCall, ListExpr, Literal, VarRef +from modeleon.functions.registry import ( + admits_in_loop, + excel_name, + file_prefix, + is_passthrough, + per_period_arity, + range_positions, + spec, +) +from modeleon.functions.statistics import ( + LARGE, + MEDIAN, + PERCENTILE, + PERCENTILE_EXC, + PERCENTILE_INC, + QUARTILE, + QUARTILE_EXC, + QUARTILE_INC, + RANK, + RANK_AVG, + RANK_EQ, + SMALL, + STDEV, + STDEV_P, + STDEV_S, + STDEVP, + VAR, + VAR_P, + VAR_S, + VARP, +) + +DATA = [10, 20, 30, 40, 50] +TIES = [5, 3, 5, 1, 3, 5] + + +def _row(values=DATA, **kwargs): + return mo.Variable(list(values), **kwargs) + + +class Comp(mo.MultiVariableClass): + def compute(self, multiple, region="North"): + self.region = mo.Variable(region, display_name="Region") + self.multiple = mo.Variable(multiple, display_name="Multiple", unit="x") + + +def _comps(*multiples): + m = mo.Model("m", display_name="M", default_grain="year", default_start="2026", + default_periods=3) + with m: + m.comps = mo.MultiVariable("Comps", excel_props={"tab": True, "orient": "records"}) + with m.comps as c: + for i, x in enumerate(multiples): + setattr(c, f"r{i}", Comp(multiple=x, display_name=f"R{i}")) + return m, [getattr(m.comps, f"r{i}") for i in range(len(multiples))] + + +# ══════════════════════════════════════════════════════════════════════ +# The reductions: MEDIAN and the spreads +# ══════════════════════════════════════════════════════════════════════ + + +class TestMedian: + @pytest.mark.parametrize("values, middle", [ + (DATA, 30), + ([40, 10, 30, 20], 25.0), # an even count: the mean of the middle two + ([7], 7), + ([3, 3, 1], 3), + ([-2.5, 1e-9, 4], 1e-9), + ]) + def test_the_middle_of_a_row(self, values, middle): + out = MEDIAN(_row(values)) + assert out.value == middle + assert out.var_type == "scalar" and out.value_type == "float" + + def test_one_row_or_values_typed_in_are_one_number_two_rows_one_per_period(self): + assert MEDIAN(_row([1, 9, 5, 7, 3])).value == 5 + assert MEDIAN(1, 2, 3, 4).value == 2.5 + assert MEDIAN(_row([1, 9]), _row([5, 7])).value == [3.0, 8.0] + assert MEDIAN(_row([1, 9]), 100, _row([5, 7])).value == [5, 9] + with pytest.raises(ValueError, match="different lengths"): + MEDIAN(_row([1, 9]), _row([5, 7, 3])) + + def test_a_blank_text_and_a_boolean_in_a_line_are_skipped(self): + row = _row([3, None, "x", True, 1, 4, 1.5]) + assert MEDIAN(row).value == 2.25 + assert MEDIAN(mo.Variable(True), 4).value == 4 # a cell holding TRUE + assert MEDIAN(mo.Variable("2"), 4).value == 4 # a cell holding text "2" + + def test_values_typed_into_the_call_count_as_the_spreadsheet_counts_them(self): + assert MEDIAN(1, "3").value == 2.0 # numeric text + assert MEDIAN(1, True, 5).value == 1 # TRUE is 1 + assert MEDIAN(1, None, 5).value == 1 # None is the 0 written + assert MEDIAN(1, "abc").value == "#VALUE!" + assert MEDIAN(1, "#N/A").value == "#VALUE!" # typed, it is text + + def test_an_error_is_the_result_the_first_in_reading_order(self): + assert MEDIAN(_row([1, "#DIV/0!", 3])).value == "#DIV/0!" + na, div0 = mo.Variable("#N/A"), mo.Variable("#DIV/0!") + assert MEDIAN([na, mo.Variable(1)], div0).value == "#N/A" + assert MEDIAN("x", div0).value == "#VALUE!" + assert MEDIAN(div0, "x").value == "#DIV/0!" + assert MEDIAN(_row(["#tag", 4])).value == 4 # text that only starts with # + # Over two rows, each period's first. + assert MEDIAN(_row(["#N/A", 1]), _row(["#DIV/0!", 2])).value == ["#N/A", 1.5] + + def test_no_number_is_num(self): + assert MEDIAN(_row(["a", None, "b"])).value == "#NUM!" + assert MEDIAN(mo.Variable(None)).value == "#NUM!" + + def test_a_number_past_any_cell_is_num(self): + assert MEDIAN(1e308, 1.7e308).value == "#NUM!" + assert MEDIAN(_row([1.0, math.inf])).value == "#NUM!" + + def test_dates_give_a_date(self): + d = _row([dt.date(2026, 1, 31), dt.date(2026, 3, 31), dt.date(2026, 2, 28)]) + out = MEDIAN(d) + assert out.value == dt.date(2026, 2, 28) and out.value_type == "datetime" + even = MEDIAN(_row([dt.date(2026, 2, 27), dt.date(2026, 2, 28)])) + assert even.value == dt.datetime(2026, 2, 27, 12) + assert MEDIAN(dt.date(2026, 1, 1), dt.date(2026, 1, 3)).value == dt.date(2026, 1, 2) + + def test_the_values_are_in_one_unit_and_the_middle_in_it(self): + usd = _row([1.0, 2.0], unit="USD") + assert str(MEDIAN(usd)._unit) == "USD" + assert str(MEDIAN(usd, 5.0, _row([4.0]))._unit) == "USD" # a number, an unlabelled line + with pytest.raises(ValueError, match="mo.MEDIAN takes its values in one unit"): + MEDIAN(usd, _row([3.0], unit="EUR")) + with pytest.raises(ValueError, match="mo.MEDIAN takes its values in one unit"): + MEDIAN([mo.Variable(1.0, unit="USD"), mo.Variable(2.0, unit="hr")]) + + +class TestTheSpreads: + def test_sample_and_population_over_a_row(self): + row = _row() + assert STDEV_S(row).value == pytest.approx(statistics.stdev(DATA), rel=1e-15) + assert STDEV_P(row).value == pytest.approx(statistics.pstdev(DATA), rel=1e-15) + assert VAR_S(row).value == 250.0 + assert VAR_P(row).value == 200.0 + + def test_the_older_names_are_the_same_functions(self): + row = _row([2.0, 4.0, 4.0, 4.0, 5.0, 5.0, 7.0, 9.0]) + assert STDEV(row).value == STDEV_S(row).value + assert STDEVP(row).value == STDEV_P(row).value == 2.0 + assert VAR(row).value == VAR_S(row).value + assert VARP(row).value == VAR_P(row).value == 4.0 + + def test_two_passes_keep_large_numbers_exact(self): + assert STDEV_S(1e15 + 1, 1e15 + 2, 1e15 + 3).value == 1.0 + assert VAR_P(_row([1e9 + 0.1, 1e9 + 0.3])).value == pytest.approx(0.01, rel=1e-6) + + @pytest.mark.parametrize("func, cells, value", [ + (STDEV_S, [7], "#DIV/0!"), # a sample needs two numbers + (VAR_S, [7, "x", None], "#DIV/0!"), + (STDEV_P, [7], 0.0), # a population one + (VAR_P, ["a", None], "#DIV/0!"), + (STDEV_S, ["a"], "#DIV/0!"), + ]) + def test_too_few_numbers_divide_by_zero(self, func, cells, value): + assert func(_row(cells)).value == value + + def test_lines_skip_what_typed_values_count(self): + assert STDEV_S(_row([True, 4, 6, "9", None])).value == pytest.approx(math.sqrt(2)) + assert STDEV_S(1, True, 5).value == pytest.approx(statistics.stdev([1, 1, 5])) + assert STDEV_S(1, "3", 5).value == 2.0 + assert STDEV_S(1, "abc", 5).value == "#VALUE!" + + def test_an_error_is_the_result_the_first_in_reading_order(self): + assert VAR_S(mo.Variable("#DIV/0!"), mo.Variable("#N/A")).value == "#DIV/0!" + assert STDEV_P(_row([1.0, "#NUM!"])).value == "#NUM!" + assert VAR_S(_row(["#DIV/0!", 1.0]), _row(["#N/A", 3.0])).value == ["#DIV/0!", 2.0] + + def test_too_large_a_spread_is_num(self): + assert VAR_P(-1e200, 1e200).value == "#NUM!" + + def test_a_spread_of_dates_is_a_number_of_days(self): + d = _row([dt.date(2026, 1, 31), dt.date(2026, 3, 31), dt.date(2026, 2, 28)]) + out = STDEV_S(d) + assert out.value == pytest.approx(statistics.stdev([46053, 46112, 46081])) + assert out.value_type == "float" + + def test_a_deviation_keeps_the_unit_and_a_variance_squares_it(self): + usd = _row([1.0, 2.0, 4.0], unit="USD") + assert str(STDEV_S(usd)._unit) == "USD" + assert str(VAR_S(usd)._unit) == "USD^2" + assert VAR_P(_row([0.1, 0.2], unit="%"))._unit is None # a label squared is none + assert VAR_S(_row([1.0, 2.0]))._unit is None + with pytest.raises(ValueError, match="mo.STDEV_S takes its values in one unit"): + STDEV_S(usd, _row([1.0], unit="hr")) + + +class TestTheReductionsArguments: + def test_at_least_one_and_at_most_255(self): + with pytest.raises(ValueError, match="at least one argument"): + MEDIAN() + assert MEDIAN(*range(255)).value == 127 + with pytest.raises(ValueError, match="at most 255 arguments"): + STDEV_S(*range(256)) + + def test_a_container_or_a_mapping_is_refused(self): + m, recs = _comps(8.0, 9.0) + with pytest.raises(TypeError, match="row, a list of lines"): + MEDIAN(m.comps) + with pytest.raises(TypeError, match="dict"): + VAR_S({"a": 1}) + + def test_a_line_with_tracks_is_refused(self): + m = mo.Model("m", tracks=mo.Tracks("plan", "actual"), default_grain="month", + default_start="2026-01", default_periods=2) + with m: + m.x = mo.Variable(plan=[1.0, 2.0], actual=[3.0, 4.0]) + with pytest.raises(ValueError, match="slice a coordinate first"): + MEDIAN(m.x) + + def test_a_list_holding_one_series_alone_is_refused(self): + with pytest.raises(ValueError, match="two or more values"): + MEDIAN([_row()]) + # With a value beside it, the list is one sample per period. + assert MEDIAN([_row([1.0, 5.0]), mo.Variable(3.0)]).value == [2.0, 4.0] + + +# ══════════════════════════════════════════════════════════════════════ +# One range: percentiles and quartiles +# ══════════════════════════════════════════════════════════════════════ + + +class TestPercentile: + @pytest.mark.parametrize("k, value", [ + (0, 10), (1, 50), (0.3, 22.0), (0.5, 30), (0.25, 20), (0.9, 46.0), + (-0.1, "#NUM!"), (1.1, "#NUM!"), + ]) + def test_inclusive(self, k, value): + got = PERCENTILE_INC(_row(), k).value + assert got == value if isinstance(value, str) else got == pytest.approx(value) + + @pytest.mark.parametrize("k, value", [ + (0.25, 15.0), (0.5, 30), (0.4, 24.0), (0.167, 10.02), (0.833, 49.98), + (1 / 6, 10), (5 / 6, 50), # the bounds: rank 1 and rank n + (0.16, "#NUM!"), (0.84, "#NUM!"), (0, "#NUM!"), (1, "#NUM!"), (-1, "#NUM!"), + ]) + def test_exclusive_between_the_first_and_the_last_rank(self, k, value): + got = PERCENTILE_EXC(_row(), k).value + assert got == value if isinstance(value, str) else got == pytest.approx(value) + + def test_the_cells_need_not_be_sorted(self): + row = _row([40, 10, 30, 20, 50]) + assert PERCENTILE_INC(row, 0.3).value == 22.0 + assert PERCENTILE_EXC(row, 0.25).value == 15.0 + + def test_a_position_within_15_digits_of_a_whole_one_is_that_cell(self): + assert 100 * 0.29 == 28.999999999999996 + assert PERCENTILE_INC(_row(range(1, 102)), 0.29).value == 30 + assert QUARTILE_INC(_row(range(1, 6)), 2.9999999999999996).value == 4 + + def test_one_number_is_every_percentile_of_it(self): + assert PERCENTILE_INC(_row([7]), 0.3).value == 7 + assert PERCENTILE_EXC(_row([7]), 0.5).value == 7 + assert PERCENTILE_EXC(_row([7]), 0.25).value == "#NUM!" + + def test_k_is_read_as_the_spreadsheet_reads_a_number(self): + row = _row() + assert PERCENTILE_INC(row, "0.5").value == 30 + assert PERCENTILE_INC(row, True).value == 50 + assert PERCENTILE_INC(row, mo.Variable(None)).value == 10 # a blank cell is 0 + assert PERCENTILE_INC(row, mo.Variable("")).value == 10 # typed empty text too + assert PERCENTILE_INC(row, mo.Variable("0.5")).value == 30 + assert PERCENTILE_INC(row, "abc").value == "#VALUE!" + assert PERCENTILE_INC(row, mo.Variable("#DIV/0!")).value == "#DIV/0!" + assert PERCENTILE_INC(row, "#DIV/0!").value == "#VALUE!" # typed, it is text + + def test_k_is_read_before_the_range(self): + bad = _row([1, "#N/A", 3]) + assert PERCENTILE_INC(bad, 0.5).value == "#N/A" + assert PERCENTILE_INC(bad, 2).value == "#NUM!" + assert PERCENTILE_INC(bad, mo.Variable("#DIV/0!")).value == "#DIV/0!" + assert PERCENTILE_EXC(bad, 0).value == "#NUM!" + + def test_no_number_is_num(self): + empty = _row(["a", None, "b", False]) + assert PERCENTILE_INC(empty, 0.5).value == "#NUM!" + assert PERCENTILE_EXC(empty, 0.5).value == "#NUM!" + + def test_a_series_k_is_one_call_per_period(self): + out = PERCENTILE_INC(_row(), mo.Variable([0, 0.5, 1, 2])) + assert out.value == [10, 30, 50, "#NUM!"] + assert out.var_type == "list" + + def test_dates_give_a_date(self): + d = _row([dt.date(2026, 1, 31), dt.date(2026, 3, 31), dt.date(2026, 2, 28)]) + out = PERCENTILE_INC(d, 0.25) + assert out.value == dt.date(2026, 2, 14) and out.value_type == "datetime" + + def test_the_unit_is_the_range_s_never_k_s(self): + out = PERCENTILE_INC(_row([1.0, 2.0], unit="USD"), mo.Variable(0.5, unit="%")) + assert str(out._unit) == "USD" + with pytest.raises(ValueError, match="mo.PERCENTILE_INC takes its values in one unit"): + PERCENTILE_INC([mo.Variable(1.0, unit="USD"), mo.Variable(2.0, unit="EUR")], 0.5) + + def test_the_older_name_is_the_inclusive_one(self): + assert PERCENTILE(_row(), 0.3).value == PERCENTILE_INC(_row(), 0.3).value + assert PERCENTILE(_row(), 1.5).value == "#NUM!" + + +class TestQuartile: + @pytest.mark.parametrize("quart, value", [ + (0, 10), (1, 20), (2, 30), (3, 40), (4, 50), + (1.5, 20), (4.5, 50), (-0.5, 10), # a fraction is truncated + (5, "#NUM!"), (-1, "#NUM!"), + ]) + def test_inclusive(self, quart, value): + assert QUARTILE_INC(_row(), quart).value == value + + @pytest.mark.parametrize("quart, value", [ + (1, 15.0), (2, 30), (3, 45.0), (3.5, 45.0), + (0, "#NUM!"), (4, "#NUM!"), (0.5, "#NUM!"), + ]) + def test_exclusive(self, quart, value): + assert QUARTILE_EXC(_row(), quart).value == value + + def test_too_few_cells_hold_an_exclusive_quartile(self): + assert QUARTILE_EXC(_row([7]), 1).value == "#NUM!" + assert QUARTILE_EXC(_row([7]), 2).value == 7 + assert QUARTILE_EXC(_row([10, 20]), 2).value == 15.0 + assert QUARTILE_EXC(_row([10, 20]), 1).value == "#NUM!" + + def test_the_second_quartile_is_the_median(self): + row = _row([0.1, 0.7, 0.2, 0.9]) + assert QUARTILE_INC(row, 2).value == MEDIAN(row).value + assert QUARTILE_EXC(row, 2).value == MEDIAN(row).value + + def test_no_number_and_an_error(self): + assert QUARTILE_INC(_row(["a"]), 1).value == "#NUM!" + assert QUARTILE_INC(_row([1, "#N/A"]), mo.Variable("#DIV/0!")).value == "#DIV/0!" + assert QUARTILE_INC(_row([1, "#N/A"]), 9).value == "#NUM!" + assert QUARTILE_INC(_row([1, "#N/A"]), 1).value == "#N/A" + + def test_the_older_name_is_the_inclusive_one(self): + assert QUARTILE(_row(), 1).value == 20 and QUARTILE(_row(), 0).value == 10 + + +# ══════════════════════════════════════════════════════════════════════ +# One range: the k-th largest and smallest +# ══════════════════════════════════════════════════════════════════════ + + +class TestLargeAndSmall: + @pytest.mark.parametrize("k, large, small", [ + (1, 50, 10), (2, 40, 20), (5, 10, 50), + (2.5, 30, 20), # LARGE takes k up, SMALL down + (2.9, 30, 20), + (0.5, 50, "#NUM!"), + (5.5, "#NUM!", 50), + (2.0000000000000004, 40, 20), # within 15 digits of 2 + (1.9999999999999998, 40, 20), + (2.000000000001, 30, 20), + (0, "#NUM!", "#NUM!"), (6, "#NUM!", "#NUM!"), (-1, "#NUM!", "#NUM!"), + ]) + def test_the_k_th(self, k, large, small): + row = _row([40, 10, 30, 20, 50]) + assert LARGE(row, k).value == large + assert SMALL(row, k).value == small + + def test_ties_each_count(self): + assert LARGE(_row(TIES), 2).value == 5 and LARGE(_row(TIES), 4).value == 3 + assert SMALL(_row(TIES), 2).value == 3 + + def test_k_is_read_as_the_spreadsheet_reads_a_number(self): + assert LARGE(_row(), "2").value == 40 + assert LARGE(_row(), mo.Variable("2")).value == 40 # a cell holding text "2" + assert SMALL(_row(), mo.Variable("2")).value == 20 + assert LARGE(_row(), True).value == 50 + assert LARGE(_row(), mo.Variable(None)).value == "#NUM!" + assert LARGE(_row(), "two").value == "#VALUE!" + + def test_k_s_error_then_the_range_s_then_k_against_the_count(self): + bad = _row([1, "#N/A"]) + assert LARGE(bad, mo.Variable("#DIV/0!")).value == "#DIV/0!" + assert LARGE(bad, 0).value == "#N/A" + assert LARGE(bad, 9).value == "#N/A" + assert SMALL(_row(["a"]), 1).value == "#NUM!" + + def test_the_cell_comes_back_as_it_holds_it(self): + out = LARGE(_row([3, 1, 2]), 1) + assert out.value == 3 and isinstance(out.value, int) and out.value_type == "int" + d = _row([dt.date(2026, 1, 31), dt.date(2026, 3, 31)]) + assert SMALL(d, 1).value == dt.date(2026, 1, 31) + assert SMALL(d, 1).value_type == "datetime" + assert str(LARGE(_row([1.0, 2.0], unit="USD"), 1)._unit) == "USD" + + def test_a_series_k_is_one_call_per_period(self): + assert SMALL(_row(), [1, 2, 3]).value == [10, 20, 30] + + +# ══════════════════════════════════════════════════════════════════════ +# One range: ranks +# ══════════════════════════════════════════════════════════════════════ + + +class TestRank: + @pytest.mark.parametrize("number, order, eq, avg", [ + (3, 0, 4, 4.5), (3, 1, 2, 2.5), + (5, 0, 1, 2.0), (5, 1, 4, 5.0), + (1, 0, 6, 6.0), (1, 1, 1, 1.0), + ]) + def test_ties_share_the_best_rank_or_their_mean(self, number, order, eq, avg): + ref = _row(TIES) + assert RANK_EQ(number, ref, order).value == eq + assert RANK_AVG(number, ref, order).value == avg + + def test_descending_unless_the_order_is_not_zero(self): + ref = _row(TIES) + assert RANK_EQ(3, ref).value == 4 + for order in (True, -1, 0.5, "1", mo.Variable(2)): + assert RANK_EQ(3, ref, order).value == 2, order + assert RANK_EQ(3, ref, mo.Variable(None)).value == 4 # a blank cell is 0 + + def test_a_number_the_range_lacks_is_na(self): + assert RANK_EQ(4, _row(TIES)).value == "#N/A" + assert RANK_EQ(9, _row(TIES)).value == "#N/A" + assert RANK_EQ(0.3, [mo.Variable(0.1) + mo.Variable(0.2), mo.Variable(0.5)] + ).value == "#N/A" # equal exactly, or not + assert RANK_EQ(1, _row(["a", None, "b"])).value == "#N/A" + + def test_the_number_is_read_as_the_spreadsheet_reads_one(self): + ref = _row(TIES) + assert RANK_EQ("3", ref).value == 4 + assert RANK_EQ(True, ref).value == 6 + assert RANK_EQ(mo.Variable(None), ref).value == "#N/A" # a blank is 0 + assert RANK_EQ("abc", ref).value == "#VALUE!" + + def test_the_order_the_range_then_the_number(self): + div0, na = mo.Variable("#DIV/0!"), mo.Variable("#N/A") + assert RANK_EQ(div0, _row(DATA)).value == "#DIV/0!" + assert RANK_EQ(10, _row(DATA), div0).value == "#DIV/0!" + assert RANK_EQ(div0, _row([1, "#N/A"])).value == "#N/A" + assert RANK_EQ(na, _row(DATA), div0).value == "#DIV/0!" + assert RANK_EQ(div0, _row(["a"])).value == "#N/A" + + def test_a_blank_text_and_a_boolean_in_the_range_are_skipped(self): + assert RANK_EQ(1, _row([3, None, "x", True, 1, 4, 1.5])).value == 4 + + def test_dates_rank_by_their_serial(self): + d = _row([dt.date(2026, 1, 31), dt.date(2026, 3, 31), dt.date(2026, 2, 28)]) + assert RANK_EQ(dt.date(2026, 2, 28), d).value == 2 + assert RANK_EQ(46081, d).value == 2 + + def test_a_rank_is_an_int_and_an_average_rank_a_float_without_a_unit(self): + ref = _row([1.0, 2.0], unit="USD") + eq, avg = RANK_EQ(mo.Variable(2.0, unit="USD"), ref), RANK_AVG(2.0, ref) + assert (eq.value, eq.value_type, eq._unit) == (1, "int", None) + assert (avg.value, avg.value_type, avg._unit) == (1.0, "float", None) + + def test_the_number_and_the_range_are_in_one_unit_the_order_is_no_value(self): + ref = _row([1.0, 2.0], unit="USD") + assert RANK_EQ(2.0, ref, mo.Variable(1, unit="flag")).value == 2 + with pytest.raises(ValueError, match="mo.RANK_EQ takes its values in one unit"): + RANK_EQ(mo.Variable(2.0, unit="hr"), ref) + + def test_a_series_number_ranks_each_period_among_the_row(self): + row = _row([30, 10, 20]) + out = RANK_EQ(row, row) + assert out.value == [1, 3, 2] and out.var_type == "list" + + def test_each_record_among_its_peers(self): + m, recs = _comps(8.0, 12.0, 10.0, 9.0, 10.0) + peers = [r.multiple for r in recs] + assert [RANK_EQ(r.multiple, peers).value for r in recs] == [5, 1, 2, 4, 2] + assert [RANK_AVG(r.multiple, peers).value for r in recs] == [5.0, 1.0, 2.5, 4.0, 2.5] + assert [RANK_EQ(r.multiple, peers, 1).value for r in recs] == [1, 5, 3, 2, 3] + + def test_the_older_name_is_rank_eq(self): + assert RANK(3, _row(TIES)).value == RANK_EQ(3, _row(TIES)).value == 4 + + +# ══════════════════════════════════════════════════════════════════════ +# The two range kinds and the parameters +# ══════════════════════════════════════════════════════════════════════ + + +class TestTheRangeKinds: + def test_one_field_down_the_records_reads_as_a_row_does(self): + m, recs = _comps(8.0, 12.0, 10.0, 9.0) + field = [r.multiple for r in recs] + row = _row([8.0, 12.0, 10.0, 9.0]) + for func, args in ((MEDIAN, ()), (STDEV_S, ()), (VAR_P, ())): + assert func(field, *args).value == func(row, *args).value, func + for func, k in ((PERCENTILE_INC, 0.25), (PERCENTILE_EXC, 0.4), (QUARTILE_INC, 3), + (QUARTILE_EXC, 1), (LARGE, 2), (SMALL, 2)): + assert func(field, k).value == func(row, k).value, func + assert str(MEDIAN(field)._unit) == "x" + + def test_a_tuple_one_element_of_a_row_and_typed_values(self): + row = _row([5.0, 1.0, 3.0]) + assert MEDIAN((row[0], row[1], row[2])).value == 3.0 + assert LARGE([row[0], 7.0, "x", True], 2).value == 5.0 # a list is an array + assert MEDIAN([1, True, "3", None]).value == 1 # numbers only + + def test_a_range_is_a_row_or_a_list_never_one_typed_value(self): + with pytest.raises(TypeError, match="a range is"): + LARGE(5, 1) + with pytest.raises(ValueError, match="at least one cell"): + PERCENTILE_INC([], 0.5) + + def test_a_parameter_is_one_value_or_a_line(self): + with pytest.raises(TypeError, match="k is one value"): + LARGE(_row(), (1, 2)) + with pytest.raises(TypeError, match="number is one value"): + RANK_EQ([mo.Variable(1.0)], _row()) + + def test_a_parameter_with_tracks_is_refused(self): + m = mo.Model("m", tracks=mo.Tracks("plan", "actual"), default_grain="month", + default_start="2026-01", default_periods=2) + with m: + m.k = mo.Variable(plan=[0.1, 0.2], actual=[0.3, 0.4]) + with pytest.raises(ValueError, match="slice a coordinate first"): + PERCENTILE_INC(_row(), m.k) + + def test_ranges_and_parameters_of_different_lengths_are_fine(self): + # One range, read whole by every period's call. + assert LARGE(_row([1, 2, 3, 4, 5, 6]), _row([1, 2])).value == [6, 5] + + +# ══════════════════════════════════════════════════════════════════════ +# The call: the AST, the registry, the loop +# ══════════════════════════════════════════════════════════════════════ + +ALL = { + "MEDIAN": MEDIAN, "STDEV_S": STDEV_S, "STDEV_P": STDEV_P, "VAR_S": VAR_S, + "VAR_P": VAR_P, "STDEV": STDEV, "STDEVP": STDEVP, "VAR": VAR, "VARP": VARP, + "PERCENTILE_INC": PERCENTILE_INC, "PERCENTILE_EXC": PERCENTILE_EXC, + "QUARTILE_INC": QUARTILE_INC, "QUARTILE_EXC": QUARTILE_EXC, + "PERCENTILE": PERCENTILE, "QUARTILE": QUARTILE, "LARGE": LARGE, "SMALL": SMALL, + "RANK_EQ": RANK_EQ, "RANK_AVG": RANK_AVG, "RANK": RANK, +} +REDUCTIONS = {"MEDIAN", "STDEV_S", "STDEV_P", "VAR_S", "VAR_P", "STDEV", "STDEVP", "VAR", "VARP"} +EXCEL = { + "STDEV_S": "STDEV.S", "STDEV_P": "STDEV.P", "VAR_S": "VAR.S", "VAR_P": "VAR.P", + "PERCENTILE_INC": "PERCENTILE.INC", "PERCENTILE_EXC": "PERCENTILE.EXC", + "QUARTILE_INC": "QUARTILE.INC", "QUARTILE_EXC": "QUARTILE.EXC", + "RANK_EQ": "RANK.EQ", "RANK_AVG": "RANK.AVG", +} + + +def _call(name): + row = mo.Variable([10.0, 20.0, 30.0], display_name="row") + if name in REDUCTIONS: + return ALL[name](row, 5) + if name.startswith("RANK"): + return ALL[name](mo.Variable(20.0, display_name="x"), row, 1) + return ALL[name](row, mo.Variable(1, display_name="k")) + + +class TestTheCall: + @pytest.mark.parametrize("name", sorted(ALL)) + def test_the_call_is_named_as_the_function_and_holds_its_arguments(self, name): + out = _call(name) + assert isinstance(out._expr, FuncCall) and out._expr.func == name + assert mo.to_json(out._expr)["func"] == name + args = out._expr.args + if name in REDUCTIONS: + assert isinstance(args[0], VarRef) and args[1] == Literal(5) + assert out.formula == f"{name}(row, 5)" + elif name.startswith("RANK"): + assert [type(a) for a in args] == [VarRef, VarRef, Literal] + assert out.formula == f"{name}(x, row, 1)" + else: + assert [type(a) for a in args] == [VarRef, VarRef] + assert out.formula == f"{name}(row, k)" + + def test_a_list_of_lines_is_one_list_node(self): + m, recs = _comps(8.0, 12.0) + out = PERCENTILE_INC([r.multiple for r in recs], 0.5) + assert isinstance(out._expr.args[0], ListExpr) + assert out._expr.args[0].items == [VarRef(r.multiple) for r in recs] + assert out._source_code == "PERCENTILE_INC([r0.multiple, r1.multiple], 0.5)" + + def test_an_omitted_order_is_left_out_of_the_call(self): + out = RANK_EQ(20.0, _row([10.0, 20.0])) + assert len(out._expr.args) == 2 + + @pytest.mark.parametrize("name", sorted(ALL)) + def test_each_name_is_registered_with_its_shape_and_excel_name(self, name): + s = spec(name) + assert s.shape == ("reduction" if name in REDUCTIONS else "range"), name + assert s.range_args == (None if name in REDUCTIONS + else (1,) if name.startswith("RANK") else (0,)), name + assert excel_name(name) == EXCEL.get(name, name) + assert file_prefix(name) == ("_xlfn." if name in EXCEL else "") + assert is_passthrough(name) and not s.bare + assert range_positions(name) == s.range_args + # Two values or more, a series among them: one call per period. + assert per_period_arity(name) == s.per_period_arity == 2 + assert not admits_in_loop(name, 1) and admits_in_loop(name, 2) + + @pytest.mark.parametrize("make", [ + lambda d: MEDIAN(d.bal), + lambda d: STDEV_S(d.bal), + lambda d: LARGE(d.bal, 1), + lambda d: RANK_EQ(d.bal, d.bal), + ]) + def test_a_loop_container_refuses_a_statistic_over_one_of_its_rows(self, make): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with pytest.raises(ValueError, match="reads a whole series"): + with m: + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + d.bal = mo.lag(d.x, fill=10.0) * 1.37 + d.stat = make(d) + d.x = d.bal - d.stat diff --git a/tests/test_statistics_per_period.py b/tests/test_statistics_per_period.py new file mode 100644 index 0000000..b7ae0b6 --- /dev/null +++ b/tests/test_statistics_per_period.py @@ -0,0 +1,617 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Statistics over several series are one call per period. + +The rule ``MIN`` / ``MAX`` follow: several series are combined period by +period, as ``a + b`` combines them; one series alone is reduced over its +periods; scalars only stay one number. ``mo.MEDIAN(a, b)`` over two rows +is the median of each period's two cells — ``=MEDIAN(C5, C9)`` in every +column — and a list of two or more rows (one field down the records of a +table, or sibling lines) is one sample per period: the median, the +percentile, the k-th largest and each line's rank of this period's cells, +written as the run those cells form in the column +(``=_xlfn.PERCENTILE.INC(Peers!B3:B5, 0.25)``). A scalar, a scalar line +and a one-value series stand in every period; series of other lengths +are refused; a line with tracks lifts per track; a loop container runs +the call one period at a time. +""" + +import datetime as dt +import math + +import pytest +from openpyxl import load_workbook + +import modeleon as mo +from modeleon.compile import check_compat +from modeleon.core.expr import FuncCall, ListExpr, Literal, VarRef +from modeleon.functions.registry import admits_in_loop, per_period_arity + +A = [3.0, 8.0, 5.0] +B = [4.0, 4.0, 6.0] +C = [1.0, 9.0, 5.0] + + +def _rows(ws): + out = {} + for row in ws.iter_rows(): + label = row[0].value + if label: + out[str(label).split(" (")[0]] = [c.value for c in row[1:]] + return out + + +def _book(model, tmp_path): + path = tmp_path / "book.xlsx" + model.to_excel(path) + return load_workbook(path) + + +class Peer(mo.MultiVariableClass): + def compute(self, rev): + self.rev = mo.Variable(rev, display_name="Revenue", unit="USD") + + +def _model(): + """Inputs on I; three sibling lines a, b, c on X (rows 2–4, periods in + B–D); a records table of three peers with a revenue per year on Peers + (rows 3–5); a Summary sheet.""" + m = mo.Model("m", display_name="M", default_grain="year", default_start="2026", + default_periods=3) + with m: + m.i = mo.MultiVariable("I") + m.i.floor = mo.Variable(5.0, display_name="floor") + m.i.k = mo.Variable(0.25, display_name="k") + m.x = mo.MultiVariable("X") + m.x.a = mo.Variable(list(A), display_name="a") + m.x.b = mo.Variable(list(B), display_name="b") + m.x.c = mo.Variable(list(C), display_name="c") + m.peers = mo.MultiVariable("Peers", excel_props={"tab": True, "orient": "records"}) + with m.peers as p: + p.a = Peer(rev=[10.0, 30.0, 25.0], display_name="A") + p.b = Peer(rev=[20.0, 10.0, 45.0], display_name="B") + p.c = Peer(rev=[35.0, 40.0, 10.0], display_name="C") + m.s = mo.MultiVariable("Summary", excel_props={"tab": True}) + return m + + +def _revs(m): + p = m.peers + return [p.a.rev, p.b.rev, p.c.rev] + + +def _lines(m): + return [m.x.a, m.x.b, m.x.c] + + +# ══════════════════════════════════════════════════════════════════════ +# The reductions: MEDIAN and the spreads +# ══════════════════════════════════════════════════════════════════════ + + +class TestTheReductionsPerPeriod: + def test_two_series_are_combined_period_by_period(self): + a, b = mo.Variable(list(A)), mo.Variable(list(B)) + out = mo.MEDIAN(a, b) + assert out.value == [3.5, 6.0, 5.5] + assert out.var_type == "list" and out.value_type == "float" + assert mo.VAR_P(a, b).value == [0.25, 4.0, 0.25] + assert mo.STDEV_S(a, b).value == pytest.approx( + [math.sqrt(0.5), math.sqrt(8), math.sqrt(0.5)]) + assert mo.STDEV_P(a, b).value == [0.5, 2.0, 0.5] + assert mo.VAR_S(a, b).value == [0.5, 8.0, 0.5] + + def test_three_series(self): + a, b, c = (mo.Variable(list(v)) for v in (A, B, C)) + assert mo.MEDIAN(a, b, c).value == [3.0, 8.0, 5.0] + assert mo.VARP(a, b, c).value == pytest.approx( + [((3 - 8 / 3) ** 2 + (4 - 8 / 3) ** 2 + (1 - 8 / 3) ** 2) / 3, + ((8 - 7) ** 2 + (4 - 7) ** 2 + (9 - 7) ** 2) / 3, + ((5 - 16 / 3) ** 2 + (6 - 16 / 3) ** 2 + (5 - 16 / 3) ** 2) / 3]) + + def test_the_older_names_follow_the_same_rule(self): + a, b = mo.Variable(list(A)), mo.Variable(list(B)) + assert mo.STDEV(a, b).value == mo.STDEV_S(a, b).value + assert mo.STDEVP(a, b).value == mo.STDEV_P(a, b).value + assert mo.VAR(a, b).value == mo.VAR_S(a, b).value + assert mo.VARP(a, b).value == mo.VAR_P(a, b).value + + def test_a_scalar_broadcasts_on_either_side_as_a_plus_b(self): + a = mo.Variable(list(A)) + assert mo.MEDIAN(a, 5.0).value == [4.0, 6.5, 5.0] + assert mo.MEDIAN(5.0, a).value == [4.0, 6.5, 5.0] + assert mo.MEDIAN(a, mo.Variable(5.0)).value == [4.0, 6.5, 5.0] + assert mo.STDEV_P(a, 5.0).value == [1.0, 1.5, 0.0] + + def test_a_one_value_series_broadcasts_and_other_lengths_are_refused(self): + a = mo.Variable(list(A)) + assert mo.MEDIAN(mo.Variable([5.0]), a).value == [4.0, 6.5, 5.0] + with pytest.raises(ValueError, match="different lengths"): + mo.MEDIAN(mo.Variable([1.0, 2.0]), a) + with pytest.raises(ValueError, match="different lengths"): + mo.VAR_S(a, [1.0, 2.0]) + + def test_a_list_typed_in_is_a_series_as_the_operators_read_it(self): + a = mo.Variable(list(A)) + assert mo.MEDIAN(a, [1.0, 9.0, 1.0]).value == [2.0, 8.5, 3.0] + assert mo.MEDIAN([1, 2, 3], [4, 5, 6]).value == [2.5, 3.5, 4.5] + + def test_one_series_alone_is_reduced_over_its_periods(self): + a = mo.Variable(list(A)) + out = mo.MEDIAN(a) + assert out.value == 5.0 and out.var_type == "scalar" + assert mo.MEDIAN(list(A)).value == 5.0 + assert mo.VAR_P(a).value == pytest.approx(38 / 9) + + def test_scalars_only_stay_one_number(self): + out = mo.MEDIAN(mo.Variable(3.0), mo.Variable(4.0)) + assert out.value == 3.5 and out.var_type == "scalar" + assert mo.MEDIAN(1, 2, 3, 10).value == 2.5 + m = _model() + field = [m.peers.a.rev[0], m.peers.b.rev[0], m.peers.c.rev[0]] + assert mo.MEDIAN(field).value == 20.0 # one element of each row + + def test_a_list_of_series_lines_is_one_sample_per_period(self): + m = _model() + assert mo.MEDIAN(_lines(m)).value == mo.MEDIAN(m.x.a, m.x.b, m.x.c).value + assert mo.MEDIAN(_revs(m)).value == [20.0, 30.0, 25.0] + assert mo.STDEV_S(_revs(m)).value == pytest.approx( + [12.583057392117917, 15.275252316519467, 17.559422921421234]) + assert mo.MEDIAN(_revs(m)).var_type == "list" + + def test_a_field_of_scalar_lines_with_a_series_is_read_whole_in_every_period(self): + cells = [mo.Variable(1.0), mo.Variable(2.0), mo.Variable(10.0)] + a = mo.Variable(list(A)) + assert mo.MEDIAN(cells, a).value == [2.5, 5.0, 3.5] + assert mo.MEDIAN(a, (1.0, 2.0, 10.0)).value == [2.5, 5.0, 3.5] # a typed array + + def test_a_list_holding_one_series_is_refused_with_both_readings(self): + m = _model() + with pytest.raises(ValueError, match=r"two or more") as err: + mo.MEDIAN([m.x.a]) + assert "mo.MEDIAN(x.a) reduces the line over its periods" in str(err.value) + with pytest.raises(ValueError, match=r"two or more") as err: + mo.RANK_EQ(m.x.a, (m.x.a,)) + assert "mo.RANK_EQ(number, x.a, order)" not in str(err.value) + assert "mo.RANK_EQ(number, x.a) reads the line over its periods" in str(err.value) + + def test_a_range_is_lines_and_values_never_a_list_inside_it(self): + a = mo.Variable(list(A)) + with pytest.raises(TypeError, match="one value"): + mo.MEDIAN([a, mo.Variable(list(B)), [1.0, 2.0]]) + + def test_this_period_s_cells_as_the_spreadsheet_reads_them(self): + # In a line a blank, text and a boolean are skipped; typed into the + # call a number, TRUE and numeric text count. + mixed = mo.Variable([None, "x", True]) + b = mo.Variable([1.0, 2.0, 3.0]) + assert mo.MEDIAN(mixed, b).value == [1.0, 2.0, 3.0] + assert mo.MEDIAN(b, True).value == [1.0, 1.5, 2.0] + assert mo.MEDIAN(b, "3").value == [2.0, 2.5, 3.0] + assert mo.MEDIAN(b, None).value == [0.5, 1.0, 1.5] + assert mo.MEDIAN(b, "abc").value == ["#VALUE!"] * 3 + + def test_an_error_is_the_period_s_result_the_first_in_reading_order(self): + e = mo.Variable([2.0, 2.0, 2.0]) / mo.Variable([1.0, 0.0, 2.0]) + na = mo.Variable(["#N/A", 1.0, 1.0]) + assert mo.MEDIAN(e, 4.0).value == [3.0, "#DIV/0!", 2.5] + assert mo.MEDIAN(na, e).value == ["#N/A", "#DIV/0!", 1.0] + assert mo.STDEV_S(e, na).value == ["#N/A", "#DIV/0!", 0.0] + + def test_too_few_numbers_in_a_period(self): + blank = mo.Variable([None, 2.0, "x"]) + b = mo.Variable([None, None, 5.0]) + assert mo.MEDIAN(blank, b).value == ["#NUM!", 2.0, 5.0] + assert mo.STDEV_S(blank, b).value == ["#DIV/0!", "#DIV/0!", "#DIV/0!"] + assert mo.VAR_P(blank, b).value == ["#DIV/0!", 0.0, 0.0] + + def test_dates_give_a_date_each_period(self): + d1 = mo.Variable([dt.date(2026, 1, 31), dt.date(2026, 2, 28)]) + d2 = mo.Variable([dt.date(2026, 3, 31), dt.date(2026, 3, 2)]) + out = mo.MEDIAN(d1, d2, dt.date(2026, 2, 28)) + assert out.value == [dt.date(2026, 2, 28), dt.date(2026, 2, 28)] + assert out.value_type == "datetime" + + def test_the_values_are_in_one_unit_and_a_variance_in_its_square(self): + usd = mo.Variable([1.0, 2.0, 4.0], unit="USD") + more = mo.Variable([3.0, 2.0, 1.0], unit="USD") + assert str(mo.MEDIAN(usd, more)._unit) == "USD" + assert str(mo.STDEV_S(usd, more, 5.0)._unit) == "USD" + assert str(mo.VAR_S(usd, more)._unit) == "USD^2" + with pytest.raises(ValueError, match="mo.MEDIAN takes its values in one unit"): + mo.MEDIAN(usd, mo.Variable([1.0, 1.0, 1.0], unit="hr")) + with pytest.raises(ValueError, match="mo.VAR_P takes its values in one unit"): + mo.VAR_P([usd, mo.Variable([1.0, 1.0, 1.0], unit="EUR")]) + + def test_a_line_with_tracks_lifts_per_track(self): + m = mo.Model("m", tracks=mo.Tracks("plan", "actual"), default_grain="month", + default_start="2026-01", default_periods=3) + with m: + m.x = mo.Variable(plan=[1.0, 5.0, 3.0], actual=[2.0, 2.0, 2.0]) + m.mid = mo.MEDIAN(m.x, 4.0) + m.spread = mo.VAR_P([m.x, m.x * 0.0 + 3.0]) + assert dict(m.mid._value.items()) == {"plan": [2.5, 4.5, 3.5], "actual": [3.0, 3.0, 3.0]} + assert dict(m.spread._value.items()) == {"plan": [1.0, 1.0, 0.0], + "actual": [0.25, 0.25, 0.25]} + assert m.mid.var_type == "list" + + def test_one_line_with_tracks_alone_stays_refused(self): + m = mo.Model("m", tracks=mo.Tracks("plan", "actual"), default_grain="month", + default_start="2026-01", default_periods=2) + with m: + m.x = mo.Variable(plan=[1.0, 2.0], actual=[3.0, 4.0]) + with pytest.raises(ValueError, match="slice a coordinate first"): + mo.MEDIAN(m.x) + + def test_the_call_holds_its_operands(self): + m = _model() + out = mo.MEDIAN(m.x.a, m.x.b, 5.0) + assert out._expr == FuncCall("MEDIAN", [VarRef(m.x.a), VarRef(m.x.b), Literal(5.0)]) + assert out.formula == "MEDIAN(a, b, 5.0)" + typed = mo.MEDIAN(m.x.a, [1.0, 9.0, 1.0]) + assert typed._expr.args[1] == Literal([1.0, 9.0, 1.0]) + sample = mo.PERCENTILE_INC(_revs(m), 0.25) + assert sample._expr.args[0] == ListExpr([VarRef(r) for r in _revs(m)]) + assert mo.to_json(sample._expr)["func"] == "PERCENTILE_INC" + + +# ══════════════════════════════════════════════════════════════════════ +# One range: a list of two or more series lines is a sample per period +# ══════════════════════════════════════════════════════════════════════ + + +class TestOneRangePerPeriod: + def test_percentiles_quartiles_and_the_k_th_of_each_period(self): + m = _model() + lines = _lines(m) + assert mo.PERCENTILE_INC(lines, 0.5).value == [3.0, 8.0, 5.0] + assert mo.PERCENTILE_INC(lines, 0.25).value == [2.0, 6.0, 5.0] + assert mo.PERCENTILE(lines, 0.25).value == [2.0, 6.0, 5.0] + assert mo.PERCENTILE_EXC(lines, 0.5).value == [3.0, 8.0, 5.0] + assert mo.QUARTILE_INC(lines, 1).value == [2.0, 6.0, 5.0] + assert mo.QUARTILE(lines, 3).value == [3.5, 8.5, 5.5] + assert mo.QUARTILE_EXC(lines, 2).value == [3.0, 8.0, 5.0] + assert mo.LARGE(lines, 1).value == [4.0, 9.0, 6.0] + assert mo.SMALL(lines, 1).value == [1.0, 4.0, 5.0] + assert mo.LARGE(lines, 1).var_type == "list" + + def test_a_series_k_reads_this_period_s_k(self): + m = _model() + ks = mo.Variable([0.0, 0.5, 1.0]) + assert mo.PERCENTILE_INC(_lines(m), ks).value == [1.0, 8.0, 6.0] + assert mo.SMALL(_lines(m), [1, 2, 3]).value == [1.0, 8.0, 6.0] + with pytest.raises(ValueError, match="different lengths"): + mo.PERCENTILE_INC(_lines(m), mo.Variable([0.0, 0.5])) + + def test_each_line_ranked_among_its_peers_each_period(self): + m = _model() + lines = _lines(m) + assert mo.RANK_EQ(m.x.c, lines).value == [3, 1, 2] + assert mo.RANK_AVG(m.x.c, lines).value == [3.0, 1.0, 2.5] + assert mo.RANK_EQ(m.x.c, lines, 1).value == [1, 3, 1] + assert mo.RANK(m.x.c, lines).value == [3, 1, 2] + revs = _revs(m) + assert [mo.RANK_EQ(r, revs).value for r in revs] == [[3, 2, 2], [2, 3, 1], [1, 1, 3]] + assert mo.RANK_EQ(m.x.c, lines).value_type == "int" + + def test_a_number_typed_in_is_ranked_each_period(self): + m = _model() + assert mo.RANK_EQ(5.0, _lines(m)).value == ["#N/A", "#N/A", 2] + + def test_a_scalar_line_in_the_list_stands_in_every_period(self): + m = _model() + assert mo.PERCENTILE_INC([m.x.a, m.x.b, m.i.floor], 0.5).value == [4.0, 5.0, 5.0] + assert mo.LARGE([m.x.a, 7.0], 1).value == [7.0, 8.0, 7.0] + + def test_a_single_series_or_a_list_of_scalars_is_read_as_before(self): + m = _model() + assert mo.PERCENTILE_INC(m.x.a, 0.5).value == 5.0 + assert mo.LARGE(m.x.b, mo.Variable([1, 2, 3])).value == [6.0, 4.0, 4.0] + cells = [mo.Variable(8.0), mo.Variable(12.0), mo.Variable(10.0)] + assert mo.PERCENTILE_INC(cells, 0.5).value == 10.0 + assert mo.RANK_EQ(cells[0], cells).value == 3 + + def test_a_list_holding_one_series_is_refused(self): + m = _model() + with pytest.raises(ValueError, match="two or more"): + mo.PERCENTILE_INC([m.x.a], 0.5) + with pytest.raises(ValueError, match="two or more"): + mo.RANK_EQ(m.x.a, [m.x.a]) + + def test_an_error_no_number_and_dates_each_period(self): + e = mo.Variable([2.0, 2.0, 2.0]) / mo.Variable([1.0, 0.0, 2.0]) + blank = mo.Variable([None, None, 3.0]) + assert mo.LARGE([e, blank], 1).value == [2.0, "#DIV/0!", 3.0] + assert mo.PERCENTILE_INC([blank, mo.Variable(["x", None, 1.0])], 0.5).value == [ + "#NUM!", "#NUM!", 2.0] + assert mo.RANK_EQ(1.0, [blank, mo.Variable(["x", None, 1.0])]).value == [ + "#N/A", "#N/A", 2] + d1 = mo.Variable([dt.date(2026, 1, 31), dt.date(2026, 2, 28)]) + d2 = mo.Variable([dt.date(2026, 3, 31), dt.date(2026, 3, 2)]) + out = mo.SMALL([d1, d2], 1) + assert out.value == [dt.date(2026, 1, 31), dt.date(2026, 2, 28)] + assert out.value_type == "datetime" + + def test_units(self): + usd = [mo.Variable([1.0, 2.0], unit="USD"), mo.Variable([3.0, 1.0], unit="USD")] + assert str(mo.LARGE(usd, 1)._unit) == "USD" + assert mo.RANK_EQ(usd[0], usd)._unit is None + with pytest.raises(ValueError, match="mo.PERCENTILE_INC takes its values in one unit"): + mo.PERCENTILE_INC([usd[0], mo.Variable([1.0, 1.0], unit="hr")], 0.5) + with pytest.raises(ValueError, match="mo.RANK_EQ takes its values in one unit"): + mo.RANK_EQ(mo.Variable([1.0, 1.0], unit="hr"), usd) + + def test_a_tracked_line_in_the_list_lifts_per_track(self): + m = mo.Model("m", tracks=mo.Tracks("plan", "actual"), default_grain="month", + default_start="2026-01", default_periods=2) + with m: + m.x = mo.Variable(plan=[1.0, 5.0], actual=[2.0, 2.0]) + m.y = mo.Variable(plan=[3.0, 3.0], actual=[1.0, 4.0]) + m.top = mo.LARGE([m.x, m.y], 1) + assert dict(m.top._value.items()) == {"plan": [3.0, 5.0], "actual": [2.0, 4.0]} + + +# ══════════════════════════════════════════════════════════════════════ +# The registry and the loop container +# ══════════════════════════════════════════════════════════════════════ + +REDUCTIONS = ("MEDIAN", "STDEV_S", "STDEV_P", "VAR_S", "VAR_P", "STDEV", "STDEVP", "VAR", "VARP") +ONE_RANGE = ("PERCENTILE_INC", "PERCENTILE_EXC", "QUARTILE_INC", "QUARTILE_EXC", "PERCENTILE", + "QUARTILE", "LARGE", "SMALL", "RANK_EQ", "RANK_AVG", "RANK") + + +class TestTheRegistry: + @pytest.mark.parametrize("name", REDUCTIONS + ONE_RANGE) + def test_two_values_make_a_call_one_per_period(self, name): + assert per_period_arity(name) == 2 + assert not admits_in_loop(name, 1) + assert admits_in_loop(name, 2) and admits_in_loop(name, 5) + + +def _loop_model(): + """A balance paid down by a cap the loop computes each period: the + median of a row and last period's balance, the larger of two loop rows + (a list of them), and their spread.""" + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with m: + m.x = mo.MultiVariable("X") + m.x.a = mo.Variable(list(A), display_name="a") + m.x.c = mo.Variable(list(C), display_name="c") + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + d.bal = (mo.lag(d.bal, fill=10.0) - d.paid).set_display_name("bal") + d.mid = mo.MEDIAN(m.x.a, mo.lag(d.bal, fill=10.0)).set_display_name("mid") + d.cap = (mo.lag(d.bal, fill=10.0) * 0.8).set_display_name("cap") + d.top = mo.LARGE([d.mid, d.cap], 1).set_display_name("top") + d.sd = mo.STDEV_P(d.mid, d.cap).set_display_name("sd") + d.paid = mo.MIN(d.top - d.sd, mo.lag(d.bal, fill=10.0)).set_display_name("paid") + d.mix = mo.LARGE([m.x.a, m.x.c, d.paid], 1).set_display_name("mix") + return m + + +class TestInsideALoop: + def test_a_statistic_over_two_or_more_values_runs_one_period_at_a_time(self): + m = _loop_model() + d = m.d + assert d.mid.value == [6.5, 5.375, 2.5] + assert d.cap.value == pytest.approx([8.0, 2.2, 0.0]) + assert d.top.value == [8.0, 5.375, 2.5] + assert d.sd.value == pytest.approx([0.75, 1.5875, 1.25]) + assert d.paid.value == [7.25, 2.75, 0.0] + assert d.bal.value == [2.75, 0.0, 0.0] + assert d.mix.value == [7.25, 9.0, 5.0] + + def test_the_rows_match_the_same_calls_outside_a_loop(self): + m = _loop_model() + d = m.d + assert mo.LARGE([d.mid, d.cap], 1).value == d.top.value + assert mo.STDEV_P(d.mid, d.cap).value == d.sd.value + + def test_a_list_of_loop_rows_read_before_the_loop_closes(self): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with m: + m.x = mo.MultiVariable("X") + m.x.a = mo.Variable(list(A), display_name="a") + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + d.early = mo.MEDIAN([d.bal, m.x.a]).set_display_name("early") + d.bal = (mo.lag(d.bal, fill=10.0) * 0.5).set_display_name("bal") + assert m.d.bal.value == [5.0, 2.5, 1.25] + assert m.d.early.value == [4.0, 5.25, 3.125] + + def test_the_loop_reads_a_row_s_cell_as_a_cell(self): + # A blank cell is skipped and an error a row holds is the result, + # as the workbook reads its cells — never 0, nor the #VALUE! that + # text typed into the call would give. + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with m: + m.x = mo.MultiVariable("X") + m.x.den = mo.Variable([1.0, 0.0, 2.0], display_name="den") + m.x.blank = mo.Variable([None, 4.0, None], display_name="blank") + m.x.k = mo.Variable(["#N/A", 0.5, 1.0], display_name="k") + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + d.r = (2.0 / m.x.den + mo.lag(d.n, fill=0.0) * 0.0).set_display_name("r") + d.mid = mo.MEDIAN(d.r, m.x.blank).set_display_name("mid") + d.big = mo.LARGE([d.r, m.x.blank], 1).set_display_name("big") + d.pk = mo.PERCENTILE_INC([d.r, m.x.blank], m.x.k).set_display_name("pk") + d.n = mo.IFERROR(d.mid + d.big + d.pk, 0.0).set_display_name("n") + assert m.d.r.value == [2.0, "#DIV/0!", 1.0] + assert m.d.mid.value == [2.0, "#DIV/0!", 1.0] + assert m.d.big.value == [2.0, "#DIV/0!", 1.0] + assert m.d.pk.value == ["#N/A", "#DIV/0!", 1.0] + assert m.d.n.value == [0.0, 0.0, 3.0] + # The same calls outside the loop read the same. + assert mo.MEDIAN(m.d.r, m.x.blank).value == m.d.mid.value + assert mo.PERCENTILE_INC([m.d.r, m.x.blank], m.x.k).value == m.d.pk.value + + @pytest.mark.parametrize("make", [ + lambda d: mo.MEDIAN(d.bal), + lambda d: mo.STDEV_S(d.bal), + lambda d: mo.PERCENTILE_INC(d.bal, 0.5), + lambda d: mo.RANK_EQ(d.bal, d.bal), + ]) + def test_one_row_alone_reads_a_whole_series_and_is_refused(self, make): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with pytest.raises(ValueError, match="reads a whole series") as err: + with m: + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + d.bal = mo.lag(d.x, fill=10.0) * 1.37 + d.stat = make(d) + d.x = d.bal - d.stat + assert "one call per period" in str(err.value) + + def test_the_loop_derives_the_units_by_the_function_s_law(self): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=2) + with m: + m.x = mo.MultiVariable("X") + m.x.a = mo.Variable([1.0, 2.0], unit="USD", display_name="a") + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + d.bal = (mo.lag(d.bal, fill=m.x.a[0]) + d.step).set_display_name("bal") + d.mid = mo.MEDIAN(mo.lag(d.bal, fill=m.x.a[0]), m.x.a).set_display_name("mid") + d.v = mo.VAR_P(mo.lag(d.bal, fill=m.x.a[0]), m.x.a).set_display_name("v") + d.step = (d.mid * 0.5 + d.v / d.mid * 0.0).set_display_name("step") + assert m.d.v.value == [0.0, 0.0625] + assert m.d.bal.value == [1.5, 2.375] + assert str(m.d.v._unit) == "USD^2" + assert str(m.d.mid._unit) == "USD" + assert str(m.d.step._unit) == "USD" + + def test_two_units_in_a_loop_are_refused(self): + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=2) + with pytest.raises(ValueError, match="mo.MEDIAN takes its values in one unit"): + with m: + m.x = mo.MultiVariable("X") + m.x.a = mo.Variable([1.0, 2.0], unit="USD", display_name="a") + m.x.h = mo.Variable([1.0, 2.0], unit="hr", display_name="h") + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + d.bal = (mo.lag(d.bal, fill=m.x.a[0]) + (d.mid > 0.0) * 0.0 + ).set_display_name("bal") + d.mid = mo.MEDIAN(mo.lag(d.bal, fill=m.x.a[0]), m.x.h).set_display_name("mid") + + +# ══════════════════════════════════════════════════════════════════════ +# The workbook: one call per column over this column's cells +# ══════════════════════════════════════════════════════════════════════ + + +class TestTheWorkbook: + def test_operands_are_this_column_s_cells(self, tmp_path): + m = _model() + x, i = m.x, m.i + with m: + x.mid = mo.MEDIAN(x.a, x.b).set_display_name("mid") + x.sd = mo.STDEV_S(x.a, x.b, x.c).set_display_name("sd") + x.fl = mo.MEDIAN(x.a, i.floor).set_display_name("fl") + x.lit = mo.VAR_P(x.a, [1.0, 9.0, 1.0]).set_display_name("lit") + x.twice = (mo.MEDIAN(x.a, x.b) * 2).set_display_name("twice") + rows = _rows(_book(m, tmp_path)["X"]) + assert rows["mid"] == ["=MEDIAN(B2, B3)", "=MEDIAN(C2, C3)", "=MEDIAN(D2, D3)"] + assert rows["sd"] == [f"=_xlfn.STDEV.S({c}2, {c}3, {c}4)" for c in "BCD"] + assert rows["fl"] == [f"=MEDIAN({c}2, I!B1)" for c in "BCD"] + assert rows["lit"] == ["=_xlfn.VAR.P(B2, 1.0)", "=_xlfn.VAR.P(C2, 9.0)", + "=_xlfn.VAR.P(D2, 1.0)"] + assert rows["twice"] == [f"=MEDIAN({c}2, {c}3) * 2" for c in "BCD"] + assert check_compat(m, "excel") == [] + + def test_a_list_of_sibling_lines_is_the_run_of_their_cells_in_each_column(self, tmp_path): + m = _model() + x, i = m.x, m.i + with m: + x.mid = mo.MEDIAN(_lines(m)).set_display_name("mid") + x.q1 = mo.PERCENTILE_INC(_lines(m), i.k).set_display_name("q1") + x.top = mo.LARGE(_lines(m), 1).set_display_name("top") + x.rank = mo.RANK_EQ(x.c, _lines(m)).set_display_name("rank") + rows = _rows(_book(m, tmp_path)["X"]) + assert rows["mid"] == [f"=MEDIAN({c}2:{c}4)" for c in "BCD"] + assert rows["q1"] == [f"=_xlfn.PERCENTILE.INC({c}2:{c}4, I!B2)" for c in "BCD"] + assert rows["top"] == [f"=LARGE({c}2:{c}4, 1)" for c in "BCD"] + assert rows["rank"] == [f"=_xlfn.RANK.EQ({c}4, {c}2:{c}4)" for c in "BCD"] + assert check_compat(m, "excel") == [] + + def test_a_records_table_of_yearly_lines_gives_a_statistic_per_year(self, tmp_path): + m = _model() + revs = _revs(m) + with m.s as s: + s.median = mo.MEDIAN(revs).set_display_name("median") + s.q1 = mo.PERCENTILE_INC(revs, m.i.k).set_display_name("q1") + s.sd = mo.STDEV_S(revs).set_display_name("sd") + with m.peers as p: + for rec in (p.a, p.b, p.c): + rec.rank = mo.RANK_EQ(rec.rev, revs).set_display_name("Rank") + book = _book(m, tmp_path) + rows = _rows(book["Summary"]) + assert rows["median"] == [f"=MEDIAN(Peers!{c}3:{c}5)" for c in "BCD"] + assert rows["q1"] == [f"=_xlfn.PERCENTILE.INC(Peers!{c}3:{c}5, I!B2)" for c in "BCD"] + assert rows["sd"] == [f"=_xlfn.STDEV.S(Peers!{c}3:{c}5)" for c in "BCD"] + assert m.s.median.value == [20.0, 30.0, 25.0] + assert m.s.q1.value == [15.0, 20.0, 17.5] + ws = book["Peers"] + ranks = {(cell.row, cell.column_letter): cell.value + for row in ws.iter_rows(min_row=8, max_row=10) for cell in row[1:4]} + assert ranks == { + (row, c): f"=_xlfn.RANK.EQ({c}{row - 5}, {c}3:{c}5)" + for row in (8, 9, 10) for c in "BCD" + } + assert [r.rank.value for r in (m.peers.a, m.peers.b, m.peers.c)] == [ + [3, 2, 2], [2, 3, 1], [1, 1, 3]] + assert check_compat(m, "excel") == [] + + def test_cells_that_form_no_run_carry_the_value(self, tmp_path): + m = _model() + x = m.x + with m: + x.gap = mo.PERCENTILE_INC([x.a, x.c], 0.5).set_display_name("gap") + x.back = mo.MEDIAN([x.c, x.b, x.a]).set_display_name("back") + x.sheets = mo.LARGE([x.a, m.peers.a.rev], 1).set_display_name("sheets") + rows = _rows(_book(m, tmp_path)["X"]) + assert rows["gap"] == [2.0, 8.5, 5.0] + assert rows["back"] == [3.0, 8.0, 5.0] + assert rows["sheets"] == [10.0, 30.0, 25.0] + # One entry per cell: each period's call carries its value. + assert [(i.kind, i.qpath) for i in check_compat(m, "excel")] == ( + [("value", x.gap.id)] * 3 + [("value", x.back.id)] * 3 + + [("value", x.sheets.id)] * 3) + + def test_a_field_of_scalar_lines_beside_a_series_is_its_range_in_every_column( + self, tmp_path): + m = _model() + with m: + m.i.lo = mo.Variable(1.0, display_name="lo") + m.i.hi = mo.Variable(10.0, display_name="hi") + m.x.mid = mo.MEDIAN([m.i.floor, m.i.k, m.i.lo, m.i.hi], m.x.a).set_display_name("mid") + rows = _rows(_book(m, tmp_path)["X"]) + assert rows["mid"] == [f"=MEDIAN(I!B1:B4, {c}2)" for c in "BCD"] + assert m.x.mid.value == [3.0, 5.0, 5.0] + + def test_the_reduction_of_one_line_and_the_scalars_are_unchanged(self, tmp_path): + m = _model() + with m: + m.s.red = mo.MEDIAN(m.x.a).set_display_name("red") + m.s.pct = mo.PERCENTILE_INC(m.x.a, m.i.k).set_display_name("pct") + m.i.pair = mo.MEDIAN(m.i.floor, m.i.k).set_display_name("pair") + book = _book(m, tmp_path) + assert _rows(book["Summary"])["red"][0] == "=MEDIAN(X!B2:D2)" + assert _rows(book["Summary"])["pct"][0] == "=_xlfn.PERCENTILE.INC(X!B2:D2, I!B2)" + assert _rows(book["I"])["pair"][0] == "=MEDIAN(B1, B2)" + + def test_inside_a_loop_container(self, tmp_path): + m = _loop_model() + rows = _rows(_book(m, tmp_path)["D"]) + assert rows["mid"] == ["=MEDIAN(X!B2, 10.0)", "=MEDIAN(X!C2, B2)", "=MEDIAN(X!D2, C2)"] + assert rows["top"] == [f"=LARGE({c}3:{c}4, 1)" for c in "BCD"] + assert rows["sd"] == [f"=_xlfn.STDEV.P({c}3, {c}4)" for c in "BCD"] + # Two sheets' cells are no run: the value, listed by check_compat. + assert rows["mix"] == [7.25, 9, 5] + assert [(i.kind, i.qpath) for i in check_compat(m, "excel")] == [ + ("value", m.d.mix.id)] * 3 + + def test_the_notebook_shows_the_per_cell_formula(self): + m = _model() + with m: + m.x.mid = mo.MEDIAN(m.x.a, m.x.b).set_display_name("mid") + html = m._repr_html_() + assert "MEDIAN(B2, B3)" in html + assert "B2:D2" not in html diff --git a/tests/test_text_functions.py b/tests/test_text_functions.py new file mode 100644 index 0000000..bd4ab16 --- /dev/null +++ b/tests/test_text_functions.py @@ -0,0 +1,778 @@ +# SPDX-License-Identifier: Apache-2.0 +"""``LEFT``, ``RIGHT``, ``MID``, ``TRIM``, ``FIND``, ``SEARCH``, +``SUBSTITUTE``, ``VALUE``, ``TEXT``, ``REPT`` and ``TEXTJOIN`` compute +what the spreadsheet computes, with the formula as the product. + +Each of the first ten is one call per period through the shared helper: +an operand reads as the cell does (a blank is empty text, a number its +General text, a date its serial number, TRUE the word), a count is +truncated to a whole number, numeric text is a number and other text +``#VALUE!`` where a number is asked for, the first spreadsheet error +among a period's operands is that period's result while text that only +starts with ``#`` is text, scalars broadcast, tracks lift, and no result +carries a unit. ``TEXTJOIN`` joins whole rows into one text. +""" + +from __future__ import annotations + +import logging +from datetime import date, datetime, timedelta + +import pytest + +import modeleon as mo +from modeleon.core.expr import FuncCall, Literal, VarRef +from modeleon.functions.registry import ( + admits_in_loop, + excel_name, + file_prefix, + is_elementwise, + is_range_taking, + range_positions, + spec, +) +from modeleon.functions.text import ( + FIND, + LEFT, + MID, + REPT, + RIGHT, + SEARCH, + SUBSTITUTE, + TEXT, + TEXTJOIN, + TRIM, + VALUE, +) + +ELEMENTWISE = { + "LEFT": LEFT, "RIGHT": RIGHT, "MID": MID, "TRIM": TRIM, "FIND": FIND, + "SEARCH": SEARCH, "SUBSTITUTE": SUBSTITUTE, "VALUE": VALUE, "TEXT": TEXT, + "REPT": REPT, +} + + +# ══════════════════════════════════════════════════════════════════════ +# Slicing text +# ══════════════════════════════════════════════════════════════════════ + + +class TestLeft: + @pytest.mark.parametrize("text, n, expected", [ + ("Alpha Fund", 5, "Alpha"), + ("abc", 1, "a"), + ("abc", 0, ""), + ("abc", 10, "abc"), + ("abcdef", 2.9, "ab"), + ("abcdef", "2", "ab"), + ("abc", -1, "#VALUE!"), + ("abc", "x", "#VALUE!"), + ("", 3, ""), + ]) + def test_the_first_characters(self, text, n, expected): + assert LEFT(text, n).value == expected + + def test_one_character_by_default(self): + assert LEFT("abc").value == "a" + assert LEFT("abc").formula == "LEFT('abc')" + + def test_a_blank_is_empty_text_and_a_blank_count_is_zero(self): + assert LEFT(None, 2).value == "" + assert LEFT("abc", mo.Variable(None)).value == "" + + def test_a_number_a_date_and_a_boolean_read_as_the_cell_shows_them(self): + assert LEFT(12345.5, 3).value == "123" + assert LEFT(1 / 3, 20).value == "0.333333333333333" + assert LEFT(2022.0, 4).value == "2022" + assert LEFT(999999999999999, 20).value == "999999999999999" + assert LEFT(0.0001, 20).value == "0.0001" + assert LEFT(date(2026, 1, 1), 4).value == "4602" + assert LEFT(mo.Variable(datetime(2026, 1, 1, 12)), 7).value == "46023.5" + assert LEFT(timedelta(hours=36), 3).value == "1.5" + assert LEFT(True, 2).value == "TR" + + @pytest.mark.parametrize("number, text", [ + (0.00001, "0.00001"), + (-0.00001, "-0.00001"), + (3e-7, "0.0000003"), + (5.6789012345e-8, "0.000000056789012345"), + (9.999999999999999e-19, "0.000000000000000001"), + (1e15, "1000000000000000"), + (1234567890123456, "1234567890123460"), + (1.2345678901234567e19, "12345678901234600000"), + (-1.2345678901234567e19, "-12345678901234600000"), + (999999.9999999999, "1000000"), + (123.45678901234568, "123.456789012346"), + (0.1 + 0.2, "0.3"), + # E notation from 1E20, and for a fraction whose spelling would + # pass 20 characters; the exponent has two digits at least. + (1e20, "1E+20"), + (1.2345678901234568e20, "1.23456789012346E+20"), + (1.2345678901234568e-5, "1.23456789012346E-05"), + (5.67890123456e-8, "5.67890123456E-08"), + (1e-19, "1E-19"), + (2e-50, "2E-50"), + ]) + def test_a_number_reads_as_its_general_text(self, number, text): + # Excel's General text: 15 significant digits, written out in full + # between those bounds. LibreOffice writes a 16th digit and three- + # digit exponents (pinned in test_excel_text.py). + assert LEFT(number, 30).value == text + + def test_a_number_no_cell_holds_is_an_error(self): + assert LEFT(float("inf"), 2).value == "#NUM!" + assert LEFT("abc", float("nan")).value == "#NUM!" + assert REPT("a", float("inf")).value == "#NUM!" + + def test_a_series_and_a_count_that_varies_by_period(self): + names = mo.Variable(["Alpha Fund", "beta co", None]) + assert LEFT(names, 2).value == ["Al", "be", ""] + assert LEFT(names, [1, 2, 3]).value == ["A", "be", ""] + assert LEFT(names, mo.Variable(3)).value == ["Alp", "bet", ""] + + +class TestRight: + @pytest.mark.parametrize("text, n, expected", [ + ("AB-123", 3, "123"), + ("abc", 1, "c"), + ("abc", 0, ""), + ("abc", 10, "abc"), + ("abcdef", 2.9, "ef"), + ("abc", -1, "#VALUE!"), + (1234.5, 2, ".5"), + ]) + def test_the_last_characters(self, text, n, expected): + assert RIGHT(text, n).value == expected + + def test_one_character_by_default_and_a_series(self): + assert RIGHT("abc").value == "c" + assert RIGHT(mo.Variable(["AB-123", "CD-45", None]), 3).value == ["123", "-45", ""] + + +class TestMid: + @pytest.mark.parametrize("text, start, n, expected", [ + ("AB-123", 4, 2, "12"), + ("abcdef", 1, 3, "abc"), + ("abcdef", 2, 100, "bcdef"), + ("abcdef", 7, 2, ""), + ("abcdef", 2.7, 2.2, "bc"), + ("abcdef", 0, 1, "#VALUE!"), + ("abcdef", 1, -1, "#VALUE!"), + ("abcdef", "x", 1, "#VALUE!"), + ]) + def test_from_a_one_based_position(self, text, start, n, expected): + assert MID(text, start, n).value == expected + + def test_a_series(self): + codes = mo.Variable(["AB-123", "CD-45", "EF-6"]) + assert MID(codes, 4, 2).value == ["12", "45", "6"] + assert MID(codes, [1, 4, 1], 2).value == ["AB", "45", "EF"] + + +class TestTrim: + def test_collapses_runs_of_spaces(self): + assert TRIM(" a b ").value == "a b" + assert TRIM("a").value == "a" + assert TRIM(" ").value == "" + + def test_spaces_only(self): + assert TRIM("\ta b c ").value == "\ta b c" + + def test_a_blank_a_number_and_a_series(self): + assert TRIM(None).value == "" + assert TRIM(12.5).value == "12.5" + assert TRIM(mo.Variable([" x ", "y z", None])).value == ["x", "y z", ""] + + +# ══════════════════════════════════════════════════════════════════════ +# Finding text +# ══════════════════════════════════════════════════════════════════════ + + +class TestFind: + @pytest.mark.parametrize("find, within, expected", [ + ("-", "AB-123", 3), + ("a", "banana", 2), + ("A", "banana", "#VALUE!"), + ("na", "banana", 3), + ("x", "abc", "#VALUE!"), + ("", "abc", 1), + ("?", "a?b", 2), + ("*", "a*b", 2), + ("a", "", "#VALUE!"), + (1, 2015, 3), + ]) + def test_case_sensitive_without_wildcards(self, find, within, expected): + assert FIND(find, within).value == expected + + @pytest.mark.parametrize("start, expected", [ + (1, 2), (2, 2), (3, 4), (2.9, 2), (5, 6), (6, 6), (7, "#VALUE!"), + (0, "#VALUE!"), (-1, "#VALUE!"), + ]) + def test_from_a_start_position(self, start, expected): + assert FIND("a", "banana", start).value == expected + + def test_empty_text_is_found_at_the_start_position(self): + # Excel's documented rule: empty find_text matches at start_num, + # and a start past the end is #VALUE! whatever is looked for. + # LibreOffice gives #VALUE! for any empty find_text (pinned in + # test_excel_text.py). + assert FIND("", "abc").value == 1 + assert FIND("", "abc", 3).value == 3 + assert FIND("", "abc", 4).value == "#VALUE!" + assert FIND("", "").value == "#VALUE!" + assert FIND("", "abc", 0).value == "#VALUE!" + assert SEARCH("", "abc").value == 1 + assert SEARCH("", "abc", 4).value == "#VALUE!" + assert SEARCH("*", "").value == "#VALUE!" + + def test_a_series_a_position_type_and_the_formula(self): + names = mo.Variable(["Alpha Fund", "beta co", None]) + found = FIND("a", names, 2) + assert found.value == [5, 4, "#VALUE!"] + assert found.value_type == 'int' and found.var_type == 'list' + assert FIND("-", "AB-123").formula == "FIND('-', 'AB-123')" + assert FIND("-", "AB-123", 2).formula == "FIND('-', 'AB-123', 2)" + + +class TestSearch: + @pytest.mark.parametrize("find, within, expected", [ + ("A", "banana", 2), + ("NA", "banana", 3), + ("b*d", "xabcdx", 3), + ("a?c", "xabcx", 2), + ("a~?", "xa?b", 2), + ("~*", "a*b", 2), + ("~~", "a~b", 2), + ("~a", "xab", 2), + ("a~", "a~b", 1), + ("~", "a~b", 2), + ("?", "ab", 1), + ("x", "abc", "#VALUE!"), + ("a*f", "Alpha Fund", 1), + ("a*f", "beta co", "#VALUE!"), + ]) + def test_case_insensitive_with_wildcards(self, find, within, expected): + assert SEARCH(find, within).value == expected + + def test_from_a_start_position(self): + assert SEARCH("a", "Banana", 3).value == 4 + assert SEARCH("a", "Banana", 7).value == "#VALUE!" + assert SEARCH("a", "Banana", 0).value == "#VALUE!" + + def test_a_series_and_the_formula(self): + names = mo.Variable(["Alpha Fund", "beta co", None]) + assert SEARCH("a*f", names).value == [1, "#VALUE!", "#VALUE!"] + assert SEARCH("A", "banana", 3).formula == "SEARCH('A', 'banana', 3)" + + +# ══════════════════════════════════════════════════════════════════════ +# Rewriting text +# ══════════════════════════════════════════════════════════════════════ + + +class TestSubstitute: + @pytest.mark.parametrize("text, old, new, expected", [ + ("AB-123", "-", "", "AB123"), + ("a.b.c", ".", ",", "a,b,c"), + ("banana", "a", "o", "bonono"), + ("banana", "A", "o", "banana"), + ("abc", "", "x", "abc"), + ("abc", "x", "y", "abc"), + (1234.5, ".", ",", "1234,5"), + (None, "a", "b", ""), + ]) + def test_every_occurrence(self, text, old, new, expected): + assert SUBSTITUTE(text, old, new).value == expected + + @pytest.mark.parametrize("text, old, new, instance, expected", [ + ("banana", "a", "o", 1, "bonana"), + ("banana", "a", "o", 3, "banano"), + ("banana", "a", "o", 4, "banana"), + ("abab", "b", "x", 1.9, "axab"), + ("aaa", "aa", "b", 1, "ba"), + ("aaa", "aa", "b", 2, "aaa"), + ("a.b.c", ".", "", 0, "#VALUE!"), + ("a.b.c", ".", "", -1, "#VALUE!"), + ("a.b.c", ".", "", "x", "#VALUE!"), + ]) + def test_one_instance(self, text, old, new, instance, expected): + assert SUBSTITUTE(text, old, new, instance).value == expected + + def test_a_series_and_the_formula(self): + names = mo.Variable(["Alpha Fund", "beta co", None]) + assert SUBSTITUTE(names, "a", "A", 1).value == ["AlphA Fund", "betA co", ""] + assert SUBSTITUTE(names, " ", "_").value == ["Alpha_Fund", "beta_co", ""] + assert SUBSTITUTE("a-b", "-", "").formula == "SUBSTITUTE('a-b', '-', '')" + assert SUBSTITUTE("a-b", "-", "", 1).formula == "SUBSTITUTE('a-b', '-', '', 1)" + + +class TestRept: + @pytest.mark.parametrize("text, n, expected", [ + ("ab", 3, "ababab"), + ("ab", 2.9, "abab"), + ("ab", 0, ""), + ("", 5, ""), + ("#", 3, "###"), + (1.5, 2, "1.51.5"), + (None, 3, ""), + ("ab", -1, "#VALUE!"), + ("ab", "x", "#VALUE!"), + ("a", 32767, "a" * 32767), + ("a", 32768, "#VALUE!"), + ]) + def test_repeats(self, text, n, expected): + assert REPT(text, n).value == expected + + def test_a_series_and_the_formula(self): + assert REPT("|", mo.Variable([3, 0, 1])).value == ["|||", "", "|"] + assert REPT("|", 3).formula == "REPT('|', 3)" + + +# ══════════════════════════════════════════════════════════════════════ +# Numbers from text and text from numbers +# ══════════════════════════════════════════════════════════════════════ + + +class TestValue: + @pytest.mark.parametrize("text, expected", [ + ("1,234", 1234), + ("1,234,567.5", 1234567.5), + ("12%", 0.12), + ("(1,234)", -1234), + ("(5)", -5), + ("1e3", 1000.0), + (" -12.5% ", -0.125), + ("+5", 5), + (".5", 0.5), + ("5.", 5.0), + ("-0.75", -0.75), + ("1234,567", 1234567), + ("1,234.", 1234.0), + ("1,234.5e2", 123450.0), + ("- 5", -5), + ("5 %", 0.05), + ("( 5 )", -5), + ("(1,234)%", -12.34), + ("-5%", -0.05), + ("1E+15", 1e15), + ]) + def test_numeric_text(self, text, expected): + assert VALUE(text).value == expected + + def test_an_iso_date_is_its_serial(self): + assert VALUE("2026-01-01").value == 46023 + assert VALUE(" 2026-01-31 ").value == 46053 + assert VALUE("2026-13-01").value == "#VALUE!" + + @pytest.mark.parametrize("text, expected", [ + ("12:00", 0.5), + ("1:30", 0.0625), + ("01:30", 0.0625), + ("12:30", 12.5 / 24), + ("12:5", (12 * 60 + 5) / 1440), + ("25:00", 25 / 24), + ("12:30:45", (12 * 3600 + 30 * 60 + 45) / 86400), + ("12:30:45.5", (12 * 3600 + 30 * 60 + 45.5) / 86400), + ("0:00", 0.0), + ("24:00", 1.0), + ("1:30 PM", 0.5625), + ("1:30pm", 0.5625), + ("12:00 AM", 0.0), + ("12:30 AM", 0.5 / 24), + ("12 PM", 0.5), + ("1 AM", 1 / 24), + ("12:30:45 PM", (12 * 3600 + 30 * 60 + 45) / 86400), + (" 3:00 PM ", 0.625), + ("+1:30", 0.0625), + ("+12 PM", 0.5), + ("2026-01-31 18:00", 46053.75), + ("2026-01-31 12:30:15", 46053 + (12 * 3600 + 30 * 60 + 15) / 86400), + ("2026-01-31 25:00", 46053 + 25 / 24), + ]) + def test_a_time_of_day_is_its_fraction_of_a_day(self, text, expected): + assert VALUE(text).value == pytest.approx(expected, abs=1e-12) + + @pytest.mark.parametrize("text", [ + "12:60", "0:90", "1:30:60", "13:30 PM", "1:30 P", "1:30 a.m.", ":30", "12:", "1.5:30", + "-1:30", "-12 PM", "+-1:30", "2026-01-31T12:00", "2026-01-31 1:30 P", + "2026-02-30 12:00", + ]) + def test_other_time_text_is_an_error(self, text): + assert VALUE(text).value == "#VALUE!" + + @pytest.mark.parametrize("text", [ + "abc", "", " ", "1,23", "12,34,567", "1,234,", ",123", "1 234", "$1,234", "(-5)", "-(5)", + "+-5", "--5", "%", "%5", "5%%", "(5%)", "1e3%", "1e", "0x10", "1.2.3", "01/02/2026", + "TRUE", "\u0663", True, + ]) + def test_other_text_is_an_error(self, text): + assert VALUE(text).value == "#VALUE!" + + def test_a_number_too_large_for_a_cell(self): + assert VALUE("1e999").value == "#NUM!" + assert VALUE("1" + "0" * 400).value == "#NUM!" + assert VALUE("1" + "0" * 5000).value == "#NUM!" + + def test_an_integral_text_is_an_int_while_a_cell_holds_it_exactly(self): + assert VALUE("123456789012345").value == 123456789012345 + assert isinstance(VALUE("000000000000001").value, int) + long = VALUE("12345678901234567891").value + assert isinstance(long, float) and long == 1.2345678901234567e19 + + def test_a_number_a_blank_and_a_date(self): + assert VALUE(12.5).value == 12.5 + assert VALUE(None).value == 0 + assert VALUE(date(2026, 1, 1)).value == 46023 + assert VALUE(timedelta(hours=36)).value == 1.5 + assert VALUE(float("inf")).value == "#NUM!" + + def test_a_series_its_type_and_the_formula(self): + texts = mo.Variable(["1,234", "12%", "x"], display_name="texts") + got = VALUE(texts) + assert got.value == [1234, 0.12, "#VALUE!"] + assert got.value_type == 'float' and got.var_type == 'list' + assert got.formula == "VALUE(texts)" + + +class TestText: + @pytest.mark.parametrize("value, fmt, expected", [ + (2.5, "0", "3"), + (-2.5, "0", "-3"), + (0.5, "0", "1"), + (-0.4, "0", "-0"), + (12.345, "0.0", "12.3"), + (1.005, "0.00", "1.01"), + (1234.5678, "0.000", "1234.568"), + (1234.5678, "#,##0", "1,235"), + (-1234.5, "#,##0", "-1,235"), + (1e6, "#,##0", "1,000,000"), + (1234.5678, "#,##0.00", "1,234.57"), + (1234.5678, "0%", "123457%"), + (0.1234, "0.0%", "12.3%"), + (0.125, "0.00%", "12.50%"), + (7, "0.00", "7.00"), + (None, "0", "0"), + (5, "00", "05"), + (-5, "000", "-005"), + (1234, "00", "1234"), + (0.5, "00", "01"), + (7, "000%", "700%"), + (3, "000.0", "003.0"), + # the padding counts digits; the thousands are grouped after it + (5, "#,##00", "05"), + (1234, "#,##000000", "001,234"), + (-1234.5, "#,##000000.0", "-001,234.5"), + (1234567, "#,##0000000000", "0,001,234,567"), + ]) + def test_number_formats(self, value, fmt, expected): + assert TEXT(value, fmt).value == expected + + @pytest.mark.parametrize("fmt, expected", [ + ("yyyy-mm-dd", "2026-01-01"), + ("dd.mm.yyyy", "01.01.2026"), + ("mmm yyyy", "Jan 2026"), + ("yyyy", "2026"), + ("yy", "26"), + ("mmmm d, yyyy", "January 1, 2026"), + ("d mmmm yyyy", "1 January 2026"), + ("ddd dd", "Thu 01"), + ("dddd", "Thursday"), + ("m/d/yyyy", "1/1/2026"), + ("YYYY-MM-DD", "2026-01-01"), + ("mmm", "Jan"), + ("mmmmm", "J"), + ("mmmmmm", "J1"), + ("ddddd", "Thursday1"), + ]) + def test_date_formats_over_a_date_and_its_serial(self, fmt, expected): + assert TEXT(date(2026, 1, 1), fmt).value == expected + assert TEXT(46023, fmt).value == expected + + def test_general(self): + assert TEXT(1234.5678, "General").value == "1234.5678" + assert TEXT(1 / 3, "general").value == "0.333333333333333" + assert TEXT(0.1 + 0.2, "General").value == "0.3" + assert TEXT(2022.0, "@").value == "2022" + assert TEXT(date(2026, 1, 1), "General").value == "46023" + assert TEXT(0.00001, "General").value == "0.00001" + assert TEXT(1e15, "General").value == "1000000000000000" + assert TEXT(1e20, "General").value == "1E+20" + + @pytest.mark.parametrize("value, fmt, expected", [ + (123456789012345678.0, "0", "123456789012346000"), + (123456789012345678.0, "#,##0.00", "123,456,789,012,346,000.00"), + (1234567890123456, "#,##0", "1,234,567,890,123,460"), + (1 / 3, "0.000000000000000000", "0.333333333333333000"), + (0.1 + 0.2, "0.000000000000000000", "0.300000000000000000"), + (0.1 + 0.2, "0.00000000000000000%", "30.00000000000000000%"), + (12345678901234.5678, "0.0000", "12345678901234.6000"), + (2.675, "0.00", "2.68"), + (1e300, "0", "1" + "0" * 300), + ]) + def test_a_format_shows_15_significant_digits(self, value, fmt, expected): + assert TEXT(value, fmt).value == expected + + @pytest.mark.parametrize("value, expected", [ + (1.5, ""), (0, ""), (-1.5, ""), (None, ""), ("12", ""), (date(2026, 1, 1), ""), + ("abc", "abc"), (True, "TRUE"), + ]) + def test_an_empty_format_shows_nothing(self, value, expected): + assert TEXT(value, "").value == expected + + def test_a_number_no_cell_holds_is_an_error(self): + assert TEXT(float("nan"), "0").value == "#NUM!" + assert TEXT(float("inf"), "General").value == "#NUM!" + + def test_text_and_booleans(self): + assert TEXT("12", "0.00").value == "12.00" + assert TEXT("12%", "0.00").value == "0.12" + assert TEXT("2026-01-01", "mmm yyyy").value == "Jan 2026" + assert TEXT("abc", "0").value == "abc" + assert TEXT("abc", "yyyy").value == "abc" + assert TEXT(True, "0").value == "TRUE" + assert TEXT(False, "0.0").value == "FALSE" + + def test_the_format_is_text_even_when_it_starts_with_a_hash(self): + assert TEXT(mo.Variable([1234.5, -0.4, None]), "#,##0").value == ["1,235", "-0", "0"] + assert TEXT(1234.5, mo.Variable("#,##0.0")).value == "1,234.5" + + def test_a_format_that_varies_by_period(self): + got = TEXT(1234.5, mo.Variable(["0", "0.0", "#,##0"])) + assert got.value == ["1235", "1234.5", "1,235"] + assert got.var_type == 'list' + + def test_a_series_its_type_and_the_formula(self): + amounts = mo.Variable([1234.5678, -0.4, 2.5], display_name="amounts") + got = TEXT(amounts, "#,##0.00") + assert got.value == ["1,234.57", "-0.40", "2.50"] + assert got.value_type == 'string' and got.var_type == 'list' + assert got.formula == "TEXT(amounts, '#,##0.00')" + + def test_an_unknown_format_warns_once_and_gives_general_text(self, caplog): + with caplog.at_level(logging.WARNING, logger="modeleon.functions.text"): + got = TEXT(mo.Variable([1.5, 2.5]), "0.0E+00") + assert got.value == ["1.5", "2.5"] + warnings = [r for r in caplog.records if "0.0E+00" in r.getMessage()] + assert len(warnings) == 1 + with caplog.at_level(logging.WARNING, logger="modeleon.functions.text"): + caplog.clear() + assert TEXT(46023, "hh:mm").value == "46023" + assert any("hh:mm" in r.getMessage() for r in caplog.records) + + def test_a_known_format_does_not_warn(self, caplog): + with caplog.at_level(logging.WARNING, logger="modeleon.functions.text"): + TEXT(1.5, "0.0") + TEXT(46023, "yyyy-mm-dd") + TEXT(1.5, "General") + assert not caplog.records + + +# ══════════════════════════════════════════════════════════════════════ +# Joining a row +# ══════════════════════════════════════════════════════════════════════ + + +class TestTextjoin: + def test_joins_a_series_into_one_text(self): + names = mo.Variable(["Alpha Fund", "beta co", None], display_name="names") + got = TEXTJOIN(", ", True, names) + assert got.value == "Alpha Fund, beta co" + assert got.var_type == 'scalar' and got.value_type == 'string' + assert got.formula == "TEXTJOIN(', ', True, names)" + + def test_ignore_empty(self): + assert TEXTJOIN("-", True, "a", "", None, "b").value == "a-b" + assert TEXTJOIN("-", False, "a", "", None, "b").value == "a---b" + assert TEXTJOIN("-", 0, "a", "", "b").value == "a--b" + assert TEXTJOIN("-", None, "a", "", "b").value == "a--b" + assert TEXTJOIN("-", 2, "a", "", "b").value == "a-b" + assert TEXTJOIN("-", "0", "a", "", "b").value == "a--b" + assert TEXTJOIN("-", "TRUE", "a", "", "b").value == "a-b" + assert TEXTJOIN("-", "false", "a", "", "b").value == "a--b" + assert TEXTJOIN("-", "x", "a", "", "b").value == "#VALUE!" + assert TEXTJOIN("-", mo.Variable(True), "a", "", "b").value == "a-b" + assert TEXTJOIN("-", False, mo.Variable(["x", None, "y"])).value == "x--y" + + def test_scalars_series_and_the_delimiter_from_a_cell(self): + codes = mo.Variable(["AB-123", "CD-45", "EF-6"]) + sep = mo.Variable(" | ") + assert TEXTJOIN(sep, False, codes, "end").value == "AB-123 | CD-45 | EF-6 | end" + assert TEXTJOIN("", True, "a", "b").value == "ab" + assert TEXTJOIN(",", True, ["a", "b"], "c").value == "a,b,c" + assert TEXTJOIN("#", True, "a", "b").value == "a#b" + + def test_operands_read_as_the_spreadsheet_reads_them(self): + assert TEXTJOIN(", ", True, 1.5, True, date(2026, 1, 1), 2022.0).value == \ + "1.5, TRUE, 46023, 2022" + + def test_an_error_among_the_operands_is_the_result(self): + assert TEXTJOIN(",", True, mo.Variable(["a", "#DIV/0!", "b"])).value == "#DIV/0!" + assert TEXTJOIN(mo.Variable("#N/A"), True, "a").value == "#N/A" + assert TEXTJOIN(",", True, "#12", "b").value == "#12,b" + + def test_a_text_longer_than_a_cell_holds_is_an_error(self): + assert TEXTJOIN("", True, "a" * 20000, "b" * 20000).value == "#VALUE!" + + def test_a_series_delimiter_or_flag_is_refused(self): + with pytest.raises(ValueError, match="delimiter is one value"): + TEXTJOIN(mo.Variable([",", ";"]), True, "a") + with pytest.raises(ValueError, match="ignore_empty is one value"): + TEXTJOIN(",", mo.Variable([True, False]), "a") + + def test_no_text_is_refused(self): + with pytest.raises(ValueError, match=r"Syntax: mo\.TEXTJOIN.*delimiter and ignore_empty"): + TEXTJOIN(",", True) + + def test_a_line_with_tracks_is_refused(self): + m = mo.Model("m", tracks=mo.Tracks("plan", "actual"), + default_grain="month", default_start="2026-01", default_periods=2) + with m: + m.x = mo.Variable(plan=["a", "b"], actual=["c", None]) + with pytest.raises(ValueError, match="one track at a time"): + TEXTJOIN(",", True, m.x) + + def test_the_ast_and_the_registry(self): + names = mo.Variable(["a", "b"]) + expr = TEXTJOIN(", ", True, names)._expr + assert isinstance(expr, FuncCall) and expr.func == "TEXTJOIN" + assert isinstance(expr.args[0], Literal) and expr.args[0].value == ", " + assert isinstance(expr.args[1], Literal) and expr.args[1].value is True + assert isinstance(expr.args[2], VarRef) and expr.args[2].var is names + assert spec("TEXTJOIN").shape == 'range' + assert is_range_taking("TEXTJOIN") and range_positions("TEXTJOIN") is None + assert file_prefix("TEXTJOIN") == "_xlfn." and excel_name("TEXTJOIN") == "TEXTJOIN" + assert not admits_in_loop("TEXTJOIN", 3) + + +# ══════════════════════════════════════════════════════════════════════ +# The shared law, across the element-wise helpers +# ══════════════════════════════════════════════════════════════════════ + + +class TestTheSharedLaw: + def test_scalars_only_is_one_scalar(self): + for fn, args in ((LEFT, ("abc", 2)), (FIND, ("b", "abc")), (VALUE, ("12",)), + (TEXT, (1.5, "0.0")), (REPT, ("a", 2)), (TRIM, (" a ",))): + out = fn(*args) + assert out.var_type == 'scalar' and not isinstance(out.value, list), fn + + def test_a_one_value_series_broadcasts(self): + assert LEFT(mo.Variable(["abc"]), mo.Variable([1, 2, 3])).value == ["a", "ab", "abc"] + assert REPT(mo.Variable(["ab"]), [1, 2]).value == ["ab", "abab"] + + def test_series_of_different_lengths_are_refused_as_the_operators_refuse_them(self): + with pytest.raises(ValueError, match="different lengths"): + LEFT(mo.Variable(["abc", "de", "f"]), [1, 2]) + + def test_the_first_error_among_a_period_s_operands_is_the_result(self): + assert LEFT(mo.Variable(["ab", "#NUM!", "cd"]), 1).value == ["a", "#NUM!", "c"] + assert MID("#DIV/0!", "#VALUE!", 1).value == "#DIV/0!" + assert FIND("a", "banana", "#N/A").value == "#N/A" + assert TEXT(mo.Variable(["#REF!", 1.5]), "0.0").value == ["#REF!", "1.5"] + assert VALUE("#NAME?").value == "#NAME?" + assert SUBSTITUTE("abc", "b", "#NULL!").value == "#NULL!" + + def test_text_that_only_starts_with_a_hash_is_text(self): + assert LEFT("#12", 1).value == "#" + assert FIND("#", "a#b").value == 2 + assert SUBSTITUTE("#1", "#", "No. ").value == "No. 1" + assert REPT("#", 3).value == "###" + assert TEXT(1234, "#,##0").value == "1,234" + assert VALUE("#12").value == "#VALUE!" + + @pytest.mark.parametrize("call, value", [ + (lambda: LEFT(None, 2), ""), + (lambda: RIGHT(None), ""), + (lambda: MID(None, 1, 1), ""), + (lambda: TRIM(None), ""), + (lambda: FIND("0", None), "#VALUE!"), + (lambda: SEARCH(None, "abc"), 1), + (lambda: SUBSTITUTE(None, "a", "b"), ""), + (lambda: SUBSTITUTE("a0b", "0", None), "ab"), + (lambda: SUBSTITUTE("abc", None, "x"), "abc"), + (lambda: TEXT(1.5, None), ""), + (lambda: REPT(None, 3), ""), + (lambda: TEXTJOIN(None, False, "a", "b"), "ab"), + (lambda: TEXTJOIN("-", False, "a", None, "b"), "a--b"), + (lambda: mo.LEN(None), 0), + (lambda: mo.UPPER(None), ""), + (lambda: mo.LOWER(["AB", None]), ["ab", ""]), + (lambda: mo.CONCAT("a", None, "b"), "ab"), + ]) + def test_a_literal_blank_where_text_is_read_is_written_as_empty_text(self, call, value): + # The workbook spells a literal None as 0, whose text is "0"; a + # blank where text is read goes in as "" — what Python read. + out = call() + assert out.value == value + literals = [a.value for a in out._expr.args if isinstance(a, Literal)] + flat = [v for lit in literals for v in (lit if isinstance(lit, list) else [lit])] + assert None not in flat and "" in flat + + def test_a_gap_in_a_literal_list_is_empty_text(self): + out = LEFT(["ab", None, "cd"], 1) + assert out.value == ["a", "", "c"] + assert out._expr.args[0].value == ["ab", "", "cd"] + + def test_a_blank_count_stays_a_blank(self): + # A count reads a blank as 0, as the workbook's 0 does. + out = MID("abc", 2, None) + assert out.value == "" + assert out._expr.args[2].value is None + + def test_tracked_operands_lift_per_track(self): + m = mo.Model("m", tracks=mo.Tracks("plan", "actual"), + default_grain="month", default_start="2026-01", default_periods=2) + with m: + m.x = mo.Variable(plan=["a1", "b2"], actual=["c3", None]) + m.l = LEFT(m.x, 1) + m.f = FIND("1", m.x) + assert dict(m.l._value.items()) == {"plan": ["a", "b"], "actual": ["c", ""]} + assert dict(m.f._value.items()) == {"plan": [2, "#VALUE!"], "actual": ["#VALUE!", "#VALUE!"]} + assert m.l.var_type == 'list' + + def test_an_axised_operand_is_refused(self): + region = mo.Variable(["N", "S"], display_name="region") + by_region = mo.Variable(["x", "y"], indexed_by=[region]) + with pytest.raises(ValueError, match="LEFT"): + LEFT(by_region, 1) + + def test_value_types(self): + text = mo.Variable(["abc", "de"]) + for fn, args in ((LEFT, (text, 1)), (RIGHT, (text, 1)), (MID, (text, 1, 1)), + (TRIM, (text,)), (SUBSTITUTE, (text, "a", "b")), + (TEXT, (mo.Variable([1.5, 2.5]), "0")), (REPT, (text, 2))): + assert fn(*args).value_type == 'string', fn + assert FIND("a", text).value_type == 'int' + assert SEARCH("a", text).value_type == 'int' + assert VALUE(mo.Variable(["1", "2"])).value_type == 'float' + + def test_no_result_carries_a_unit(self): + labels = mo.Variable(["a1", "b2"], unit="$") + amount = mo.Variable([1234.5, 2.5], unit="$") + k = mo.Variable(2, unit="$") + for out in (LEFT(labels, k), RIGHT(labels, 1), MID(labels, 1, 1), TRIM(labels), + FIND("1", labels), SEARCH("1", labels), SUBSTITUTE(labels, "1", "2"), + VALUE(amount), TEXT(amount, "#,##0"), REPT(labels, k), + TEXTJOIN(",", True, labels)): + assert out._unit is None, out.formula + + def test_the_ast_is_a_call_over_the_operands(self): + names = mo.Variable(["abc", "de"]) + expr = LEFT(names, 2)._expr + assert isinstance(expr, FuncCall) and expr.func == "LEFT" + assert isinstance(expr.args[0], VarRef) and expr.args[0].var is names + assert isinstance(expr.args[1], Literal) and expr.args[1].value == 2 + for name, fn in ELEMENTWISE.items(): + assert fn(*(["abc", 1, 1][:_arity(name)]))._expr.func == name + + def test_every_helper_is_registered_one_call_per_period(self): + for name in ELEMENTWISE: + assert is_elementwise(name), name + assert admits_in_loop(name, 2), name + assert file_prefix(name) == "" and excel_name(name) == name, name + assert not spec(name).bare, name + + +def _arity(name: str) -> int: + return {"TRIM": 1, "VALUE": 1, "MID": 3, "SUBSTITUTE": 3}.get(name, 2) diff --git a/tests/test_time_axis.py b/tests/test_time_axis.py index f55188b..48baaf3 100644 --- a/tests/test_time_axis.py +++ b/tests/test_time_axis.py @@ -37,6 +37,21 @@ def test_length_matches_periods(self): def test_single_period(self): assert mo.Time('2024-01', '2024-01')._value == ['2024-01'] + def test_a_window_ending_in_9999_stops_on_its_last_label(self, tmp_path): + # The labels step between periods, never past the last one: the + # period after December 9999 is one no date can hold. + assert mo.Time('9999-10', periods=3)._value == ['9999-10', '9999-11', '9999-12'] + assert mo.Time('9999-Q3', periods=2, grain='quarter')._value == ['9999-Q3', '9999-Q4'] + assert mo.Time('9998', periods=2, grain='year')._value == ['9998', '9999'] + blend = mo.blend(given='actual', follow='plan', until='9999-10') + m = mo.Model('m', default_grain='month', default_start='9999-10', default_periods=3, + tracks=mo.Tracks('plan', 'actual', blend=blend)) + with m: + m.inp = mo.MultiVariable('Inputs') + m.inp.rev = mo.Variable(plan=[1.0, 2.0, 3.0], actual=[1.5, 0.0, 0.0]) + assert m.inp.rev.value['live'] == [1.5, 2.0, 3.0] + m.to_excel(tmp_path / 'm.xlsx') + class TestTimeIsAnAxisVariable: """``mo.Time`` is a Variable, so it works as an axis in ``indexed_by`` @@ -82,11 +97,14 @@ def test_nonpositive_periods(self): mo.Time('2024-01', periods=0) def test_unknown_grain_raises(self): - # 'week' is not a supported grain; 'fortnight' is nonsense — both refused. - with pytest.raises(ValueError): + # 'fortnight' is no ruler of the calendar; a week is, and reads + # an ISO week label, never a month's. + with pytest.raises(ValueError, match="Unknown time grain 'fortnight'"): mo.Time('2024-01', periods=4, grain='fortnight') - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Time week start must be 'YYYY-Www'"): mo.Time('2024-01', periods=4, grain='week') + t = mo.Time('2024-W01', periods=2, grain='week') + assert t.value == ['2024-W01', '2024-W02'] class TestMultiGrain: diff --git a/tests/test_time_golden.py b/tests/test_time_golden.py new file mode 100644 index 0000000..db3cf84 --- /dev/null +++ b/tests/test_time_golden.py @@ -0,0 +1,209 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The calendar's answers, frozen. + +Every question the engine asks its calendar — the labels of a window, +the buckets a coarser grain groups it into, the day-count weights, the +fields of ``mo.time``, the short first period, the header's Excel +spelling, the totals interleave — is asked here over a matrix of +windows and compared with the answers recorded in +``golden/time_calendar.json``. The file is the gate under which the +calendar's code may move: a refactor that changes any answer for the +grains covered here is a behaviour change and must say so by +regenerating the file on purpose:: + + MODELEON_REGEN_TIME_GOLDEN=1 pytest tests/test_time_golden.py +""" +import json +import os +from datetime import date +from pathlib import Path +from typing import Any, Dict, List + +import pytest + +from modeleon.compile.excel import header as H +from modeleon.compile.excel.totals import build_totals_plan, drawable_kinds, header_rows +from modeleon.core import time as T + +GOLDEN = Path(__file__).parent / "golden" / "time_calendar.json" +REGEN = os.environ.get("MODELEON_REGEN_TIME_GOLDEN") == "1" + +#: (grain, start, n, first_day) — whole windows, windows crossing a +#: year end, leap days, one-period windows, and short first periods. +WINDOWS = [ + ("day", "2026-01-05", 40, None), + ("day", "2025-12-25", 62, None), + ("day", "2024-02-27", 5, None), + ("day", "2026-03-30", 4, None), + ("day", "2026-01-01", 1, None), + ("month", "2026-01", 13, None), + ("month", "2025-11", 5, None), + ("month", "2024-12", 1, None), + ("month", "2026-01", 26, None), + ("month", "2026-02", 3, None), + ("quarter", "2026-Q1", 8, None), + ("quarter", "2025-Q4", 3, None), + ("quarter", "2026-Q3", 5, None), + ("quarter", "2026-Q1", 5, date(2026, 2, 1)), + ("quarter", "2026-Q2", 4, date(2026, 5, 1)), + ("quarter", "2025-Q4", 2, date(2025, 12, 1)), + ("year", "2026", 3, None), + ("year", "2024", 2, None), + ("year", "2026", 3, date(2026, 5, 1)), + ("year", "2026", 2, date(2026, 12, 1)), + ("year", "2025", 1, date(2025, 2, 1)), + ("week", "2026-W01", 10, None), + ("week", "2026-W51", 6, None), + ("week", "2025-W52", 4, None), + ("iso_year", "2026", 3, None), + # Hours: a whole day from midnight, across a day end, a month end, a + # year end and a leap day, and one hour alone. + ("hour", "2026-10-21T00", 24, None), + ("hour", "2026-10-21T22", 30, None), + ("hour", "2026-01-31T22", 5, None), + ("hour", "2025-12-31T22", 5, None), + ("hour", "2024-02-28T23", 26, None), + ("hour", "2026-03-29T01", 3, None), + ("hour", "2026-10-21T13", 1, None), +] + +STARTS = [ + date(2026, 1, 1), date(2026, 2, 1), date(2026, 4, 1), date(2026, 5, 17), + date(2026, 7, 1), date(2026, 12, 31), date(2024, 2, 29), +] + +KINDS = [("quarter", "year"), ("year", "quarter"), ("quarter",), ("year",), + ("month", "quarter"), ("month",), ("day", "month"), + ("week",), ("iso_year",), ("week", "iso_year"), ("week", "month"), ("year", "iso_year"), + ("day",), ("day", "week"), ("month", "day", "year")] + + +def _iso(v: Any) -> Any: + if isinstance(v, date): + return v.isoformat() + if isinstance(v, (list, tuple)): + return [_iso(x) for x in v] + if isinstance(v, dict): + return {str(k): _iso(x) for k, x in v.items()} + return v + + +def _ask(fn, *args, **kwargs) -> Any: + try: + return _iso(fn(*args, **kwargs)) + except (ValueError, KeyError, TypeError) as e: + return {"error": f"{type(e).__name__}: {e}"} + + +def _cell_chain(opening: bool): + """A fake anchor sheet: row letters S/E/D/M/I at column j+3; an + opening cell when asked.""" + def cell(field_: str, j: int): + if j < 0: + return "O9" if opening else None + return f"{field_[0].upper()}{j + 3}" + return cell + + +def _window_answers(grain: str, start: str, n: int, first_day: Any) -> Dict[str, Any]: + out: Dict[str, Any] = {} + out["labels"] = {s: _ask(T._period_labels, start, None, n, grain, s) for s in T.TIME_LABEL_STYLES} + out["weights"] = _ask(T.period_weights, grain, start, n, first_day) + out["fields"] = {f: _ask(T.time_field_series, f, start, grain, n, first_day) for f in T.TIME_FIELDS} + anchor = T._parse(start, grain) + out["first_day_of_start"] = _iso(T._first_day(anchor, grain)) + out["containing"] = _ask(T._containing_label, first_day or T._first_day(anchor, grain), grain) + out["days_in_first"] = _ask(T._days_in_period, anchor, grain) + out["step"] = _ask(lambda: T._format(T._step(anchor, grain), grain, "iso")) + out["count_to_last"] = _ask(lambda: T._count(anchor, T._parse(out["labels"]["iso"][-1], grain), grain)) + window = T.DefaultWindow(start, grain, n, None, first_day) + out["spelled_first_day"] = _ask(T.spelled_first_day, window) + out["spelled_first_day_sourced"] = _ask(T.spelled_first_day, T.DefaultWindow(start, grain, n, True, first_day)) + coarser = [g for g in T.GRAINS if T._ORDER[g] > T._ORDER[grain]] + out["targets"] = {} + for target in coarser: + out["targets"][target] = { + "buckets": _ask(T.bucket_ranges, grain, target, start, n), + "coarse_first_day": _ask(T.coarse_first_day, start, grain, first_day, target), + "target_label_of_start": _ask(T._target_label, anchor, grain, target), + "regrain_sum": _ask(T.regrain_series, [float(i + 1) for i in range(n)], grain, target, start, "sum", first_day), + "regrain_mean": _ask(T.regrain_series, [float(i % 3) for i in range(n)], grain, target, start, "mean", first_day), + } + out["spell_inline"] = { + f"{f}@{i}": _ask(H.spell_inline, f, i, grain, "B2", first_day) + for f in T.TIME_FIELDS for i in (0, 1, 2) + } + out["spell_anchor"] = { + f"{f}@{i}{'+o' if opening else ''}": _ask(H.spell_anchor, f, i, grain, "B2", _cell_chain(opening), first_day) + for f in T.TIME_FIELDS for i in (0, 1, 2) for opening in (False, True) + } + out["totals"] = {} + out["totals_notes"] = {} + labels = out["labels"]["finance"] + for kinds in KINDS: + # As the layout draws them: only the kinds this grain nests in, + # nesting with each other; the rest are notes. + kept, notes = drawable_kinds(kinds, grain) + if notes: + out["totals_notes"]["+".join(kinds)] = notes + for stride in ((1, 3) if kinds == ("quarter", "year") else (1,)): + plan = build_totals_plan(kept, grain, start, n, 3, stride) + key = "+".join(kinds) + f"/s{stride}" + if plan is None: + out["totals"][key] = None + continue + out["totals"][key] = { + "kinds": list(plan.kinds), + "columns": [list(c) for c in plan.columns], + "col_of": [[list(k), v] for k, v in plan.col_of.items()], + "buckets": _iso(plan.buckets), + "depth": plan.depth, + # The tiered header is long for a wide window; the + # columns above already pin its geometry there. + "header_rows": header_rows(plan, labels) if isinstance(labels, list) and n <= 13 else None, + } + return out + + +def _answers() -> Dict[str, Any]: + golden: Dict[str, Any] = {"grains": list(T.GRAINS), "order": dict(T._ORDER), + "months_in_period": dict(T.MONTHS_IN_PERIOD), + "step_months": dict(H.STEP), "windows": {}, "starts": {}} + for grain, start, n, first_day in WINDOWS: + key = f"{grain}|{start}|{n}|{first_day.isoformat() if first_day else '-'}" + golden["windows"][key] = _window_answers(grain, start, n, first_day) + for grain in T.GRAINS: + golden["starts"][grain] = { + d.isoformat(): { + "window_start_label": _ask(T.window_start_label, d, grain), + "stub_first_day": _ask(T.stub_first_day, d, grain), + "containing_label": _ask(T._containing_label, d, grain), + } + for d in STARTS + } + golden["starts"][grain]["labels"] = { + lab: _ask(T._parse, lab, grain) + for lab in ("2026-01-05", "2026-01", "2026-Q1", "2026", "2026-W01", "2026-H1", + "2026-01-05T13", "garbage") + } + return golden + + +def test_the_calendars_answers_are_the_recorded_ones(): + answers = _answers() + if REGEN or not GOLDEN.exists(): + GOLDEN.write_text(json.dumps(answers, indent=0, sort_keys=True, default=str) + "\n") + if REGEN: + pytest.skip("golden regenerated") + recorded = json.loads(GOLDEN.read_text()) + # Compare per window so a mismatch names the window and the question. + assert answers["grains"] == recorded["grains"] + assert answers["order"] == recorded["order"] + assert answers["months_in_period"] == recorded["months_in_period"] + assert answers["step_months"] == recorded["step_months"] + for key, got in answers["windows"].items(): + want = recorded["windows"][key] + for question in want: + assert json.loads(json.dumps(got[question], default=str)) == want[question], (key, question) + assert json.loads(json.dumps(answers["starts"], default=str)) == recorded["starts"] + assert set(answers["windows"]) == set(recorded["windows"]) diff --git a/tests/test_track_expr_list_refs.py b/tests/test_track_expr_list_refs.py index 82de539..3cecd6e 100644 --- a/tests/test_track_expr_list_refs.py +++ b/tests/test_track_expr_list_refs.py @@ -58,3 +58,76 @@ def test_all_literal_list_stays_data(): with m.s as s: s.points = mo.Variable(forecast=[1.0, None, 3.0], display_name="Points") assert m.s.points.track_expr("forecast") is None + + +class TestATrackedRowReadByPosition: + """``row[-1]`` / ``row[0]`` of a plan/actual row is that period of every + track, one live cell per track (before, ``[-1]`` raised "Index -1 on + a scalar Variable … use var.value", and ``[0]`` copied the whole row as + numbers).""" + + @staticmethod + def _model(): + m = mo.Model("m", display_name="M", tracks=mo.Tracks("plan", "actual"), + default_grain="month", default_start="2026-01", default_periods=3) + with m: + m.inputs = mo.MultiVariable("Inputs") + with m.inputs as a: + a.opening = mo.Variable(1000.0, display_name="Opening") + m.pnl = mo.MultiVariable("P&L") + with m.pnl as p: + p.revenue = mo.Variable(plan=[100.0, 100.0, 100.0], actual=[110.0, 190.0, None], + display_name="Revenue", regrain=mo.up("sum")) + p.cash = mo.recurrence(start=a.opening, formula="{prev} + {r}", r=p.revenue, + display_name="Cash").set_regrain(mo.frozen(values="last")) + p.rent = mo.Variable(plan=1500.0, actual=[1500.0, 1500.0, None], display_name="Rent") + m.summary = mo.MultiVariable("Summary") + with m.summary as s: + s.end_cash = m.pnl.cash[-1].set_display_name("End cash") + s.first_revenue = m.pnl.revenue[0].set_display_name("First revenue") + s.end_rent = m.pnl.rent[-1].set_display_name("End rent") + return m + + def test_each_track_reads_its_own_period(self): + m = self._model() + cash = m.summary.end_cash.value + # the recurrence starts at the opening: plan 1000 → 1100 → 1200, + # actual 1000 → 1190 → 1190 (March not entered yet counts 0) + assert cash["plan"] == 1200.0 and cash["actual"] == 1190.0 + assert m.summary.first_revenue.value["plan"] == 100.0 + assert m.summary.first_revenue.value["actual"] == 110.0 + rent = m.summary.end_rent.value + # a scalar track is the same in every period; an actual not entered + # yet reads as a blank cell does in a formula: 0 + assert rent["plan"] == 1500.0 and rent["actual"] == 0 + + def test_the_book_refers_to_each_track_s_cell(self): + m = self._model() + pnl, summary = _book(m, "P&L"), _book(m, "Summary") + prow = _rows(pnl) + srow = _rows(summary) + last_col = openpyxl.utils.get_column_letter(pnl.max_column) + assert summary.cell(srow["End cash"], 2).value == f"='P&L'!{last_col}{prow['Cash']}" + assert summary.cell(srow["End cash · actual"], 2).value == ( + f"='P&L'!{last_col}{prow['Cash · actual']}") + assert summary.cell(srow["First revenue · actual"], 2).value == ( + f"='P&L'!C{prow['Revenue · actual']}") + + def test_a_period_past_the_window_is_an_index_error(self): + m = self._model() + try: + m.pnl.revenue[5] + except IndexError: + pass + else: + raise AssertionError("a period past the window must not read") + + def test_a_scalar_row_says_to_read_it_by_name(self): + m = self._model() + try: + m.inputs.opening[-1] + except IndexError as exc: + assert "read it as it is, by its name" in str(exc) + assert "var.value" not in str(exc) + else: + raise AssertionError("an index past [0] on a scalar must refuse") diff --git a/tests/test_unattached_aggregates.py b/tests/test_unattached_aggregates.py index 3a76c36..f4fae9a 100644 --- a/tests/test_unattached_aggregates.py +++ b/tests/test_unattached_aggregates.py @@ -167,7 +167,11 @@ def test_a_window_of_an_attached_row_keeps_its_range(self, tmp_path): class TestOtherListsWithNoCells: """Every list an aggregate can reduce that has no row in the book — an intermediate expression, a literal list, a slice of an unattached - list, one list among several arguments — follows the same rule.""" + list — follows the same rule. One list among several arguments is no + reduction (several series are combined period by + period, as ``a + b`` combines them): the call is one per period, the + list's element of each period written into that period's cell — the + ``mixed`` case, ``=SUM(10, B1) + B1``, ``=SUM(20, C1) + C1``, …""" @pytest.mark.parametrize("build", [ lambda m, x: mo.SUM(m.a * 2) + m.a, @@ -202,18 +206,40 @@ def test_a_standalone_aggregate_is_its_value(self, tmp_path): class TestScalarOperandsAreUnchanged: def test_an_unattached_scalar_is_inlined_inside_the_live_call(self, tmp_path): + # A scalar beside a series: one call per period (just + # as ``a + 5``), the scalar inlined into each period's call and the + # row's own cell live; the reduction of the row stays its range. m = mo.Model("M") m.a = mo.Variable([1, 2, 3], display_name="A") s = mo.Variable(5) m.u = mo.SUM(s, m.a) + m.a m.u._display_name = "U" + m.t = mo.SUM(s, mo.SUM(m.a)) + m.a + m.t._display_name = "T" out = tmp_path / "u.xlsx" warned = _emit(m, out) assert warned + assert m.u.value == [7, 9, 11] and m.t.value == [12, 13, 14] rows = _written(out) a_range = f"{rows['A'][0].coordinate}:{rows['A'][2].coordinate}" for formula, a in zip([c.value for c in rows["U"][:3]], rows["A"][:3]): - assert formula == f"=SUM(5, {a_range}) + {a.coordinate}" + assert formula == f"=SUM(5, {a.coordinate}) + {a.coordinate}" + for formula, a in zip([c.value for c in rows["T"][:3]], rows["A"][:3]): + assert formula == f"=SUM(5, SUM({a_range})) + {a.coordinate}" + + def test_one_list_among_several_arguments_is_read_period_by_period(self, tmp_path): + m = mo.Model("M") + m.a = mo.Variable([1, 2, 3], display_name="A") + x = mo.Variable([10, 20, 30]) + m.u = mo.SUM(x, m.a) + m.a + m.u._display_name = "U" + out = tmp_path / "u.xlsx" + assert _emit(m, out) + assert m.u.value == [12, 24, 36] + rows = _written(out) + for formula, a, ten in zip([c.value for c in rows["U"][:3]], rows["A"][:3], + (10, 20, 30)): + assert formula == f"=SUM({ten}, {a.coordinate}) + {a.coordinate}" class TestTrackedLines: diff --git a/tests/test_unit.py b/tests/test_unit.py index 63a54ef..cc612fe 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 """Unit propagation through arithmetic and Excel emission.""" +import warnings + import pytest from openpyxl import load_workbook @@ -56,6 +58,109 @@ def test_compound_division_cancels(self): assert str((cost / rate)._unit) == "hr" +class TestParsing: + """A unit string reads the way it is written: everything after the + first ``/`` is the denominator, and each further ``/`` divides again.""" + + def test_a_chain_of_slashes_divides_again(self): + assert mo.Unit("USD/kW/month") == mo.Unit("USD/(kW * month)") + + def test_a_chain_prints_as_written(self): + assert str(mo.Unit("USD/kW/month")) == "USD/kW/month" + assert str(mo.Unit(" USD/kW/month ")) == "USD/kW/month" + + def test_a_chain_cancels_through_arithmetic(self): + capacity = mo.Variable(20_000.0, unit="kW") + rent = mo.Variable(130.0, unit="USD/kW/month") + assert str((capacity * rent).unit) == "USD/month" + + def test_a_chain_keeps_its_spelling_in_the_workbook(self, tmp_path): + lease = mo.MultiVariable("Lease") + lease.capacity = mo.Variable(20_000.0, unit="kW", display_name="Capacity") + lease.rent = mo.Variable(130.0, unit="USD/kW/month", display_name="Rent") + lease.monthly = (lease.capacity * lease.rent).set_display_name("Monthly rent") + path = tmp_path / "lease.xlsx" + lease.to_excel(path) + labels = [row[0] for row in load_workbook(path).worksheets[0].iter_rows(values_only=True)] + assert "Rent (USD/kW/month)" in labels + assert "Monthly rent (USD/month)" in labels + + def test_a_parenthesised_quotient_in_the_denominator_inverts(self): + assert mo.Unit("USD/(kW/month)") == mo.Unit("USD * month/kW") + + def test_a_parenthesised_quotient_in_the_numerator(self): + assert mo.Unit("(USD/kW)/month") == mo.Unit("USD/kW/month") + + @pytest.mark.parametrize("text, rebuilt", [ + ("$/hr", "$/hr"), + ("$/hr * Gb", "$/(hr * Gb)"), + ("$/(hr * Gb)", "$/(hr * Gb)"), + ("$ * hr / Gb", "$ * hr/Gb"), + ("1/hr", "1/hr"), + ("USD m", "USD m"), + ("EUR/kWh", "EUR/kWh"), + ]) + def test_one_slash_reads_as_before(self, text, rebuilt): + assert str(mo.Unit(text)) == rebuilt + + @pytest.mark.parametrize("text", [ + "USD/kW/month", "USD/(kW/month)", "(USD/kW)/month", "$/(hr * Gb)", "1/hr", + ]) + def test_the_printed_unit_reads_back_as_itself(self, text): + unit = mo.Unit(text) + assert mo.Unit(str(unit)) == unit + + def test_a_unit_built_from_a_chain_reads_back_as_itself(self): + unit = mo.Unit("USD/kW/month") * mo.Unit("kW") + assert str(unit) == "USD/month" + assert mo.Unit(str(unit)) == unit + + def test_a_word_with_a_note_cancels_in_a_chain(self): + capacity = mo.Variable(20_000.0, unit="kW (AC)") + rent = mo.Variable(130.0, unit="USD/kW (AC)/month") + assert str((capacity * rent).unit) == "USD/month" + real = mo.Variable(20_000.0, unit="kW") * mo.Variable(130.0, unit="USD (real)/kW/month") + total = real + mo.Variable(1.0, unit="USD (real)/month") + assert str(total.unit) == "USD (real)/month" + assert str(mo.Unit("kW (AC)")) == "kW (AC)" + + def test_parentheses_inside_a_group_change_nothing(self): + assert mo.Unit("USD/(kW * (month))/yr") == mo.Unit("USD/(kW * month)/yr") + assert mo.Unit("USD/(kW (AC) * month)/yr") == mo.Unit("USD/(month * kW (AC))/yr") + + def test_the_convention_after_a_later_slash(self): + assert mo.Unit("USD/kW * month/year") == mo.Unit("USD/(kW * month * year)") + assert mo.Unit("USD/(kW*(month/year))") == mo.Unit("USD * year/(kW * month)") + + def test_spaces_around_the_slashes(self): + assert mo.Unit("USD / kW / month") == mo.Unit("USD/kW/month") + assert str(mo.Unit("USD / kW / month")) == "USD / kW / month" + + def test_parentheses_around_the_whole_unit_do_not_print(self): + assert str(mo.Unit("(USD/kWh)")) == "USD/kWh" + assert str(mo.Unit("($/hr)")) == "$/hr" + + def test_unbalanced_parentheses_read_back_as_themselves(self): + unit = mo.Unit("USD/(kW/month") + assert mo.Unit(str(unit)) == unit + + def test_a_unit_restored_without_its_written_text_prints(self): + restored = mo.Unit.__new__(mo.Unit) + restored.__dict__.update({"_unit_components": {"USD m": 1}, "_display_symbol": None}) + assert str(restored) == "USD m" + + def test_a_chain_adds_to_its_parenthesised_spelling(self): + a = mo.Variable(1.0, unit="USD/kW/month") + b = mo.Variable(2.0, unit="USD/(kW * month)") + assert (a + b).value == 3.0 + assert str((a + b).unit) == "USD/kW/month" + + def test_an_amount_per_month_still_names_the_month(self): + monthly = mo.Variable(20_000.0, unit="kW") * mo.Variable(130.0, unit="USD/kW/month") + with pytest.raises(ValueError, match="per 'month'"): + monthly + mo.Variable(1.0, unit="USD") + + class TestUnitReuse: """When ``*`` / ``/`` produce a unit one operand already carries, that operand's own Unit is the result (as ``+`` / ``-`` do) — equal to what @@ -105,6 +210,1176 @@ def counting(self, *args, **kwargs): assert len(built) == 1 +class TestCountedLabels: + """bps and pp label a pure number without rescaling it: 50 bps is + written 0.005. A line holding the count (50) added to a % line holding + fractions is refused - that sum is wrong under any reading. Anything + the label cannot tell apart keeps the plain arithmetic.""" + + def test_a_count_of_bps_added_to_a_rate_is_refused(self): + base = mo.Variable(0.05, unit="%") + spread = mo.Variable(50, unit="bps") + with pytest.raises(ValueError, match=r"50 bps is 0\.005"): + base + spread + with pytest.raises(ValueError, match="never rescales"): + spread - base + + def test_a_count_of_pp_added_to_a_margin_is_refused(self): + margin = mo.Variable(0.30, unit="%") + change = mo.Variable(2, unit="pp") + with pytest.raises(ValueError, match=r"2 pp is 0\.02"): + margin + change + + def test_a_count_in_one_period_is_enough(self): + base = mo.Variable([0.05, 0.05], unit="%") + spread = mo.Variable([0.005, 50], unit="bps") + with pytest.raises(ValueError): + base + spread + + def test_the_fraction_adds(self): + base = mo.Variable(0.05, unit="%") + spread = mo.Variable(0.005, unit="bps") + assert (base + spread).value == pytest.approx(0.055) + + def test_an_explicit_conversion_adds(self): + base = mo.Variable(0.05, unit="%") + spread = mo.Variable(50, unit="bps") + assert (base + spread / 10_000).value == pytest.approx(0.055) + + def test_the_excel_habit_still_computes(self): + principal = mo.Variable(1_000_000.0, unit="USD") + spread = mo.Variable(50, unit="bps") + assert (principal * spread / 10_000).value == pytest.approx(5_000.0) + + def test_points_written_as_points_add(self): + margin = mo.Variable(30, unit="%") + change = mo.Variable(2, unit="pp") + assert (margin + change).value == 32 + + def test_counts_of_the_same_label_add(self): + a = mo.Variable(50, unit="bps") + b = mo.Variable(25, unit="bps") + assert (a + b).value == 75 + assert str((a + b).unit) == "bps" + + +class TestCountedLabelsInALoop: + """The check ``+`` makes of a count added to a share waits while a + loop is open; it is made on the settled lines when the loop closes, + under the row that adds.""" + + @staticmethod + def _model(): + m = mo.Model("M", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.inp = mo.MultiVariable("Inputs") + with m.inp as i: + i.base0 = mo.Variable(0.05, unit="%") # an opening rate, a fraction + i.base = mo.Variable([0.05] * 4, unit="%") + i.drift = mo.Variable([0.0] * 4, unit="%") + i.step = mo.Variable(50, unit="bps") # a COUNT: 50 bps is 0.005 + i.step0 = mo.Variable(50, unit="bps") + i.pp = mo.Variable(2, unit="pp") # a COUNT of points + i.frac_step = mo.Variable(0.005, unit="bps") # 50 bps written right + i.pts0 = mo.Variable(30.0, unit="%") # a % written as points + i.zero0 = mo.Variable(0.0, unit="%") + return m + + def test_a_count_stepping_a_rate_inside_a_loop_is_refused_when_it_closes(self): + m = self._model() + with pytest.raises(ValueError, match=r"x\.rate: 'step' is in bps and holds a count" + r".*50 bps is 0\.005"): + with m: + m.x = mo.MultiVariable("X", loop=True) + with m.x as d: + d.rate = mo.lag(d.rate, fill=m.inp.base0) + m.inp.step + + def test_a_pp_count_on_a_fraction_inside_a_loop_is_refused(self): + m = self._model() + with pytest.raises(ValueError, match=r"x\.rate: 'pp' is in pp and holds a count" + r".*2 pp is 0\.02"): + with m: + m.x = mo.MultiVariable("X", loop=True) + with m.x as d: + d.rate = mo.lag(d.rate, fill=m.inp.base0) + m.inp.pp + + def test_a_row_read_before_its_line_is_checked_when_the_loop_closes(self): + m = self._model() + with pytest.raises(ValueError, match=r"x\.all_in: 'step' is in bps and holds a count"): + with m: + m.x = mo.MultiVariable("X", loop=True) + with m.x as d: + d.all_in = d.rate + m.inp.step + d.rate = mo.lag(d.rate, fill=m.inp.base0) + m.inp.drift + + def test_a_count_row_of_the_loop_beside_a_share_is_refused(self): + m = self._model() + with pytest.raises(ValueError, match=r"x\.all_in: .*is in bps and holds a count"): + with m: + m.x = mo.MultiVariable("X", loop=True) + with m.x as d: + d.all_in = m.inp.base + d.spread_row + d.spread_row = mo.lag(d.spread_row, fill=m.inp.step0) + m.inp.step * 0 + + def test_a_sum_inside_a_loop_rows_formula_is_checked(self): + m = self._model() + with pytest.raises(ValueError, match=r"'step' is in bps and holds a count"): + with m: + m.x = mo.MultiVariable("X", loop=True) + with m.x as d: + d.rate = (mo.lag(d.rate, fill=m.inp.base0) + m.inp.step) + m.inp.drift + + def test_the_fraction_steps_inside_a_loop(self): + m = self._model() + with m: + m.x = mo.MultiVariable("X", loop=True) + with m.x as d: + d.rate = mo.lag(d.rate, fill=m.inp.base0) + m.inp.frac_step + assert m.x.rate.value == pytest.approx([0.055, 0.06, 0.065, 0.07]) + assert str(m.x.rate.unit) == "%" + + def test_points_step_by_points_inside_a_loop(self): + m = self._model() + with m: + m.x = mo.MultiVariable("X", loop=True) + with m.x as d: + d.rate = mo.lag(d.rate, fill=m.inp.pts0) + m.inp.pp + assert m.x.rate.value == pytest.approx([32.0, 34.0, 36.0, 38.0]) + + def test_a_zero_seed_inside_a_loop_reads_as_a_fraction(self): + # As outside a loop: ``0 % + 2 pp`` is refused, a zero tells nothing + # of points, and a count on a fraction is the error the check is for. + m = self._model() + with pytest.raises(ValueError, match=r"'pp' is in pp and holds a count"): + with m: + m.x = mo.MultiVariable("X", loop=True) + with m.x as d: + d.rate = mo.lag(d.rate, fill=m.inp.zero0) + m.inp.pp + + def test_a_pick_inside_a_loop_is_checked_when_it_closes(self): + # MAX(lag(rate, fill=0.05 %), 50 bps) settles the row to counts, so + # the fraction and the counts meet in the lag: its source holds the + # counts (judged by its first period), its fill the fraction. + m = self._model() + with pytest.raises(ValueError, match=r"'rate' is in bps and holds a count, beside " + r"'base0' \(%\) in mo\.lag"): + with m: + m.x = mo.MultiVariable("X", loop=True) + with m.x as d: + d.rate = mo.MAX(mo.lag(d.rate, fill=m.inp.base0), m.inp.step) + + def test_a_recurrence_template_inside_a_loop_is_checked_when_it_closes(self): + # The row read before its line is settled when the loop closes; + # the template's + is read then, {prev} as the start. + m = self._model() + with pytest.raises(ValueError, match=r"x\.cum: recurrence\(.*\): 'steps' is in bps and " + r"holds a count, added to 'base0' \(%\)"): + with m: + m.x = mo.MultiVariable("X", loop=True) + with m.x as d: + d.cum = mo.recurrence(m.inp.base0, "{prev} + {s}", s=d.steps) + d.steps = m.inp.step + mo.Variable([0.0, 0.0, 0.0, 0.0]) + + +class TestCountedLabelsThroughFunctions: + """The check ``+`` makes of a count added to a share is made wherever + a function adds, compares or picks the lines - a total, a pick, a + lag's fill, a conditional total - and over a recurrence template.""" + + @staticmethod + def _lines(): + return dict( + share=mo.Variable([0.05, 0.06, 0.07], unit="%", display_name="Share"), + share0=mo.Variable(0.05, unit="%", display_name="Opening share"), + step=mo.Variable(50, unit="bps", display_name="Step"), + steps=mo.Variable([50, 50, 50], unit="bps", display_name="Steps"), + frac=mo.Variable(0.005, unit="bps", display_name="Frac"), + pts=mo.Variable([30.0, 32.0, 34.0], unit="%", display_name="Points"), + pp=mo.Variable(2, unit="pp", display_name="Pp"), + ) + + @pytest.mark.parametrize("where, build", [ + ("mo.SUM", lambda v: mo.SUM(v["share"], v["step"])), + ("mo.MAX", lambda v: mo.MAX(v["step"], v["share"])), + ("mo.MIN", lambda v: mo.MIN(v["share"], v["steps"])), + ("mo.IF", lambda v: mo.IF(v["share"] > 0.06, v["share"], v["step"])), + ("mo.CHOOSE", lambda v: mo.CHOOSE(1, v["share"], v["step"])), + ("mo.CEILING", lambda v: mo.CEILING(v["step"], v["share"])), + ("mo.IFS", lambda v: mo.IFS(v["share"] > 0.06, v["step"], True, v["share"])), + ("mo.lag", lambda v: mo.lag(v["share"], fill=v["step"])), + (".shift()", lambda v: v["share"].shift(1, fill_value=v["step"])), + ("mo.SUMIF", lambda v: mo.SUMIF([mo.Variable(1.0), mo.Variable(2.0)], ">0", + [v["share0"], v["step"]])), + ]) + def test_a_count_beside_a_share_is_refused_where_a_function_combines_them( + self, where, build): + v = self._lines() + with pytest.raises(ValueError, match=rf"'Steps?' is in bps and holds a count, beside " + rf"'(Share|Opening share)' \(%\) in {where}" + rf"[.(),]*, which holds a fraction"): + build(v) + + def test_a_fraction_step_or_points_pass(self): + v = self._lines() + assert mo.SUM(v["share"], v["frac"]).unit is None # not refused: two labels name neither + assert mo.MAX(v["pts"], v["pp"]).value == [30.0, 32.0, 34.0] + assert mo.lag(v["share"], fill=v["frac"]).value == pytest.approx([0.005, 0.05, 0.06]) + assert mo.IF(v["pts"] > 31, v["pts"], v["pp"]).value == [2.0, 32.0, 34.0] + + def test_a_function_over_many_lines_reads_each_line_once(self, monkeypatch): + # A line is a count, a fraction or neither on its own: the check + # reads each line once and names the first count and the first + # fraction - never a pass over every two lines. + from modeleon.core import variable_ops + pairs = [] + check = variable_ops._refuse_a_count_added_to_a_share + monkeypatch.setattr(variable_ops, "_refuse_a_count_added_to_a_share", + lambda *a, **k: pairs.append(a[:2]) or check(*a, **k)) + amounts = [mo.Variable(float(i), unit="USD") for i in range(300)] + assert mo.SUM(*amounts).value == pytest.approx(sum(range(300))) + assert pairs == [] + v = self._lines() + more = mo.Variable([60, 60, 60], unit="bps", display_name="More steps") + other = mo.Variable([0.01, 0.01, 0.01], unit="%", display_name="Other share") + with pytest.raises(ValueError, match=r"'Steps' is in bps and holds a count, " + r"beside 'Share' \(%\) in mo\.SUM"): + mo.SUM(v["pts"], v["share"], v["steps"], more, other) + assert len(pairs) == 1 + + def test_a_recurrence_template_adding_a_count_to_a_share_is_refused(self): + v = self._lines() + with pytest.raises(ValueError, match=r'recurrence\("\{prev\} \+ \{x\}"\): \'Steps\' is in ' + r'bps and holds a count, added to \'Opening share\''): + mo.recurrence(v["share0"], "{prev} + {x}", x=v["steps"]) + with pytest.raises(ValueError, match="holds a count, added to"): + mo.recurrence_sum(v["share0"], v["steps"]) + pts0 = mo.Variable(30.0, unit="%") + ramp = mo.recurrence(pts0, "{prev} + {x}", x=mo.Variable([2, 2, 2], unit="pp")) + assert str(ramp.unit) == "%" and ramp.value[0] == pytest.approx(30.0) + + +class TestUnitsThroughFunctions: + """A function that adds, compares or picks its values refuses two + different units, as + does; a comparison of two lines does too. A + number or an unlabelled line goes with any; a condition, an index or + a compared case is not a value. A recurrence carries its start's unit.""" + + def setup_method(self): + self.usd = mo.Variable(1000.0, unit="USD", display_name="Revenue") + self.usd2 = mo.Variable(400.0, unit="USD") + self.hr = mo.Variable(160.0, unit="hr", display_name="Hours") + + @pytest.mark.parametrize("fn", [mo.SUM, mo.AVERAGE, mo.MIN, mo.MAX]) + def test_two_units_are_refused(self, fn): + with pytest.raises(ValueError, match="'Revenue' and 'Hours' are in USD and hr"): + fn(self.usd, self.hr) + + @pytest.mark.parametrize("fn", [mo.SUM, mo.AVERAGE, mo.MIN, mo.MAX]) + def test_one_unit_is_kept(self, fn): + assert str(fn(self.usd, self.usd2).unit) == "USD" + + def test_a_number_goes_with_any_unit(self): + assert str(mo.SUM(self.usd, 5).unit) == "USD" + assert str(mo.MAX(self.usd, 0).unit) == "USD" + + def test_picked_values_in_two_units_are_refused(self): + with pytest.raises(ValueError, match="mo.IF takes its values in one unit"): + mo.IF(self.usd > 0, self.usd, self.hr) + with pytest.raises(ValueError, match="mo.CHOOSE takes its values in one unit"): + mo.CHOOSE(1, self.usd, self.hr) + with pytest.raises(ValueError, match="mo.IFERROR takes its values in one unit"): + mo.IFERROR(self.usd, self.hr) + assert str(mo.IF(self.usd > 0, self.usd, 0).unit) == "USD" + assert str(mo.IF(self.hr > 100, self.usd, self.usd2).unit) == "USD" + + def test_a_comparison_of_two_units_is_refused(self): + with pytest.raises(ValueError, match="Cannot compare values with incompatible units: USD and hr"): + self.usd > self.hr + with pytest.raises(ValueError, match="Cannot compare"): + mo.Variable(12.0, unit="month") == mo.Variable(12.0, unit="months") + assert (self.usd > self.usd2).value is True + assert (self.usd > 0).value is True + + def test_mixed_by_nature_functions_still_run(self): + rate = mo.Variable(0.1, unit="%") + flows = mo.Variable([100.0, 100.0], unit="USD") + assert mo.NPV(rate, flows).value == pytest.approx(173.553719, rel=1e-6) + + def test_a_recurrence_carries_its_start_unit(self): + opening = mo.Variable(100.0, unit="USD") + balance = mo.recurrence(opening, "{prev} * 1.01", periods=3) + assert str(balance.unit) == "USD" + topped = mo.recurrence_sum(opening, mo.Variable([10.0, 10.0, 10.0], unit="USD")) + assert str(topped.unit) == "USD" + assert str(mo.recurrence(opening, "{prev} * 2", periods=2, unit="EUR").unit) == "EUR" + assert mo.recurrence(100.0, "{prev} * 2", periods=2).unit is None + + def test_a_recurrence_template_refuses_two_units(self): + # The template is read for units as any formula is: {prev} in the + # start's unit beside a variable in another is refused, as + refuses + # them, and a declared unit excuses no mix. + eur = mo.Variable(100.0, unit="EUR", display_name="Opening") + usd = mo.Variable([1.0, 2.0, 3.0], unit="USD", display_name="Fee") + cap = mo.Variable(150.0, unit="USD", display_name="Cap") + with pytest.raises(ValueError, match=r'recurrence\("\{prev\} \+ \{x\}"\): Cannot ' + r'add/subtract values with incompatible units: ' + r'EUR and USD'): + mo.recurrence(eur, "{prev} + {x}", x=usd) + with pytest.raises(ValueError, match="incompatible units: EUR and USD"): + mo.recurrence_sum(eur, usd) + with pytest.raises(ValueError, match="incompatible units: EUR and USD"): + mo.recurrence(eur, "{prev} + {x}", x=usd, unit="EUR") + with pytest.raises(ValueError, match="Cannot compare values with incompatible units: " + "EUR and USD"): + mo.recurrence(eur, "IF({prev} > {cap}, {cap}, {prev} + {x})", cap=cap, x=usd) + a = mo.Variable([1.0] * 3, unit="EUR") + b = mo.Variable([1.0] * 3, unit="USD") + with pytest.raises(ValueError, match="incompatible units"): + mo.recurrence(0.0, "{prev} + {a} - {b}", a=a, b=b) + + def test_a_recurrence_template_derives_the_lines_unit(self): + usd = mo.Variable([1.0, 2.0, 3.0], unit="USD") + eur = mo.Variable(100.0, unit="EUR") + g = mo.Variable(0.1, unit="%") + assert str(mo.recurrence(0.0, "{prev} + {x}", x=usd).unit) == "USD" + assert str(mo.recurrence(eur, "{prev} * (1 + {g})", g=g, periods=3).unit) == "EUR" + assert mo.recurrence(100.0, "{prev} * 2", periods=2).unit is None + share = mo.Variable(0.5, unit="%") + assert str(mo.recurrence(share, "{prev} * (1 + {g})", g=g, periods=3).unit) == "%" + same = mo.Variable([1.0] * 3, unit="EUR") + assert str(mo.recurrence(eur, "ABS({prev} - {x})", x=same).unit) == "EUR" + + def test_a_recurrence_template_that_does_not_settle_is_refused(self): + eur = mo.Variable(100.0, unit="EUR") + usd = mo.Variable([1.0, 2.0, 3.0], unit="USD") + with pytest.raises(ValueError, match="grows a power of it, so the line's unit does " + "not settle"): + mo.recurrence(eur, "{prev} * {x}", x=usd) + with pytest.raises(ValueError, match="the template gives USD, but the start is in EUR"): + mo.recurrence(eur, "{x} * 2", x=usd) + with pytest.raises(ValueError, match="does not settle"): + mo.recurrence(0.0, "{prev} * {x}", x=usd) + # A declared unit decides what the line prints: the product is its own. + assert str(mo.recurrence(eur, "{prev} * {x}", x=usd, unit="EUR").unit) == "EUR" + + @pytest.mark.parametrize("build", ["sum", "max", "compare"]) + def test_a_loop_refuses_two_units_when_it_closes(self, build): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=3) + with pytest.raises(ValueError, match="takes its values in one unit|Cannot compare"): + with m: + m.inp = mo.MultiVariable("Inputs") + with m.inp as i: + i.opening = mo.Variable(1000.0, unit="USD") + i.flow = mo.Variable([10.0] * 3, unit="USD") + i.hours = mo.Variable([160.0] * 3, unit="hr") + m.debt = mo.MultiVariable("Debt", loop=True) + with m.debt as d: + prev = mo.lag(d.balance, fill=m.inp.opening) + if build == "sum": + d.check = mo.SUM(prev, m.inp.hours) + elif build == "max": + d.check = mo.MAX(prev, m.inp.hours) + else: + d.check = mo.IF(prev > m.inp.hours, 1.0, 0.0) + d.balance = prev + m.inp.flow + + +class TestFunctionUnitLaws: + """Each function's unit follows its own law — the one its result is + built by and a loop container derives its rows' units again by: a + power raises the unit, a root halves it, a product multiplies its + operands' units, a remainder and a multiple share one unit; a + conditional total is in the unit its totalled lines share, and + compares a criterion in its range's unit.""" + + def setup_method(self): + self.usd = mo.Variable([3.0, 4.0], unit="USD", display_name="Amount") + self.hr = mo.Variable(2.0, unit="hr", display_name="Hours") + + def test_a_power_raises_the_unit_and_a_root_halves_it(self): + square = mo.POWER(self.usd, 2) + assert str(square.unit) == "USD^2" and str(mo.POWER(self.usd, 2.0).unit) == "USD^2" + assert str(mo.SQRT(square).unit) == "USD" and str((square ** 0.5).unit) == "USD" + assert str(mo.POWER(self.usd, -1).unit) == "1/USD" + assert str(mo.SQRT(mo.Variable(9.0, unit="USD^2")).unit) == "USD" + # A fractional power of a dimension, or a power that is a line, + # has no unit to write. + assert mo.SQRT(self.usd).unit is None + assert mo.POWER(self.usd, mo.Variable(2)).unit is None + with pytest.raises(ValueError, match="USD\\^2 and USD"): + _ = square + self.usd + + def test_a_product_multiplies_the_units(self): + price = mo.Variable(5.0, unit="USD/hr") + assert str(mo.PRODUCT(price, self.hr).unit) == "USD" + assert str(mo.PRODUCT(price, self.hr, mo.Variable(0.9, unit="%")).unit) == "USD" + with pytest.raises(ValueError, match="multiplies the cells of 'Amount', a row in USD"): + mo.PRODUCT(self.usd) + # A word on a row of growth factors is a dimension: the refusal says + # how to make it the pure number it labels. + with pytest.raises(ValueError, match=r"FACTOR = mo.Unit.dimensionless\('factor'\)"): + mo.PRODUCT(mo.Variable([1.1, 1.2], unit="factor")) + factor = mo.Unit.dimensionless("factor") + assert mo.PRODUCT(mo.Variable([1.1, 1.2], unit=factor)).value == pytest.approx(1.32) + + def test_a_remainder_and_a_multiple_take_one_unit_a_rounding_keeps_it(self): + for fn in (mo.MOD, mo.CEILING, mo.FLOOR): + with pytest.raises(ValueError, match=f"mo.{fn.__name__} takes its values in one unit"): + fn(self.usd, self.hr) + assert str(mo.MOD(mo.Variable(14.0, unit="months"), 12).unit) == "months" + assert str(mo.ROUND(self.usd, mo.Variable(1, unit="#")).unit) == "USD" + + def test_an_exponential_a_logarithm_and_a_sign_carry_none(self): + for fn in (mo.EXP, mo.LN, mo.LOG10, mo.SIGN): + assert fn(self.usd).unit is None + + def test_a_conditional_total_takes_its_lines_in_one_unit(self): + a = mo.Variable(10.0, unit="USD", display_name="A") + b = mo.Variable(5.0, unit="hr", display_name="B") + calls = [ + lambda: mo.SUMIF([a, b], ">0"), + lambda: mo.SUMIF([a, a], ">0", [a, b]), + lambda: mo.AVERAGEIF([a, b], ">0"), + lambda: mo.SUMIFS([a, b], [a, a], ">0"), + lambda: mo.AVERAGEIFS([a, b], [a, a], ">0"), + lambda: mo.MAXIFS([a, b], [a, a], ">0"), + lambda: mo.MINIFS([a, b], [a, a], ">0"), + ] + for call in calls: + with pytest.raises(ValueError, match="takes its values in one unit; 'A' and 'B' are"): + call() + assert str(mo.SUMIFS(self.usd, self.usd, ">0").unit) == "USD" + # The criteria range's unit is never the result's; a count has none, + # and lines in two units each meet a typed criteria. + hours = mo.Variable([1.0, 2.0], unit="hr") + assert str(mo.SUMIF(hours, ">1", self.usd).unit) == "USD" + assert mo.COUNTIF([a, b], ">0").value == 2 and mo.COUNTIF([a, b], ">0").unit is None + + def test_a_criteria_compares_in_its_range_unit(self): + cap = mo.Variable(3.5, unit="hr", display_name="Cap") + calls = [ + lambda: mo.SUMIFS(self.usd, self.usd, (">=", cap)), + lambda: mo.SUMIFS(self.usd, self.usd, mo.CONCAT(">=", cap)), + lambda: mo.COUNTIF(self.usd, cap), + lambda: mo.COUNTIFS(self.usd, (">=", cap)), + ] + for call in calls: + with pytest.raises(ValueError, match="compares 'Amount' with the criteria 'Cap'"): + call() + floor = mo.Variable(3.5, unit="USD") + assert mo.SUMIFS(self.usd, self.usd, (">=", floor)).value == 4.0 + + @pytest.mark.parametrize("build", ["function", "operator", "pick"]) + def test_a_loop_derives_a_row_by_the_same_law(self, build): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=3) + with pytest.raises(ValueError, match="USD\\^2 and USD|takes its values in one unit"): + with m: + m.inp = mo.MultiVariable("Inputs") + with m.inp as i: + i.opening = mo.Variable(1000.0, unit="USD") + i.flow = mo.Variable([10.0] * 3, unit="USD") + i.per_usd = mo.Variable(0.001, unit="1/USD") + m.debt = mo.MultiVariable("Debt", loop=True) + with m.debt as d: + prev = mo.lag(d.balance, fill=m.inp.opening) + if build == "function": + d.check = mo.POWER(prev, 2) + prev + elif build == "operator": + d.check = prev ** 2 + prev + else: + d.check = mo.IFS(prev > 0, prev, True, m.inp.per_usd) + d.balance = prev + m.inp.flow + + def test_a_loop_keeps_what_the_law_gives(self): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=3) + with m: + m.inp = mo.MultiVariable("Inputs") + with m.inp as i: + i.opening = mo.Variable(1000.0, unit="USD") + i.flow = mo.Variable([10.0] * 3, unit="USD") + i.per_usd = mo.Variable(0.001, unit="1/USD") + m.debt = mo.MultiVariable("Debt", loop=True) + with m.debt as d: + prev = mo.lag(d.balance, fill=m.inp.opening) + d.square = mo.POWER(prev, 2) + d.scaled = d.square * m.inp.per_usd + d.root = mo.SQRT(d.square) + d.balance = prev + m.inp.flow + assert str(m.debt.square.unit) == "USD^2" + assert str(m.debt.scaled.unit) == "USD" and str(m.debt.root.unit) == "USD" + + +def _loop_model(body, *, fill_unit="USD"): + """A loop container built by ``body(d, inputs)`` over a three-month model.""" + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=3) + with m: + m.inp = mo.MultiVariable("Inputs") + with m.inp as i: + i.start = mo.Variable(1000.0, unit=fill_unit) + i.inflow = mo.Variable([10.0] * 3, unit="USD") + i.cap = mo.Variable(500.0, unit="USD") + i.per_usd = mo.Variable(0.001, unit="1/USD") + i.growth = mo.Variable(0.01, unit="%") + m.loop = mo.MultiVariable("Loop", loop=True) + with m.loop as d: + body(d, m.inp) + return m.loop + + +class TestLoopUnitsSettle: + """A loop container derives its rows' units again when it closes, + from rows that are not derived yet: ``+`` takes the other side's + unit, a product or a ratio waits for the row, a lag's fill anchors + it — a row never takes a unit from a placeholder.""" + + def test_a_shift_fill_gives_a_compounding_row_its_unit(self): + def body(d, i): + d.balance = d.balance.shift(1, fill_value=i.start) * (1 + i.growth) + assert str(_loop_model(body).balance.unit) == "USD" + + def test_a_shift_fill_in_another_unit_is_refused_when_the_loop_closes(self): + def body(d, i): + d.opening = d.closing.shift(1, fill_value=i.start) + d.closing = d.opening + i.inflow + with pytest.raises(ValueError, match="takes its values in one unit|incompatible units"): + _loop_model(body, fill_unit="EUR") + + def test_a_product_of_a_row_waits_for_its_unit(self): + def body(d, i): + d.opening = mo.lag(d.closing) + d.square = d.opening * d.opening + d.closing = d.opening + i.inflow + d.square * i.per_usd + loop = _loop_model(body) + assert str(loop.square.unit) == "USD^2" and str(loop.closing.unit) == "USD" + + def test_a_ratio_of_a_row_takes_no_unit_it_has_not_got(self): + def body(d, i): + d.opening = mo.lag(d.closing) + d.used = d.opening / i.cap + d.closing = d.opening + i.inflow * (1 + d.used) + loop = _loop_model(body) + # Nothing in the loop says what the opening is in: it is read as a + # pure number, as the same formulas read an unlabelled line outside + # a loop - the ratio is then per USD, the closing has no unit. + assert str(loop.used.unit) == "1/USD" and loop.closing.unit is None + + def anchored(d, i): + d.opening = mo.lag(d.closing, fill=i.start) + d.used = d.opening / i.cap + d.closing = d.opening + i.inflow * (1 + d.used) + loop = _loop_model(anchored) + assert loop.used.unit is None and str(loop.closing.unit) == "USD" + + def test_a_fill_gives_a_compounding_row_its_unit(self): + def body(d, i): + d.balance = mo.lag(d.balance, fill=i.start) * (1 + i.growth) + assert str(_loop_model(body).balance.unit) == "USD" + + def test_a_fill_in_another_unit_is_refused(self): + revenue = mo.Variable([1.0, 2.0], unit="USD", display_name="Revenue") + with pytest.raises(ValueError, match="mo.lag takes its values in one unit"): + mo.lag(revenue, fill=mo.Variable(5.0, unit="EUR", display_name="Opening")) + assert str(mo.lag(revenue, fill=5.0).unit) == "USD" + # A lag is its source's cells: an unlabelled line stays unlabelled. + assert mo.lag(mo.Variable([1.0, 2.0]), fill=mo.Variable(5.0, unit="EUR")).unit is None + + def body(d, i): + d.opening = mo.lag(d.closing, fill=i.start) + d.closing = d.opening + i.inflow + with pytest.raises(ValueError, match="takes its values in one unit|incompatible units"): + _loop_model(body, fill_unit="EUR") + + def test_a_schedule_function_on_a_loop_row_keeps_the_unit(self): + def body(d, i): + d.book = mo.lag(d.closing, fill=i.start) + d.dep = mo.SLN(d.book, i.cap, 10) + d.closing = d.book - d.dep + loop = _loop_model(body) + assert str(loop.dep.unit) == "USD" and str(loop.closing.unit) == "USD" + + +class TestLoopUnitsJudgedWhenTheLoopCloses: + """How a loop's units settle: provisional units + are never judged before the loop closes, a pure-number row leaves its + products their unit, rows settle the same way on every build, and a + unit that never settles is refused.""" + + @staticmethod + def _stock(last_row): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=3) + with m: + m.inp = mo.MultiVariable("Inputs") + with m.inp as i: + i.buy = mo.Variable([50.0, 50.0, 50.0], unit="t") + i.sell = mo.Variable([40.0, 60.0, 30.0], unit="t") + i.price = mo.Variable(250.0, unit="USD/t") + i.step = mo.Variable(1000.0, unit="USD") + i.salvage = mo.Variable(0.0, unit="USD") + i.value0 = mo.Variable(25000.0, unit="USD") + i.cap = mo.Variable(20000.0, unit="USD") + m.s = mo.MultiVariable("Stock", loop=True) + with m.s as s: + s.inv_open = mo.lag(s.inv_close) + s.value = s.inv_open * i.price # provisional while the loop is open + s.x = last_row(s, i) + s.inv_close = s.inv_open + i.buy - i.sell + return m.s + + @pytest.mark.parametrize("build", [ + lambda s, i: mo.CEILING(s.value, i.step), + lambda s, i: mo.FLOOR(s.value, i.step), + lambda s, i: mo.MOD(s.value, i.step), + lambda s, i: mo.MAX(s.value, i.cap), + lambda s, i: mo.IF(s.value > i.cap, s.value, i.cap), + lambda s, i: mo.SLN(s.value, i.salvage, 12), + lambda s, i: mo.IPMT(0.01, 1, 12, s.value, i.salvage), + lambda s, i: mo.lag(s.value, fill=i.value0), + lambda s, i: mo.COUNTIF(s.value, (">", i.cap)), + ]) + def test_a_provisional_unit_is_judged_only_when_the_loop_closes(self, build): + stock = self._stock(build) + assert str(stock.value.unit) == "USD" + + @pytest.mark.parametrize("build", [ + lambda d, cap: mo.COUNTIF(d.balance, ("<", cap)), + lambda d, cap: mo.SUMIF(d.balance, ("<", cap), d.balance), + lambda d, cap: mo.AVERAGEIF(d.balance, (">=", cap)), + lambda d, cap: mo.COUNTIFS(d.balance, ("<", cap)), + lambda d, cap: mo.SUMIFS(d.balance, d.balance, ("<", cap)), + lambda d, cap: mo.AVERAGEIFS(d.balance, d.balance, (">=", cap)), + lambda d, cap: mo.MAXIFS(d.balance, d.balance, ("<", cap)), + lambda d, cap: mo.MINIFS(d.balance, d.balance, (">=", cap)), + ]) + def test_a_criteria_joined_to_a_line_in_another_unit_is_refused_when_the_loop_closes( + self, build): + # Re-run from its formula when the loop closes, the criteria is read + # as given - an operator and a line - so its unit is compared with the + # range's, as outside a loop; the join never becomes text. + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with pytest.raises(ValueError, match="d.check: mo.[A-Z]+ compares 'balance' with the " + "criteria 'Cap'; they are in USD and hr"): + with m: + m.cap = mo.Variable(500.0, unit="hr", display_name="Cap") + m.repay = mo.Variable([200.0] * 4, unit="USD") + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + d.check = build(d, m.cap) + d.balance = mo.lag(d.balance, fill=1000.0) - m.repay + + def test_a_pure_number_row_leaves_its_products_their_unit(self): + m = mo.Model("dcf", default_grain="year", default_start="2026", default_periods=4) + with m: + m.i = mo.MultiVariable("Inputs") + m.i.rate = mo.Variable(0.08, unit="%") + m.i.cf = mo.Variable([100.0, 110.0, 120.0, 130.0], unit="USD") + m.i.hours = mo.Variable([1.0] * 4, unit="hr") + m.d = mo.MultiVariable("Discounting", loop=True) + with m.d as d: + d.df_open = mo.lag(d.df_close, fill=1.0) + d.pv = m.i.cf * d.df_open + d.cum_open = mo.lag(d.cum_close) + d.cum_close = d.cum_open + d.pv + d.df_close = d.df_open / (1 + m.i.rate) + assert str(m.d.pv.unit) == "USD" and str(m.d.cum_close.unit) == "USD" + assert m.d.df_open.unit is None + with pytest.raises(ValueError, match="incompatible units"): + _ = m.d.pv + m.i.hours + + def test_a_row_whose_unit_never_settles_is_refused(self): + def grows(d, i): + d.x = mo.lag(d.x, fill=1.0) * i.cap + with pytest.raises(ValueError, match="its unit does not settle"): + _loop_model(grows) + + def test_an_unanchored_mix_is_still_refused(self): + def body(d, i): + d.open = mo.lag(d.close, fill=mo.Variable(1000.0)) + d.fee = d.open * i.cap + d.close = d.open + d.fee + with pytest.raises(ValueError, match="incompatible units"): + _loop_model(body) + + def test_a_cycle_through_a_ratio_settles_its_units(self): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.inp = mo.MultiVariable("Inputs") + with m.inp as i: + i.demand = mo.Variable([100.0, 110.0, 120.0, 130.0], unit="t") + i.capacity = mo.Variable(150.0, unit="t") + m.plant = mo.MultiVariable("Plant", loop=True) + with m.plant as d: + d.output = i.demand / (1 + mo.lag(d.load)) + d.load = d.output / i.capacity + assert str(m.plant.output.unit) == "t" and m.plant.load.unit is None + + def test_rows_settle_the_same_way_on_every_build(self): + x = mo.Unit.dimensionless("x") + outcomes = set() + ballast = [] + for trial in range(8): + ballast.append([object() for _ in range(trial * 37)]) # moves addresses only + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=3) + with m: + m.inp = mo.MultiVariable("Inputs") + with m.inp as i: + i.multiple0 = mo.Variable(0.8, unit=x) + i.share0 = mo.Variable(0.05, unit="%") + m.loop = mo.MultiVariable("Loop", loop=True) + with m.loop as d: + d.a = mo.lag(d.b, fill=i.multiple0) + d.b = mo.lag(d.a, fill=i.share0) + outcomes.add((repr(m.loop.a.unit), repr(m.loop.b.unit))) + assert len(outcomes) == 1, outcomes + + +def _assign(container, name, value): + setattr(container, name, value) + + +class TestLoopUnitsWhateverTheLineOrder: + """Whatever the line order, a loop's units and values do not + depend on the order its lines are written in; a row read through a lag + with a fill in a unit is in that unit; a temporary whose recompute + failed is never read.""" + + LEASE_ORDERS = [["index", "rent", "billed", "fees"], + ["rent", "billed", "index", "fees"], + ["rent", "index", "billed", "fees"]] + + @staticmethod + def _lease(order): + m = mo.Model("lease", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.i = mo.MultiVariable("Inputs") + with m.i as i: + i.cpi = mo.Variable(0.004, unit="%") + i.rent0 = mo.Variable(1000.0, unit="USD") + i.area = mo.Variable(50.0, unit="m^2") + i.cap_psm = mo.Variable(20.1, unit="USD/m^2") + i.fee = mo.Variable([5.0] * 4, unit="USD") + m.r = mo.MultiVariable("Rent", loop=True) + with m.r as r: + for name in order: + if name == "rent": + _assign(r, "rent", mo.lag(r.rent, fill=i.rent0) + * (r.index / mo.lag(r.index, fill=1.0))) + elif name == "index": + _assign(r, "index", mo.lag(r.index, fill=1.0) * (1 + i.cpi)) + elif name == "billed": + _assign(r, "billed", mo.MIN(r.rent / i.area, i.cap_psm) * i.area + r.fees) + else: + _assign(r, "fees", i.fee + 0.0) + return m.r + + @pytest.mark.parametrize("order", LEASE_ORDERS) + def test_line_order_changes_neither_units_nor_values(self, order): + r = self._lease(order) + assert str(r.rent.unit) == "USD" and str(r.billed.unit) == "USD" + assert r.billed.value == pytest.approx([1009.0, 1010.0, 1010.0, 1010.0]) + + @pytest.mark.parametrize("order", [["value", "multiple", "qty"], ["multiple", "qty", "value"], + ["value", "qty", "multiple"]]) + def test_a_fill_in_another_unit_is_refused_in_every_order(self, order): + x = mo.Unit.dimensionless("x") + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with pytest.raises(ValueError, match="mo.lag takes its values in one unit"): + with m: + m.i = mo.MultiVariable("Inputs") + with m.i as i: + i.multiple = mo.Variable([1.5, 1.4, 1.3, 1.2], unit=x) + i.volume = mo.Variable([10.0] * 4, unit="t") + i.price0 = mo.Variable(2.0, unit="USD/t") + m.d = mo.MultiVariable("D", loop=True) + with m.d as d: + for name in order: + if name == "value": + _assign(d, "value", mo.lag(d.multiple, fill=i.price0) * d.qty) + elif name == "multiple": + _assign(d, "multiple", i.multiple + 0.0) + else: + _assign(d, "qty", i.volume + 0.0) + + @pytest.mark.parametrize("own_row", [False, True]) + def test_a_growth_curve_takes_its_fill_unit(self, own_row): + m = mo.Model("saas", default_grain="month", default_start="2026-01", default_periods=6) + with m: + m.i = mo.MultiVariable("Inputs") + with m.i as i: + i.c0 = mo.Variable(100.0, unit="customers") + i.market = mo.Variable(10_000.0, unit="customers") + i.g = mo.Variable(0.30, unit="%") + m.s = mo.MultiVariable("S-curve", loop=True) + with m.s as s: + if own_row: + s.penetration = mo.lag(s.customers) / i.market + s.customers = mo.lag(s.customers, fill=i.c0) * (1 + i.g * (1 - s.penetration)) + else: + s.customers = (mo.lag(s.customers, fill=i.c0) + * (1 + i.g * (1 - mo.lag(s.customers) / i.market))) + assert str(m.s.customers.unit) == "customers" + # 130 * (1 + 0.3 * (1 - 130 / 10000)) + assert m.s.customers.value[1] == pytest.approx(168.493) + + @pytest.mark.parametrize("growth_first", [True, False]) + def test_a_compounding_line_is_in_its_fill_unit_whatever_comes_first(self, growth_first): + m = mo.Model("saas", default_grain="month", default_start="2026-01", default_periods=6) + with m: + m.i = mo.MultiVariable("Inputs") + with m.i as i: + i.mrr0 = mo.Variable(10_000.0, unit="USD") + i.churn = mo.Variable(0.02, unit="%") + i.costs_eur = mo.Variable([9_000.0] * 6, unit="EUR") + m.p = mo.MultiVariable("Plan", loop=True) + with m.p as p: + if growth_first: + p.growth = mo.lag(p.growth, fill=0.10) * 0.9 + p.mrr = mo.lag(p.mrr, fill=i.mrr0) * (1 + p.growth) - p.churned + p.churned = mo.lag(p.mrr) * i.churn + if not growth_first: + p.growth = mo.lag(p.growth, fill=0.10) * 0.9 + assert str(m.p.mrr.unit) == "USD" and str(m.p.churned.unit) == "USD" + with pytest.raises(ValueError, match="incompatible units"): + _ = m.p.mrr - m.i.costs_eur + + @pytest.mark.parametrize("service_first", [True, False]) + def test_a_row_typed_by_its_placeholder_keeps_its_unit_in_reports(self, service_first): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=4) + with m: + m.i = mo.MultiVariable("Inputs") + with m.i as i: + i.cfads = mo.Variable([130.0, 90.0, 150.0, 160.0], unit="USD") + i.debt0 = mo.Variable(1000.0, unit="USD") + i.rate = mo.Variable(0.01, unit="%") + i.fixed = mo.Variable(50.0, unit="USD") + m.d = mo.MultiVariable("Debt", loop=True) + with m.d as d: + d.opening = mo.lag(d.closing, fill=i.debt0) + if service_first: + d.service = d.opening * i.rate + i.fixed + d.cover = i.cfads / d.service + d.sweep = mo.IF(d.cover > 1.2, i.cfads - d.service, 0) + if not service_first: + d.service = d.opening * i.rate + i.fixed + d.closing = d.opening - d.sweep + assert m.d.sweep.value_type == "float" + for fn in (mo.SUM, mo.MAX): + assert str(fn(m.d.sweep).unit) == "USD" + assert str(mo.ROUND(m.d.sweep, 0).unit) == "USD" + + def test_a_temporary_that_failed_is_never_read(self): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=3) + with m: + m.inp = mo.MultiVariable("Inputs") + with m.inp as i: + i.start = mo.Variable(1000.0, unit="USD") + i.inflow = mo.Variable([10.0] * 3, unit="USD") + i.hours = mo.Variable([8.0] * 3, unit="hr", display_name="Hours") + m.loop = mo.MultiVariable("Loop", loop=True) + with m.loop as d: + d.open = mo.lag(d.close, fill=i.start) + rem = mo.MOD(d.open, i.hours) + twice = rem * 2 + d.close = d.open + i.inflow + d.twice = twice + d.copied = mo.Variable(rem) + for line in (m.loop.twice, m.loop.copied): + with pytest.raises(ValueError, match="mo.MOD takes its values in one unit"): + _ = line.value + + +class TestALoopRefusedWhenItCloses: + """A loop its unit check refuses when it closes hands out no numbers: + every row keeps the refusal - its value, the lines built on it, its + re-grain and the book say why - as a loop refused before it computes + does (test_loops.py::test_rows_reading_a_failed_loop_say_why).""" + + @staticmethod + def _refused(flat=False): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=3) + with m: + m.inp = mo.MultiVariable("Inputs") + with m.inp as i: + i.opening = mo.Variable(1000.0, unit="EUR") + i.fee = mo.Variable([1.0] * 3, unit="USD") + m.debt = mo.MultiVariable("Debt", loop=True) + with pytest.raises(ValueError, match="balance.*incompatible units: EUR and USD"): + if flat: + m.debt.interest = mo.lag(m.debt.balance) * 0.05 + m.debt.balance = (mo.lag(m.debt.balance, fill=i.opening) + - m.debt.interest + i.fee) + else: + with m.debt as d: + d.interest = mo.lag(d.balance) * 0.05 + d.balance = mo.lag(d.balance, fill=i.opening) - d.interest + i.fee + return m + + @pytest.mark.parametrize("flat", [False, True]) + def test_its_rows_hand_out_no_numbers(self, flat): + m = self._refused(flat) + assert m.debt._component_names == ["interest", "balance"] + for row in (m.debt.interest, m.debt.balance): + with pytest.raises(ValueError, match="loop could not be computed.*EUR and USD"): + _ = row.value + with pytest.raises(ValueError, match="loop could not be computed"): + row.at("quarter") + assert m.debt.to_dict()["components_data"]["balance"]["values"] is None + + def test_a_line_built_on_it_afterwards_says_why(self): + m = self._refused() + m.out = mo.MultiVariable("Out") + m.out.total = m.debt.balance * 2 + with pytest.raises(ValueError, match="loop could not be computed.*EUR and USD"): + _ = m.out.total.value + + def test_its_book_is_refused(self, tmp_path): + m = self._refused() + with pytest.raises(ValueError, match="loop could not be computed.*EUR and USD"): + m.to_excel(tmp_path / "refused.xlsx") + assert not (tmp_path / "refused.xlsx").exists() + + def test_a_unit_that_never_settles_leaves_no_numbers(self): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=3) + with m: + m.cap = mo.Variable(500.0, unit="USD") + m.loop = mo.MultiVariable("Loop", loop=True) + with pytest.raises(ValueError, match="its unit does not settle"): + with m.loop as d: + d.x = mo.lag(d.x, fill=1.0) * m.cap + with pytest.raises(ValueError, match="loop could not be computed.*does not settle"): + _ = m.loop.x.value + + def test_the_container_built_again_closes(self): + m = self._refused() + with m, warnings.catch_warnings(): + warnings.simplefilter("ignore") + m.debt = mo.MultiVariable("Debt", loop=True) + with m.debt as d: + d.interest = mo.lag(d.balance) * 0.05 + d.balance = mo.lag(d.balance, fill=m.inp.opening) - d.interest + assert m.debt.balance.value == pytest.approx([1000.0, 950.0, 902.5]) + assert str(m.debt.balance.unit) == "EUR" + + +class TestShiftUnits: + """A lag keeps its source's unit; a fill in a unit is the source's.""" + + def test_a_lag_of_an_unlabelled_line_stays_unlabelled(self): + revenue = mo.Variable([100.0, 110.0, 121.0], display_name="Revenue") + opening = mo.Variable(90.0, unit="USD") + prior = mo.lag(revenue, fill=opening) + assert prior.unit is None + growth = revenue / prior - 1 + assert growth.unit is None + assert (growth > mo.Variable(0.05, unit="%")).value is not None + + def test_shift_keeps_the_unit_and_checks_its_fill(self): + revenue = mo.Variable([1.0, 2.0], unit="USD", display_name="Revenue") + assert str(revenue.shift(1).unit) == "USD" + assert str(revenue.shift(1, fill_value=mo.Variable(3.0, unit="USD")).unit) == "USD" + with pytest.raises(ValueError, match="shift\\(\\) takes its values in one unit"): + revenue.shift(1, fill_value=mo.Variable(3.0, unit="EUR", display_name="Opening")) + + def test_shift_and_lag_label_a_pure_number_alike(self): + x = mo.Unit.dimensionless("x") + share = mo.Variable([0.1, 0.2], unit="%", display_name="Share") + for fill, label in ((mo.Variable(1.5, unit=x), None), (mo.Variable(0.5, unit="%"), "%"), + (0.5, "%")): + shifted, lagged = share.shift(1, fill_value=fill), mo.lag(share, fill=fill) + assert shifted.unit == lagged.unit + assert (None if shifted.unit is None else str(shifted.unit)) == label + + def test_a_shift_fill_is_one_value_assigned_above(self): + revenue = mo.Variable([1.0, 2.0], display_name="Revenue") + with pytest.raises(TypeError, match="SCALAR"): + revenue.shift(1, fill_value=mo.Variable([1.0, 2.0])) + with pytest.raises(TypeError, match="assigned further down"): + def body(d, i): + d.a = d.b.shift(1, fill_value=d.c) + d.b = d.a + i.inflow + d.c = d.b * 0 + _loop_model(body) + + +class TestUnitLawsKeepTheirOwnFunctions: + def test_a_python_function_has_no_unit_unless_declared(self): + # The engine cannot see inside it: a ratio of two USD lines is a + # pure number, a scaled USD series is still USD. Only the author + # knows, so the result has none until declared - and a name like a + # spreadsheet function's never lends that function's law. + capacity = mo.Variable([10.0, 12.0], unit="MW") + amounts = mo.Variable([5.0, 7.0], unit="USD") + + @mo.pyformula + def power(cap, hours): + return [c * hours for c in cap] + + @mo.pyformula + def sumif(rows): + return sum(rows) + + @mo.pyformula + def ratio(a, b): + return [x / y for x, y in zip(a, b)] + + @mo.pyformula(unit="USD") + def product(rows, k): + return [r * k for r in rows] + + assert power(capacity, 8760).unit is None + assert sumif(amounts).unit is None + assert ratio(amounts, amounts).unit is None + assert str(product(amounts, 2).unit) == "USD" + assert str((product(amounts, 2) + amounts).unit) == "USD" + + def test_a_loop_never_reruns_a_python_function_as_the_spreadsheet_one(self): + @mo.pyformula + def product(rows, k): + return [r * k for r in rows] + + m = mo.Model("m", default_grain="year", default_start="2026", default_periods=3) + with pytest.raises(ValueError, match="cannot be computed again when the loop closes"): + with m: + m.inp = mo.MultiVariable("Inputs") + with m.inp as i: + i.start = mo.Variable(10.0, unit="MW") + i.add = mo.Variable([1.0] * 3, unit="MW") + m.loop = mo.MultiVariable("Loop", loop=True) + with m.loop as d: + d.cap_open = mo.lag(d.cap_close, fill=i.start) + d.x = product(d.cap_open, 2) + d.cap_close = d.cap_open + i.add + + def test_a_row_typed_by_an_error_marker_keeps_its_unit_through_min_max(self): + actual = mo.Variable(["#N/A", 120.0, 130.0], unit="USD") + assert str(mo.MAX(actual, 0).unit) == "USD" + assert str(mo.MIN(actual, 100.0).unit) == "USD" + + def test_two_records_with_one_name_are_told_apart_by_path(self): + m = mo.Model("fx") + with m: + m.ledger = mo.MultiVariable("Ledger") + with m.ledger as led: + led.a = mo.MultiVariable("A") + led.a.amount = mo.Variable(10.0, unit="USD", display_name="Amount") + led.b = mo.MultiVariable("B") + led.b.amount = mo.Variable(8.0, unit="EUR", display_name="Amount") + with pytest.raises(ValueError, match="ledger.a.amount' and 'fx.ledger.b.amount"): + mo.SUM(m.ledger.a.amount, m.ledger.b.amount) + + +class TestFinancialUnits: + """The amounts a financial function reads together are in one unit, the + result's; a rate, a term, a period never are, and a rate or a count of + periods carries none.""" + + def setup_method(self): + self.rate = mo.Variable(0.01, unit="%") + self.term = mo.Variable(24, unit="months") + self.loan = mo.Variable(10_000.0, unit="USD", display_name="Loan") + self.balloon = mo.Variable(1_000.0, unit="EUR", display_name="Balloon") + self.flows = mo.Variable([-100.0, 60.0, 60.0], unit="USD") + + def test_the_result_is_in_the_amounts_unit(self): + assert str(mo.NPV(self.rate, self.flows).unit) == "USD" + assert str(mo.PMT(self.rate, self.term, self.loan).unit) == "USD" + assert str(mo.FV(self.rate, self.term, -100.0, self.loan).unit) == "USD" + assert str(mo.PV(self.rate, self.term, mo.Variable(-100.0, unit="USD")).unit) == "USD" + assert str(mo.IPMT(self.rate, 1, self.term, self.loan).unit) == "USD" + assert str(mo.PPMT(self.rate, 1, self.term, self.loan).unit) == "USD" + assert str(mo.CUMIPMT(self.rate, self.term, self.loan, 1, 12, 0).unit) == "USD" + cost, salvage = mo.Variable(900.0, unit="USD"), mo.Variable(100.0, unit="USD") + life = mo.Variable(8, unit="years") + for fn in (mo.SLN, mo.SYD, mo.DB, mo.DDB): + args = (cost, salvage, life) if fn is mo.SLN else (cost, salvage, life, 1) + assert str(fn(*args).unit) == "USD", fn + + def test_a_rate_and_a_count_carry_none(self): + assert mo.IRR(self.flows).unit is None + assert mo.NPER(self.rate, mo.Variable(-500.0, unit="USD"), self.loan).unit is None + assert mo.RATE(self.term, mo.Variable(-500.0, unit="USD"), self.loan).unit is None + + def test_amounts_in_two_units_are_refused(self): + calls = [ + lambda: mo.PMT(self.rate, self.term, self.loan, self.balloon), + lambda: mo.FV(self.rate, self.term, self.balloon, self.loan), + lambda: mo.NPER(self.rate, self.balloon, self.loan), + lambda: mo.IPMT(self.rate, 1, self.term, self.loan, self.balloon), + lambda: mo.SLN(self.loan, self.balloon, 5), + ] + for call in calls: + with pytest.raises(ValueError, match="takes its values in one unit; 'Loan' and" + " 'Balloon'|'Balloon' and 'Loan'"): + call() + + +class TestFinancialUnitsOfAListOfLines: + """A list of scalar lines is read line by line, as a row is read: one + unit among the flows, the result's for NPV / XNPV, none for IRR / XIRR; + two units refused; the book writes the lines' cells.""" + + @staticmethod + def _lines(): + return dict( + rate=mo.Variable(0.01, unit="%"), + loan=mo.Variable(10_000.0, unit="USD", display_name="Loan"), + balloon=mo.Variable(1_000.0, unit="EUR", display_name="Balloon"), + y1=mo.Variable(6_000.0, unit="USD", display_name="Y1"), + y2=mo.Variable(6_000.0, unit="USD", display_name="Y2"), + close=mo.Variable(mo.DATE(2025, 1, 1)), + exit=mo.Variable(mo.DATE(2027, 1, 1)), + ) + + def test_the_result_is_in_the_flows_unit(self): + v = self._lines() + assert str(mo.NPV(v["rate"], [v["y1"], v["y2"]]).unit) == "USD" + assert str(mo.NPV(0.1, [v["y1"], v["y2"]]).unit) == "USD" + assert str(mo.NPV(v["rate"], [v["y1"], 50.0]).unit) == "USD" + assert str(mo.XNPV(v["rate"], [-v["loan"], v["y1"]], + [v["close"], v["exit"]]).unit) == "USD" + assert mo.IRR([-v["loan"], v["y1"], v["y2"]]).unit is None + assert mo.XIRR([-v["loan"], v["y1"]], [v["close"], v["exit"]]).unit is None + + @pytest.mark.parametrize("call", ["NPV", "NPV typed rate", "IRR", "XNPV", "XIRR"]) + def test_flows_in_two_units_are_refused(self, call): + v = self._lines() + calls = { + "NPV": lambda: mo.NPV(v["rate"], [v["y1"], v["balloon"]]), + "NPV typed rate": lambda: mo.NPV(0.1, [v["y1"], v["balloon"]]), + "IRR": lambda: mo.IRR([-v["loan"], v["y1"], v["balloon"]]), + "XNPV": lambda: mo.XNPV(v["rate"], [v["y1"], v["balloon"]], + [v["close"], v["exit"]]), + "XIRR": lambda: mo.XIRR([-v["loan"], v["balloon"]], [v["close"], v["exit"]]), + } + name = call.split()[0] + with pytest.raises(ValueError, match=rf"mo\.{name} takes its values in one unit; " + r".* are in (USD and EUR|EUR and USD)"): + calls[call]() + + def test_the_result_no_longer_adds_to_another_unit(self): + v = self._lines() + eur = mo.Variable(5.0, unit="EUR") + with pytest.raises(ValueError, match="incompatible units: EUR and USD"): + eur + mo.NPV(v["rate"], [v["y1"], v["y2"]]) + + def test_the_book_writes_the_cells(self, tmp_path): + # The list's cells are the call's range: live in the book, and the + # lines are the call's dependencies. + m = mo.Model("deal") + with m: + m.rate = mo.Variable(0.10, display_name="Rate") + m.cost = mo.Variable(-100.0, display_name="Cost", unit="EUR") + m.y1 = mo.Variable(50.0, display_name="Y1", unit="EUR") + m.y2 = mo.Variable(60.0, display_name="Y2", unit="EUR") + m.npv = mo.NPV(m.rate, [m.y1, m.y2]).set_display_name("NPV") + m.irr = mo.IRR([m.cost, m.y1, m.y2]).set_display_name("IRR") + assert [v.python_name for v in m.npv._dependency_refs] == ["rate", "y1", "y2"] + m.to_excel(tmp_path / "npv.xlsx") + ws = load_workbook(tmp_path / "npv.xlsx").active + cells = {ws.cell(r, 1).value: ws.cell(r, 2).value for r in range(1, ws.max_row + 1)} + assert cells["NPV (EUR)"] == "=NPV(B1, B3:B4)" + assert cells["IRR"] == "=IRR(B2:B4, 0.1)" + + class TestAddition: def test_same_unit_keeps_unit(self): a = mo.Variable(10, unit="$") @@ -149,6 +1424,84 @@ def test_a_rate_per_another_word_names_it_without_a_year(self): _ = b + a assert "year_share" not in str(err.value) + def test_months_beside_years_write_out_the_step(self): + # Dividing by 12 does not make months years: the label stays, + # so the refusal writes the explicit step both ways. + months = mo.Variable(18.0, unit="months") + years = mo.Variable(2.0, unit="yrs") + with pytest.raises(ValueError, match="never rescales") as err: + _ = months / 12 + years + text = str(err.value) + assert "(x / 12).set_unit('yrs')" in text + assert "(x * 12).set_unit('months')" in text + assert "Otherwise convert" not in text + total = (months / 12).set_unit("yrs") + years + assert str(total._unit) == "yrs" and total.value == pytest.approx(3.5) + + def test_a_rate_per_month_beside_a_rate_per_year_writes_out_the_step(self): + monthly = mo.Variable(100.0, unit="USD/month") + yearly = mo.Variable(1200.0, unit="USD/yr") + with pytest.raises(ValueError, match="never rescales") as err: + _ = yearly + monthly + assert "(x * 12).set_unit('USD/yr')" in str(err.value) + assert "(x / 12).set_unit('USD/month')" in str(err.value) + total = yearly + (monthly * 12).set_unit("USD/yr") + assert str(total._unit) == "USD/yr" and total.value == pytest.approx(2400.0) + + def test_a_function_over_months_and_years_carries_the_step(self): + months = mo.Variable(6.0, unit="months") + years = mo.Variable(0.5, unit="yrs") + with pytest.raises(ValueError, match="mo.IF takes its values in one unit") as err: + mo.IF(months > 0, months / 12, years) + assert "(x / 12).set_unit('yrs')" in str(err.value) + + def test_a_remainder_keeps_the_unit_both_sides_share(self): + # ``%`` is MOD: a number goes with any, two units are refused. + usd = mo.Variable([25.0, 31.0], unit="USD") + assert str((usd % 12)._unit) == "USD" and (usd % 12).value == [1.0, 7.0] + assert str((100 % usd)._unit) == "USD" + assert str((usd % mo.Variable(10.0, unit="USD"))._unit) == "USD" + with pytest.raises(ValueError, match="Cannot take the remainder of values with " + "incompatible units: USD and hr"): + _ = usd % mo.Variable(2.0, unit="hr") + + def test_a_reflected_floor_division_divides_the_units(self): + usd = mo.Variable(25.0, unit="USD") + assert str((100 // usd)._unit) == str((100 / usd)._unit) + assert str((usd // 12)._unit) == "USD" + + def test_a_remainder_inside_a_loop_reads_the_same_law(self): + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=3) + with m: + m.i = mo.MultiVariable("I") + m.i.add = mo.Variable([5.0, 5.0, 5.0], unit="USD") + m.i.hours = mo.Variable(4.0, unit="hr") + m.l = mo.MultiVariable("L", loop=True) + with m.l as lp: + lp.open = mo.lag(lp.close, fill=mo.Variable(0.0, unit="USD")) + lp.rest = lp.open % 4 + lp.close = lp.open + m.i.add + assert str(m.l.rest.unit) == "USD" + m2 = mo.Model("m2", default_grain="month", default_start="2026-01", default_periods=3) + with pytest.raises(ValueError, match="remainder of values with incompatible units"): + with m2: + m2.i = mo.MultiVariable("I") + m2.i.add = mo.Variable([5.0, 5.0, 5.0], unit="USD") + m2.i.hours = mo.Variable(4.0, unit="hr") + m2.l = mo.MultiVariable("L", loop=True) + with m2.l as lp: + lp.open = mo.lag(lp.close, fill=mo.Variable(0.0, unit="USD")) + lp.rest = lp.open % m2.i.hours + lp.close = lp.open + m2.i.add + + def test_without_a_hint_the_advice_does_not_open_with_otherwise(self): + a, b = mo.Variable(1.0, unit="USD"), mo.Variable(1.0, unit="EUR") + with pytest.raises(ValueError) as err: + _ = a + b + text = str(err.value) + assert "USD and EUR. Convert one side to the other's unit" in text + assert "Otherwise" not in text + class TestPower: def test_integer_exponent_raises_unit(self): @@ -166,6 +1519,12 @@ def test_float_exponent_drops_unit(self): # Fractional/non-int exponents have no clean unit rendering assert (t ** 0.5)._unit is None + def test_a_whole_power_written_as_a_float_or_a_root_of_one_raises(self): + t = mo.Variable(3, unit="hr") + assert str((t ** 2.0)._unit) == "hr^2" + assert str(((t * t) ** 0.5)._unit) == "hr" + assert (t ** mo.Variable(2))._unit is None + class TestNegation: def test_unary_minus_preserves_unit(self): @@ -174,6 +1533,23 @@ def test_unary_minus_preserves_unit(self): class TestExcelEmission: + def test_a_format_keyed_by_a_chain_unit_follows_a_derived_line(self): + # The derived line's unit prints canonically, not as the key is + # written; the format is found by unit equality all the same. + from modeleon.compile.excel.writer import resolve_number_format + m = mo.Model("m", default_grain="month", default_start="2026-01", default_periods=2) + m.default_excel_view = mo.ExcelView(formats={"USD/kW/month": "0.000", "%": "0.0%"}) + m.capacity = mo.Variable([5.0, 5.0], unit="kW") + m.rent = mo.Variable([2.0, 2.0], unit="USD/kW/month") + m.derived = (m.capacity * m.rent) / m.capacity + m.share = mo.Variable(0.05, unit="%") + m.step = mo.Variable(0.005, unit="bps") + assert str(m.derived.unit) != "USD/kW/month" + assert resolve_number_format(m.rent) == "0.000" + assert resolve_number_format(m.derived) == "0.000" + assert resolve_number_format(m.share) == "0.0%" + assert resolve_number_format(m.step) is None # '%' never dresses bps + def test_unit_appears_in_label(self, tmp_path): m = mo.MultiVariable("UT") m.u = mo.MultiVariable("U", excel_props={'tab': True}) @@ -330,7 +1706,8 @@ def test_code_writes_the_constructor_for_a_label(self): def test_an_aggregate_over_a_dimension_and_its_namesake_names_neither(self): dim = mo.Variable(1.0, unit="factor") lab = mo.Variable(1.0, unit=FACTOR) - assert mo.SUM(dim, lab).unit is None + with pytest.raises(ValueError, match=r"factor \(a dimension\) and factor \(a labelled pure number\)"): + mo.SUM(dim, lab) # refused, as dim + lab is assert mo.SUM(lab, mo.Variable(1.0, unit=SHARE)).unit is None assert str(mo.SUM(lab, lab).unit) == "factor" @@ -354,8 +1731,11 @@ def test_a_container_in_the_units_place_carries_no_unit(self): stand_in = mo.MultiVariable("Elsewhere") assert mo.Variable(1.0, unit=stand_in).unit is None - def test_two_dimensions_written_alike_are_not_called_dimension_and_label(self): + def test_a_power_written_with_a_caret_is_that_power(self): + # A unit prints a power as hr^2, and the string reads back as it. a = mo.Variable([1.0, 2.0], unit="hr") - with pytest.raises(ValueError) as err: - (a * a) + mo.Variable([1.0, 1.0], unit="hr^2") - assert "labelled pure number" not in str(err.value) + assert str(((a * a) + mo.Variable([1.0, 1.0], unit="hr^2")).unit) == "hr^2" + assert mo.Unit("USD/m^2") * mo.Unit("m^2") == mo.Unit("USD") + assert mo.Unit("(kW * h)^2") == mo.Unit("kW") * mo.Unit("kW") * mo.Unit("h") * mo.Unit("h") + assert eval(repr(mo.Unit("hr") ** -2), {"Unit": mo.Unit}) == mo.Unit("1/hr^2") + assert str(mo.Unit("USD/m^2")) == "USD/m^2" diff --git a/tests/test_week.py b/tests/test_week.py new file mode 100644 index 0000000..dca3c69 --- /dev/null +++ b/tests/test_week.py @@ -0,0 +1,270 @@ +# SPDX-License-Identifier: Apache-2.0 +"""ISO weeks, and the ISO year that totals them. + +A week runs Monday to Sunday and is numbered in its ISO year (week 1 +holds the year's first Thursday): ``2026-W01`` starts on 29 December +2025, and 2026 has 53 weeks. A week lies inside one ISO year and in no +month, quarter or calendar year — so a weekly line rolls up to its ISO +year and is refused anywhere else, by name, never split between two +months. +""" +import warnings +from datetime import date + +import openpyxl +import pytest + +import modeleon as mo +from modeleon.compile.excel.layout import LayoutEngine +from modeleon.compile.excel.totals import TotalsWarning, build_totals_plan, header_rows +from modeleon.core.time import _period_labels + +from test_excel_time_header import _recalculated, _serial + + +def _weekly(n=6, start='2026-W51', totals=None): + kw = {'default_excel_view': mo.ExcelView(timeline={'totals': totals})} if totals else {} + m = mo.Model('m', default_grain='week', default_start=start, default_periods=n, **kw) + with m: + m.units = mo.Variable([float(i + 1) for i in range(n)], regrain=mo.up('sum'), display_name='Units') + m.level = mo.Variable([float(10 + i) for i in range(n)], regrain=mo.up('last'), display_name='Level') + return m + + +class TestTheWindow: + def test_weeks_run_monday_to_sunday_across_week_53(self): + assert _period_labels('2026-W51', None, 4, 'week', 'finance') == [ + 'W51 2026', 'W52 2026', 'W53 2026', 'W01 2027'] + m = mo.Model('m', default_grain='week', default_start='2026-W01', default_periods=2) + with m: + m.start = mo.time.start + m.end = mo.time.end + m.days = mo.time.days + m.index = mo.time.index + assert m.start.value == [date(2025, 12, 29), date(2026, 1, 5)] + assert m.end.value == [date(2026, 1, 4), date(2026, 1, 11)] + assert m.days.value == [7, 7] and m.index.value == [1, 2] + + def test_a_week_window_starts_on_a_monday(self): + m = mo.Model('m', default_grain='week', default_start=mo.DATE(2026, 1, 5), default_periods=2) + assert m.default_start == '2026-W02' + with pytest.raises(ValueError, match="A week window starts on a Monday; 2026-01-07 is a Wednesday. Start it on 2026-01-05"): + mo.Model('m', default_grain='week', default_start=mo.DATE(2026, 1, 7), default_periods=2) + + def test_a_week_has_no_months(self): + m = mo.Model('m', default_grain='week', default_start='2026-W01', default_periods=2) + with m: + with pytest.raises(ValueError, match="A week window has none"): + mo.time.months + + +class TestRollUps: + def test_weeks_roll_up_to_their_iso_year(self): + m = _weekly() + iy = m.units.at('iso_year') + assert iy.value == [6.0, 15.0] # W51–W53 2026, W01–W03 2027 + assert iy.time.start == '2026' and iy.time.grain == 'iso_year' + assert m.level.at('iso_year').value == [12.0, 15.0] + + def test_weeks_never_roll_up_to_months(self): + m = _weekly() + with pytest.raises(ValueError, match="A week does not lie inside one month: weeks straddle months"): + m.units.at('month') + with pytest.raises(ValueError, match="weeks straddle years"): + m.units.at('year') + + def test_days_roll_up_to_weeks(self): + d = mo.Model('d', default_grain='day', default_start='2026-01-01', default_periods=10) + with d: + d.x = mo.Variable([1.0] * 10, regrain=mo.up('sum')) + assert d.x.at('week').value == [4.0, 6.0] # Thu–Sun of W01, then W02's Mon–Sat + assert d.x.at('week').time.start == '2026-W01' + + def test_a_week_line_and_a_month_line_never_meet(self): + w = _weekly() + m = mo.Model('mm', default_grain='month', default_start='2026-12', default_periods=2) + with m: + m.a = mo.Variable([1.0, 2.0]) + with pytest.raises(ValueError, match="grains that do not nest: months straddle weeks"): + m.a + w.units + + +class TestTheBook: + def test_iso_year_totals_over_weeks(self, tmp_path): + m = _weekly(totals=['iso_year']) + path = tmp_path / 'w.xlsx' + m.to_excel(str(path)) + ws = openpyxl.load_workbook(str(path))['M'] + assert [c.value for c in ws[1]][:9] == [None, '2026', None, None, 'Total 2026', '2027', None, None, 'Total 2027'] + assert [c.value for c in ws[2]][:9] == [None, 'W51', 'W52', 'W53', None, 'W01', 'W02', 'W03', None] + units = next(r for r in ws.iter_rows() if r[0].value == 'Units') + assert [c.value for c in units[1:9]] == [1, 2, 3, '=SUM(B3,C3,D3)', 4, 5, 6, '=SUM(F3,G3,H3)'] + level = next(r for r in ws.iter_rows() if r[0].value == 'Level') + assert level[4].value == '=D4' and level[8].value == '=H4' + + def test_week_totals_over_days_under_the_iso_year(self): + plan = build_totals_plan(('week', 'iso_year'), 'day', '2025-12-31', 6, 2) + rows = header_rows(plan, _period_labels('2025-12-31', None, 6, 'day', 'finance')) + assert [r['cells'][0]['kind'] for r in rows] == ['iso_year', 'week', 'period'] + # Six days from Wednesday 31 Dec: W01's five, then Monday of W02 — a + # partial week the window closes. + assert [c['label'] for c in rows[1]['cells']] == ['W01', 'Total W01', 'W02', 'Total W02'] + + def test_a_weekly_header_is_a_live_chain(self, tmp_path): + m = mo.Model('m', default_grain='week', default_start='2026-W01', default_periods=3) + with m: + m.header = mo.MultiVariable('Time') + with m.header as h: + h.start = mo.time.start + h.end = mo.time.end + h.days = mo.time.days + m.default_header = m.header + m.ops = mo.MultiVariable('Ops') + with m.ops as o: + o.fee = mo.Variable(10.0, display_name='Fee') * mo.time.days + path = tmp_path / 'h.xlsx' + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + m.to_excel(str(path)) + ws = openpyxl.load_workbook(str(path))['Time'] + rows = {r[0].value: [c.value for c in r[1:4]] for r in ws.iter_rows() if r[0].value} + assert rows['Start'][1] == '=B3+1' # the previous end + 1 + assert rows['End'] == ['=B2+6', '=C2+6', '=D2+6'] + assert rows['Days'][0] == '=B3-B2+1' + + def test_a_totals_kind_a_week_cannot_draw_warns_and_is_not_drawn(self): + m = _weekly(totals=['month']) + with pytest.warns(TotalsWarning, match="'month' totals are not drawn: a week does not lie inside one month"): + eng = LayoutEngine([m]) + eng.compute_addresses() + assert eng.totals_notes and eng.totals_by_sheet == {} + + def test_weeks_and_months_never_tier_one_sheet(self): + d = mo.Model('d', default_grain='day', default_start='2026-01-01', default_periods=10, + default_excel_view=mo.ExcelView(timeline={'totals': ['week', 'month']})) + with d: + d.x = mo.Variable([1.0] * 10, regrain=mo.up('sum')) + with pytest.warns(TotalsWarning, match="'month' totals are not drawn beside 'week'"): + eng = LayoutEngine([d]) + eng.compute_addresses() + plan = next(iter(eng.totals_by_sheet.values())) + assert plan.kinds == ('week',) + + @pytest.mark.slow + def test_the_weekly_book_recalculates_to_the_engine(self, tmp_path): + m = mo.Model('m', default_grain='week', default_start='2026-W51', default_periods=5, + default_excel_view=mo.ExcelView(timeline={'totals': ['iso_year']})) + with m: + m.header = mo.MultiVariable('Time') + with m.header as h: + h.start = mo.time.start + h.end = mo.time.end + h.days = mo.time.days + m.default_header = m.header + m.ops = mo.MultiVariable('Ops') + with m.ops as o: + o.rate = mo.Variable(2.0, display_name='Rate') + o.fee = (o.rate * mo.time.days).set_display_name('Fee') + wb = _recalculated(m, tmp_path) + # W51 W52 W53 | Total 2026 | W01 W02 | Total 2027: the weeks are the + # columns whose caption row says "Wnn". + for sheet, mv, names in (('Time', m.header, ('start', 'end', 'days')), ('Ops', m.ops, ('fee',))): + ws = wb[sheet] + caption = next(r for r in ws.iter_rows(max_row=4) + if any(isinstance(c.value, str) and c.value.startswith('W') for c in r)) + weeks = [c.column for c in caption if isinstance(c.value, str) and c.value.startswith('W')] + assert len(weeks) == 5 + rows = {r[0].value: r for r in ws.iter_rows() if r[0].value is not None} + for name in names: + var = getattr(mv, name) + got = [_serial(rows[var.display_name][c - 1].value) for c in weeks] + want = [_serial(v) for v in var.value] + assert got == pytest.approx(want), (sheet, name, got, want) + + +def test_growth_by_the_week_a_year_back(): + from modeleon.compile.excel.writer import _growth_by_calendar + from modeleon.core.time import DefaultWindow + child = mo.Variable([float(i + 1) for i in range(60)]) + out = _growth_by_calendar(child, DefaultWindow('2025-W01', 'week', 60), 'week', 60, child.value) + # 2026-W01 (index 52) grows on the week holding the same date a year + # before — 2025-W01 (index 0). + assert out.value[52] == 53.0 / 1.0 + assert all(v is None for v in out.value[:52]) + + +def _iso_year_model(start): + m = mo.Model('iy', default_grain='iso_year', default_periods=3) + with m: + m.inputs = mo.MultiVariable('Inputs') + with m.inputs as i: + i.go = mo.Variable(start, display_name='Go') + m.default_start = m.inputs.go + m.header = mo.MultiVariable('Time') + with m.header as h: + h.start = mo.time.start + h.end = mo.time.end + h.days = mo.time.days + h.index = mo.time.index + m.default_header = m.header + m.ops = mo.MultiVariable('Ops') + with m.ops as o: + o.rate = mo.Variable(2.0, display_name='Rate') + o.fee = (o.rate * mo.time.days).set_display_name('Fee') + return m + + +class TestTheIsoYearBook: + """An ISO year is 52 or 53 weeks from the Monday of the week holding + 4 January: its header is spelled off the start cell — the ISO year + of a day is YEAR(day + 3), its first Monday DATE(y,1,4) - + WEEKDAY(DATE(y,1,4),3).""" + + def test_an_iso_year_header_is_spelled_off_the_start(self): + from modeleon.compile.excel.header import spell_anchor, spell_inline + assert spell_inline('start', 1, 'iso_year', 'B2') == \ + '(DATE(YEAR(B2+3)+1,1,4)-WEEKDAY(DATE(YEAR(B2+3)+1,1,4),3))' + cell = lambda f, j: None if j < 0 else f'{f[0].upper()}{j + 3}' # noqa: E731 + assert spell_anchor('start', 1, 'iso_year', 'B2', cell) == 'E3+1' + assert spell_anchor('end', 0, 'iso_year', 'B2', cell) == \ + 'DATE(YEAR(S3+3)+1,1,4)-WEEKDAY(DATE(YEAR(S3+3)+1,1,4),3)-1' + assert spell_anchor('days', 0, 'iso_year', 'B2', cell) == 'E3-S3+1' + + @pytest.mark.slow + @pytest.mark.parametrize('start, days', [ + (date(2025, 12, 29), [371, 364, 364]), # ISO 2026 has 53 weeks + (date(2026, 5, 4), [245, 364, 364]), # opened on a Monday inside it + ]) + def test_the_iso_year_book_recalculates_to_the_engine(self, tmp_path, start, days): + m = _iso_year_model(mo.DATE(start.year, start.month, start.day)) + assert m.header.days.value == days + wb = _recalculated(m, tmp_path) + rows = {r[0].value: r for r in wb['Time'].iter_rows() if r[0].value is not None} + for name in ('start', 'end', 'days', 'index'): + var = getattr(m.header, name) + got = [_serial(rows[var.display_name][c].value) for c in range(1, 4)] + assert got == pytest.approx([_serial(v) for v in var.value]), (name, got) + fee = next(r for r in wb['Ops'].iter_rows() if r[0].value == 'Fee') + assert [c.value for c in fee[2:5]] == pytest.approx(m.ops.fee.value) + + @pytest.mark.slow + def test_a_weekly_book_projected_to_iso_years_reads_its_time(self, tmp_path): + from modeleon.core.projection import project_model + m = mo.Model('wk', default_grain='week', default_start='2026-W51', default_periods=5) + with m: + m.header = mo.MultiVariable('Time') + with m.header as h: + h.start = mo.time.start + h.days = mo.time.days + m.default_header = m.header + m.ops = mo.MultiVariable('Ops') + with m.ops as o: + o.units = mo.Variable([1.0, 2.0, 3.0, 4.0, 5.0], regrain=mo.up('sum'), display_name='Units') + o.fee = (mo.Variable(2.0, display_name='Rate') * mo.time.days).set_display_name('Fee') + with warnings.catch_warnings(record=True) as seen: + warnings.simplefilter('always') + wb = _recalculated(project_model(m, 'iso_year'), tmp_path) + assert not [w for w in seen if 'translate' in str(w.message)] + ops = wb['Ops'] + assert [c.value for c in next(r for r in ops.iter_rows() if r[0].value == 'Units')[1:3]] == [6, 9] + assert [c.value for c in next(r for r in ops.iter_rows() if r[0].value == 'Fee')[1:3]] == [42, 28] From a94ac78523ed01d9195324e1e479c9ec7cd7391f Mon Sep 17 00:00:00 2001 From: Modeleon Date: Mon, 26 Oct 2026 13:41:49 +0500 Subject: [PATCH 2/2] tests: two tests pass on Windows and on Python 3.13 A parametrized case with a 32,767-character expected string gets a short id (pytest writes the id to an environment variable, which Windows caps at 32,767 characters), and a test checks an expression by its parts rather than with ==, which since Python 3.13 compares dataclass fields without the identity shortcut. --- tests/test_statistics_per_period.py | 12 ++++++++++-- tests/test_text_functions.py | 3 ++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/test_statistics_per_period.py b/tests/test_statistics_per_period.py index b7ae0b6..26002b1 100644 --- a/tests/test_statistics_per_period.py +++ b/tests/test_statistics_per_period.py @@ -245,12 +245,20 @@ def test_one_line_with_tracks_alone_stays_refused(self): def test_the_call_holds_its_operands(self): m = _model() out = mo.MEDIAN(m.x.a, m.x.b, 5.0) - assert out._expr == FuncCall("MEDIAN", [VarRef(m.x.a), VarRef(m.x.b), Literal(5.0)]) + # Compared by parts: since Python 3.13 a dataclass compares its fields + # with ==, and a Variable's == is per period, never a bool. + assert isinstance(out._expr, FuncCall) and out._expr.func == "MEDIAN" + a, b, five = out._expr.args + assert isinstance(a, VarRef) and a.var is m.x.a + assert isinstance(b, VarRef) and b.var is m.x.b + assert five == Literal(5.0) assert out.formula == "MEDIAN(a, b, 5.0)" typed = mo.MEDIAN(m.x.a, [1.0, 9.0, 1.0]) assert typed._expr.args[1] == Literal([1.0, 9.0, 1.0]) sample = mo.PERCENTILE_INC(_revs(m), 0.25) - assert sample._expr.args[0] == ListExpr([VarRef(r) for r in _revs(m)]) + items, revs = sample._expr.args[0], _revs(m) + assert isinstance(items, ListExpr) and len(items.items) == len(revs) + assert all(isinstance(i, VarRef) and i.var is r for i, r in zip(items.items, revs)) assert mo.to_json(sample._expr)["func"] == "PERCENTILE_INC" diff --git a/tests/test_text_functions.py b/tests/test_text_functions.py index bd4ab16..9fb4ec9 100644 --- a/tests/test_text_functions.py +++ b/tests/test_text_functions.py @@ -318,7 +318,8 @@ class TestRept: (None, 3, ""), ("ab", -1, "#VALUE!"), ("ab", "x", "#VALUE!"), - ("a", 32767, "a" * 32767), + pytest.param("a", 32767, "a" * 32767, id="a-32767"), # a short id: Windows caps + # an environment variable ("a", 32768, "#VALUE!"), ]) def test_repeats(self, text, n, expected):