From 92a6876378322fccb9df84f6768b0e5a1c4befdd Mon Sep 17 00:00:00 2001 From: Brendan Date: Fri, 24 Nov 2023 15:28:37 +1100 Subject: [PATCH 01/16] add redshift support --- CONTRIBUTING.md | 6 ++++++ integration_test_project/example-env.sh | 4 ++++ integration_test_project/profiles.yml | 10 ++++++++++ macros/_macros.yml | 4 ++++ .../type_helpers.sql | 20 +++++++++++++++++++ tox.ini | 18 ++++++++++++++++- 6 files changed, 61 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 35653061..0a54d90d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -78,12 +78,18 @@ Tox will take care of installing the dependencies for each environment, so you d tox -e integration_snowflake # For the Snowflake tests tox -e integration_databricks # For the Databricks tests tox -e integration_bigquery # For the BigQuery tests + tox -e integration_redshift # For the Redshift testss ``` The Spark tests require installing the [ODBC driver](https://www.databricks.com/spark/odbc-drivers-download). On a Mac, DBT_ENV_SPARK_DRIVER_PATH should be set to `/Library/simba/spark/lib/libsparkodbc_sbu.dylib`. Spark tests have not yet been added to the integration tests. +The Redshift tests require your AWS credentials configured in the current environment (either as environment variables or in your credentials +file - see [Configure the AWS cli](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html)). They are currently configured with IAM +authorisation, so your principal will require the redshift:getClusterCredentials permission to retrieve the password for the specified redshift +database user. + If you don't have access to a particular database type, this isn't a problem. Test on the one you do have, and let us know in the PR. #### SQLFluff diff --git a/integration_test_project/example-env.sh b/integration_test_project/example-env.sh index 47cb0d67..859cb0fc 100755 --- a/integration_test_project/example-env.sh +++ b/integration_test_project/example-env.sh @@ -16,6 +16,10 @@ export DBT_ENV_SECRET_DATABRICKS_TOKEN= export DBT_ENV_SECRET_GCP_PROJECT= export DBT_ENV_SPARK_DRIVER_PATH= # /Library/simba/spark/lib/libsparkodbc_sbu.dylib on a Mac export DBT_ENV_SPARK_ENDPOINT= # The endpoint ID from the Databricks HTTP path +export DBT_ENV_SECRET_REDSHIFT_HOST= +export DBT_ENV_SECRET_REDSHIFT_CLUSTER_ID= +export DBT_ENV_SECRET_REDSHIFT_DB= +export DBT_ENV_SECRET_REDSHIFT_USER= # dbt environment variables, change these export DBT_VERSION="1_5_0" diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index b24ad80d..d9f0a5d1 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -52,3 +52,13 @@ dbt_artifacts: dbname: postgres schema: public threads: 8 + redshift: + type: redshift + method: iam + threads: 8 + host: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_HOST') }}" + port: 5439 + dbname: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_DB') }}" + user: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_USER') }}" + schema: dbt_artifacts_test_commit_{{ env_var('DBT_VERSION', '') }}_{{ env_var('GITHUB_SHA_OVERRIDE', '') if env_var('GITHUB_SHA_OVERRIDE', '') else env_var('GITHUB_SHA') }} + cluster_id: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_CLUSTER_ID') }}" diff --git a/macros/_macros.yml b/macros/_macros.yml index 7b798447..baecfbbd 100644 --- a/macros/_macros.yml +++ b/macros/_macros.yml @@ -57,6 +57,10 @@ macros: description: | Dependent on the adapter type, returns the native type for storing JSON. + - name: type_string + description: | + Dependent on the adapter type, returns the native type for storing a string. + ## MIGRATION ## - name: migrate_from_v0_to_v1 description: | diff --git a/macros/database_specific_helpers/type_helpers.sql b/macros/database_specific_helpers/type_helpers.sql index 4064ad46..93959de7 100644 --- a/macros/database_specific_helpers/type_helpers.sql +++ b/macros/database_specific_helpers/type_helpers.sql @@ -26,6 +26,10 @@ json {% endmacro %} +{% macro redshift__type_json() %} + varchar(max) +{% endmacro %} + {#- ARRAY -#} {% macro type_array() %} @@ -43,3 +47,19 @@ {% macro bigquery__type_array() %} array {% endmacro %} + +{% macro redshift__type_array() %} + varchar(max) +{% endmacro %} + +{% macro type_string() %} + {{ return(adapter.dispatch('type_string', 'dbt_artifacts')()) }} +{% endmacro %} + +{% macro default__type_string() %} + {{ return(api.Column.translate_type("string")) }} +{% endmacro %} + +{% macro redshift__type_string() %} + varchar(max) +{% endmacro %} diff --git a/tox.ini b/tox.ini index 542d6e21..8cb4d600 100644 --- a/tox.ini +++ b/tox.ini @@ -35,7 +35,7 @@ rules = LT01,LT02,LT03,CP01,AL01,AL02,CP02,ST08,LT06,LT07,AM01,LT08,AL05,RF02,RF # ST08: [structure.distinct] 'DISTINCT' used with parentheses. deps = - sqlfluff-templater-dbt~=2.0.2 + sqlfluff-templater-dbt~=2.3.5 dbt-snowflake~=1.7.0 [sqlfluff:indentation] @@ -69,6 +69,9 @@ profiles_dir = integration_test_project [testenv] passenv = + AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY + AWS_SESSION_TOKEN DBT_PROFILES_DIR GITHUB_SHA_OVERRIDE GITHUB_SHA @@ -85,6 +88,10 @@ passenv = DBT_ENV_SECRET_GCP_PROJECT DBT_ENV_SPARK_DRIVER_PATH DBT_ENV_SPARK_ENDPOINT + DBT_ENV_SECRET_REDSHIFT_HOST + DBT_ENV_SECRET_REDSHIFT_CLUSTER_ID + DBT_ENV_SECRET_REDSHIFT_DB + DBT_ENV_SECRET_REDSHIFT_USER GOOGLE_APPLICATION_CREDENTIALS DBT_CLOUD_PROJECT_ID DBT_CLOUD_JOB_ID @@ -265,6 +272,15 @@ commands = dbt deps dbt build --target bigquery --vars '"my_var": "my value"' +# Redshift integration test +[testenv:integration_test_redshift] +changedir = integration_test_project +deps = dbt-redshift~=1.7.0 +commands = + dbt clean + dbt deps + dbt build --target redshift --vars '"my_var": "my value"' + # Spark integration test (disabled) [testenv:integration_spark] changedir = integration_test_project From 6ad19c4446c7777823e8eadafda722c133487aff Mon Sep 17 00:00:00 2001 From: Hans Lemm Date: Mon, 21 Oct 2024 13:35:17 +0200 Subject: [PATCH 02/16] chore(profiles): clean up --- integration_test_project/profiles.yml | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index ca284b79..dad73a43 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -52,18 +52,6 @@ dbt_artifacts: dbname: postgres schema: public threads: 8 -<<<<<<< HEAD - redshift: - type: redshift - method: iam - threads: 8 - host: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_HOST') }}" - port: 5439 - dbname: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_DB') }}" - user: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_USER') }}" - schema: dbt_artifacts_test_commit_{{ env_var('DBT_VERSION', '') }}_{{ env_var('GITHUB_SHA_OVERRIDE', '') if env_var('GITHUB_SHA_OVERRIDE', '') else env_var('GITHUB_SHA') }} - cluster_id: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_CLUSTER_ID') }}" -======= sqlserver: type: sqlserver driver: 'ODBC Driver 18 for SQL Server' @@ -75,4 +63,13 @@ dbt_artifacts: trust_cert: True user: sa password: "123" ->>>>>>> upstream/main + redshift: + type: redshift + method: iam + threads: 8 + host: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_HOST') }}" + port: 5439 + dbname: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_DB') }}" + user: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_USER') }}" + schema: dbt_artifacts_test_commit_{{ env_var('DBT_VERSION', '') }}_{{ env_var('GITHUB_SHA_OVERRIDE', '') if env_var('GITHUB_SHA_OVERRIDE', '') else env_var('GITHUB_SHA') }} + cluster_id: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_CLUSTER_ID') }}" From 2be379fff580830d261122a6532abab7206a9763 Mon Sep 17 00:00:00 2001 From: Hans Lemm Date: Mon, 21 Oct 2024 13:38:09 +0200 Subject: [PATCH 03/16] chore(type_helpers): redshift SUPER type for JSON and ARRAY --- macros/database_specific_helpers/type_helpers.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/macros/database_specific_helpers/type_helpers.sql b/macros/database_specific_helpers/type_helpers.sql index 93959de7..0f8007e5 100644 --- a/macros/database_specific_helpers/type_helpers.sql +++ b/macros/database_specific_helpers/type_helpers.sql @@ -27,13 +27,13 @@ {% endmacro %} {% macro redshift__type_json() %} - varchar(max) + super {% endmacro %} {#- ARRAY -#} {% macro type_array() %} - {{ return(adapter.dispatch('type_array', 'dbt_artifacts')()) }} + {{ return(adapter.dispatch('type_array', 'dbt_artifacts')()) }} {% endmacro %} {% macro default__type_array() %} @@ -49,11 +49,11 @@ {% endmacro %} {% macro redshift__type_array() %} - varchar(max) + super {% endmacro %} {% macro type_string() %} - {{ return(adapter.dispatch('type_string', 'dbt_artifacts')()) }} + {{ return(adapter.dispatch('type_string', 'dbt_artifacts')()) }} {% endmacro %} {% macro default__type_string() %} @@ -61,5 +61,5 @@ {% endmacro %} {% macro redshift__type_string() %} - varchar(max) + varchar(max) {% endmacro %} From 72f5e4a9ab2d0d20eb537646419508db60132a06 Mon Sep 17 00:00:00 2001 From: Hans Lemm <32077629+hanslemm@users.noreply.github.com> Date: Thu, 19 Dec 2024 18:21:12 +0100 Subject: [PATCH 04/16] Update dbt_project.yml --- dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index a02ad65a..4f62eb4f 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,7 @@ name: "dbt_artifacts" version: "2.7.0" config-version: 2 -require-dbt-version: [">=1.3.0", "<1.9.0"] +require-dbt-version: ">=1.3.0" profile: "dbt_artifacts" clean-targets: # folders to be removed by `dbt clean` From d6bd6b3115937b7670e6697ff3befaef2d6e0557 Mon Sep 17 00:00:00 2001 From: Hans Lemm Date: Thu, 12 Jun 2025 15:53:36 +0200 Subject: [PATCH 05/16] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3d60ba20..26ce21eb 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ The package currently supports - Postgres :white_check_mark: - SQL Server :white_check_mark: - Trino :white_check_mark: +- Redshift ✅ Models included: From 70d0081a19342ca07fedca2d90204456fff1dc4d Mon Sep 17 00:00:00 2001 From: Shiv Gupta <18354771+shiv-io@users.noreply.github.com> Date: Tue, 5 Aug 2025 10:52:41 -0400 Subject: [PATCH 06/16] Update type_helpers.sql --- macros/database_specific_helpers/type_helpers.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/macros/database_specific_helpers/type_helpers.sql b/macros/database_specific_helpers/type_helpers.sql index 270babb9..e9078867 100644 --- a/macros/database_specific_helpers/type_helpers.sql +++ b/macros/database_specific_helpers/type_helpers.sql @@ -62,7 +62,8 @@ {% macro redshift__type_string() %} varchar(max) - +{% endmacro %} + {% macro trino__type_array() %} array(varchar) {% endmacro %} From ab81712ef1568941b11a2e7beddbcf65f6d3aefb Mon Sep 17 00:00:00 2001 From: Hans Lemm Date: Tue, 23 Sep 2025 10:08:41 +0200 Subject: [PATCH 07/16] Update macros/database_specific_helpers/type_helpers.sql Co-authored-by: Shiv Gupta <18354771+shiv-io@users.noreply.github.com> --- macros/database_specific_helpers/type_helpers.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/database_specific_helpers/type_helpers.sql b/macros/database_specific_helpers/type_helpers.sql index 270babb9..f81c062c 100644 --- a/macros/database_specific_helpers/type_helpers.sql +++ b/macros/database_specific_helpers/type_helpers.sql @@ -62,7 +62,7 @@ {% macro redshift__type_string() %} varchar(max) - +{% endmacro %} {% macro trino__type_array() %} array(varchar) {% endmacro %} From c5f3ed1b35fc1aea19adbfcfcc72308a6c0fc2b9 Mon Sep 17 00:00:00 2001 From: Hans Lemm Date: Tue, 23 Sep 2025 10:09:39 +0200 Subject: [PATCH 08/16] Update CONTRIBUTING.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d5dfb95a..a48bad5f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -93,7 +93,7 @@ Tox will take care of installing the dependencies for each environment, so you d tox -e integration_snowflake # For the Snowflake tests tox -e integration_databricks # For the Databricks tests tox -e integration_bigquery # For the BigQuery tests - tox -e integration_redshift # For the Redshift testss + tox -e integration_redshift # For the Redshift tests ``` The Spark tests require installing the [ODBC driver](https://www.databricks.com/spark/odbc-drivers-download). On a Mac, From 4069ed74945617bc01d096bb801e1aa11dfe8056 Mon Sep 17 00:00:00 2001 From: Hans Lemm Date: Tue, 23 Sep 2025 10:12:12 +0200 Subject: [PATCH 09/16] Add Trino array type macro to type_helpers.sql --- macros/database_specific_helpers/type_helpers.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/macros/database_specific_helpers/type_helpers.sql b/macros/database_specific_helpers/type_helpers.sql index f81c062c..e553377e 100644 --- a/macros/database_specific_helpers/type_helpers.sql +++ b/macros/database_specific_helpers/type_helpers.sql @@ -63,6 +63,7 @@ {% macro redshift__type_string() %} varchar(max) {% endmacro %} + {% macro trino__type_array() %} array(varchar) {% endmacro %} From 495aecf40a7effb7413b21d7e4f42ef5549adeda Mon Sep 17 00:00:00 2001 From: "Hans C. Ravache Lemm" Date: Tue, 8 Sep 2026 23:36:38 +0200 Subject: [PATCH 10/16] chore: nest generic test arguments under arguments (dbt 1.10+) dbt-core's MissingArgumentsPropertyInGenericTestDeprecation flags the old top-level generic-test-argument style; dbt >=1.12 with require_generic_test_arguments_property: true parses only the nested `arguments:` form. 11 sites across 10 bundled fct_dbt__* models used the old style (dbt_artifacts.is_between's min_value/max_value, and one accepted_values' values). Ran dbt Labs' dbt-autofix (`deprecations --path .`) to nest them under `arguments:`; kept only the models/*.yml changes (pure re-indentation plus the nesting) and reverted the tool's unrelated dbt_project.yml edit (a `False`->`false` casing tweak and a speculative `flags:` block), since this repo's own dbt_project.yml behavior flags are a separate decision for a different change, not implied by fixing these 11 deprecation sites. Verified via `uvx --from 'dbt-core>=1.12,<1.13' --with 'dbt-postgres>=1.11,<1.12' dbt parse --target postgres --project-dir integration_test_project --profiles-dir integration_test_project --no-partial-parse --show-all-deprecations`: zero MissingArgumentsPropertyInGenericTestDeprecation remain (the one remaining ProjectFlagsMovedDeprecation is pre-existing, unrelated, and lives in integration_test_project/profiles.yml, not touched here). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016CNVbr2GyVityjqt3LPAnU --- models/fct_dbt__consumption_by_model.yml | 85 +++++++-------- models/fct_dbt__consumption_daily.yml | 57 +++++----- models/fct_dbt__consumption_daily_detail.yml | 71 ++++++------- models/fct_dbt__consumption_forecast.yml | 101 +++++++++--------- models/fct_dbt__dag_bottlenecks.yml | 72 ++++++------- models/fct_dbt__dag_bottlenecks_detail.yml | 73 ++++++------- models/fct_dbt__flaky_tests.yml | 71 ++++++------- models/fct_dbt__model_performance.yml | 103 ++++++++++--------- models/fct_dbt__run_health_daily.yml | 85 +++++++-------- models/fct_dbt__run_health_daily_detail.yml | 91 ++++++++-------- 10 files changed, 410 insertions(+), 399 deletions(-) diff --git a/models/fct_dbt__consumption_by_model.yml b/models/fct_dbt__consumption_by_model.yml index 6bfb192e..c366e9e5 100644 --- a/models/fct_dbt__consumption_by_model.yml +++ b/models/fct_dbt__consumption_by_model.yml @@ -1,46 +1,47 @@ version: 2 models: -- name: fct_dbt__consumption_by_model - description: > - Consumption attributed per model: one row per billing_month x node_id - (models only). Same SMB definition as fct_dbt__consumption_daily - (deployment-classed successful model builds on run/build/retry), so monthly - SMB totals reconcile across the two marts. Includes build cadence, total - runtime, and a dbt State ROI estimate. Primary use case: "top 20 models by - SMB burn." - columns: - - name: consumption_by_model_id - description: Surrogate key of the grain (hash of billing_month, node_id). BI join key. - tests: - - unique - - not_null - - name: billing_month - description: First day of the calendar month (UTC) the SMB were built in. - tests: - - not_null - - name: node_id - description: The model's unique node id. - tests: - - not_null - - name: name - description: The model's name. - - name: smb_quantity - description: Successful Models Built for this model in the month (deployment, run/build/retry). - - name: pct_of_month_smb - description: This model's share of the month's total SMB (0..1), a window over billing_month. - tests: - - dbt_artifacts.is_between: - min_value: 0 - max_value: 1 - - name: builds_per_day_avg - description: Average successful builds per active build-day (smb_quantity / distinct_days_built). - - name: distinct_days_built - description: Number of distinct UTC days on which the model was successfully built (deployment). - - name: total_runtime_seconds - description: Sum of total_node_runtime over the model's SMB executions in the month. - - name: estimated_monthly_datt_cost_if_reused + - name: fct_dbt__consumption_by_model description: > - dbt State ROI estimate = distinct_days_built x dbt_artifacts_datt_price - (default 0.094). A directional upper-bound of what dbt State could meter - for this model. + Consumption attributed per model: one row per billing_month x node_id + (models only). Same SMB definition as fct_dbt__consumption_daily + (deployment-classed successful model builds on run/build/retry), so monthly + SMB totals reconcile across the two marts. Includes build cadence, total + runtime, and a dbt State ROI estimate. Primary use case: "top 20 models by + SMB burn." + columns: + - name: consumption_by_model_id + description: Surrogate key of the grain (hash of billing_month, node_id). BI join key. + tests: + - unique + - not_null + - name: billing_month + description: First day of the calendar month (UTC) the SMB were built in. + tests: + - not_null + - name: node_id + description: The model's unique node id. + tests: + - not_null + - name: name + description: The model's name. + - name: smb_quantity + description: Successful Models Built for this model in the month (deployment, run/build/retry). + - name: pct_of_month_smb + description: This model's share of the month's total SMB (0..1), a window over billing_month. + tests: + - dbt_artifacts.is_between: + arguments: + min_value: 0 + max_value: 1 + - name: builds_per_day_avg + description: Average successful builds per active build-day (smb_quantity / distinct_days_built). + - name: distinct_days_built + description: Number of distinct UTC days on which the model was successfully built (deployment). + - name: total_runtime_seconds + description: Sum of total_node_runtime over the model's SMB executions in the month. + - name: estimated_monthly_datt_cost_if_reused + description: > + dbt State ROI estimate = distinct_days_built x dbt_artifacts_datt_price + (default 0.094). A directional upper-bound of what dbt State could meter + for this model. diff --git a/models/fct_dbt__consumption_daily.yml b/models/fct_dbt__consumption_daily.yml index 311d6ac7..6118bf4b 100644 --- a/models/fct_dbt__consumption_daily.yml +++ b/models/fct_dbt__consumption_daily.yml @@ -1,31 +1,32 @@ version: 2 models: -- name: fct_dbt__consumption_daily - description: > - Core consumption mart: one row per UTC day x meter. A strict roll-up of - fct_dbt__consumption_daily_detail (sum of quantity), so daily totals always - reconcile exactly to the detail grain. v1 meters: 'smb' (Successful Models - Built, mirroring dbt's published rules) and 'active_target_tables' (distinct - deployment-active nodes, the DATT upper-bound proxy). Consumption metrics - only count deployment-classed invocations (see classify_invocation_billing). - columns: - - name: consumption_daily_id - description: Surrogate key of the grain (hash of date_day, meter). Stable BI join key. - tests: - - unique - - not_null - - name: date_day - description: UTC calendar day of the executions (via cast_to_utc_date()). - tests: - - not_null - - name: meter - description: "Consumption meter: 'smb' or 'active_target_tables'." - tests: - - not_null - - accepted_values: - values: ['smb', 'active_target_tables'] - - name: quantity - description: Metered quantity for the day x meter (sum of the detail rows). - tests: - - not_null + - name: fct_dbt__consumption_daily + description: > + Core consumption mart: one row per UTC day x meter. A strict roll-up of + fct_dbt__consumption_daily_detail (sum of quantity), so daily totals always + reconcile exactly to the detail grain. v1 meters: 'smb' (Successful Models + Built, mirroring dbt's published rules) and 'active_target_tables' (distinct + deployment-active nodes, the DATT upper-bound proxy). Consumption metrics + only count deployment-classed invocations (see classify_invocation_billing). + columns: + - name: consumption_daily_id + description: Surrogate key of the grain (hash of date_day, meter). Stable BI join key. + tests: + - unique + - not_null + - name: date_day + description: UTC calendar day of the executions (via cast_to_utc_date()). + tests: + - not_null + - name: meter + description: "Consumption meter: 'smb' or 'active_target_tables'." + tests: + - not_null + - accepted_values: + arguments: + values: ['smb', 'active_target_tables'] + - name: quantity + description: Metered quantity for the day x meter (sum of the detail rows). + tests: + - not_null diff --git a/models/fct_dbt__consumption_daily_detail.yml b/models/fct_dbt__consumption_daily_detail.yml index 6ffdba87..cdbd2703 100644 --- a/models/fct_dbt__consumption_daily_detail.yml +++ b/models/fct_dbt__consumption_daily_detail.yml @@ -1,39 +1,40 @@ version: 2 models: -- name: fct_dbt__consumption_daily_detail - description: > - Detail (base) grain for consumption: UTC day x meter x materialization x - target_name. fct_dbt__consumption_daily is a strict roll-up of this model. - For 'smb' rows, materialization is the model's materialization; for - 'active_target_tables' rows materialization is null and the count is - distinct nodes per deployment target that day. Only deployment-classed - invocations are counted. - columns: - - name: consumption_daily_detail_id + - name: fct_dbt__consumption_daily_detail description: > - Surrogate key of the grain (hash of date_day, meter, materialization, - target_name). Stable BI join key. - tests: - - unique - - not_null - - name: date_day - description: UTC calendar day of the executions (via cast_to_utc_date()). - tests: - - not_null - - name: meter - description: "Consumption meter: 'smb' or 'active_target_tables'." - tests: - - not_null - - accepted_values: - values: ['smb', 'active_target_tables'] - - name: materialization - description: > - Model materialization for 'smb' rows; null for 'active_target_tables' - (a cross-resource distinct count with no single materialization). - - name: target_name - description: dbt target name (target.name) of the invocations contributing the rows. - - name: quantity - description: Metered quantity for this detail cell. - tests: - - not_null + Detail (base) grain for consumption: UTC day x meter x materialization x + target_name. fct_dbt__consumption_daily is a strict roll-up of this model. + For 'smb' rows, materialization is the model's materialization; for + 'active_target_tables' rows materialization is null and the count is + distinct nodes per deployment target that day. Only deployment-classed + invocations are counted. + columns: + - name: consumption_daily_detail_id + description: > + Surrogate key of the grain (hash of date_day, meter, materialization, + target_name). Stable BI join key. + tests: + - unique + - not_null + - name: date_day + description: UTC calendar day of the executions (via cast_to_utc_date()). + tests: + - not_null + - name: meter + description: "Consumption meter: 'smb' or 'active_target_tables'." + tests: + - not_null + - accepted_values: + arguments: + values: ['smb', 'active_target_tables'] + - name: materialization + description: > + Model materialization for 'smb' rows; null for 'active_target_tables' + (a cross-resource distinct count with no single materialization). + - name: target_name + description: dbt target name (target.name) of the invocations contributing the rows. + - name: quantity + description: Metered quantity for this detail cell. + tests: + - not_null diff --git a/models/fct_dbt__consumption_forecast.yml b/models/fct_dbt__consumption_forecast.yml index 0b0f030d..cb3881bd 100644 --- a/models/fct_dbt__consumption_forecast.yml +++ b/models/fct_dbt__consumption_forecast.yml @@ -1,54 +1,55 @@ version: 2 models: -- name: fct_dbt__consumption_forecast - description: > - Month-end consumption forecast: one row per billing_month x meter with - month-to-date usage, a weekday-aware projected month-end total, the projected - allowance-breach date, and % of allowance used. Projection averages the - trailing dbt_artifacts_run_rate_days (default 28) per ISO day-of-week - (including zero-build days) and sums the remaining calendar days' expected - values. Allowance resolves via get_smb_allowance() and applies to the 'smb' - meter only; with no plan/allowance var the breach columns are null but - projections still populate. Snowflake-only in v1 (uses Snowflake date - functions); multi-adapter support is planned. - columns: - - name: consumption_forecast_id - description: Surrogate key of the grain (hash of meter, billing_month). BI join key. - tests: - - unique - - not_null - - name: billing_month - description: First day of the calendar month (UTC). - tests: - - not_null - - name: meter - description: "Consumption meter (e.g. 'smb', 'active_target_tables')." - tests: - - not_null - - name: month_to_date_quantity - description: Sum of the meter's quantity from month start through as_of (today, clamped to month end). - tests: - - not_null - - name: allowance - description: Monthly SMB allowance (get_smb_allowance()); null for non-smb meters or when unset. - - name: daily_run_rate - description: Average daily quantity over the trailing run_rate_days window (includes zero-build days). - - name: forecast_month_end_quantity - description: month_to_date_quantity + sum of expected quantity over the remaining calendar days. - - name: forecast_exceeded_date + - name: fct_dbt__consumption_forecast description: > - The day the allowance is (or is projected to be) crossed: the actual day - the running total of elapsed daily quantity first reached the allowance if - it already has (historical), otherwise the first future day the projected - cumulative (MTD + weekday-aware expected) reaches it; null when there is no - allowance, the meter is not smb, or it never crosses. - - name: pct_of_allowance_used - description: month_to_date_quantity / allowance for the smb meter; null otherwise. - tests: - - dbt_artifacts.is_between: - min_value: 0 - - name: days_remaining - description: Calendar days remaining in the month after as_of (0 for fully-elapsed months). - - name: is_on_pace_to_exceed - description: True when forecast_month_end_quantity > allowance (smb meter); null when no allowance. + Month-end consumption forecast: one row per billing_month x meter with + month-to-date usage, a weekday-aware projected month-end total, the projected + allowance-breach date, and % of allowance used. Projection averages the + trailing dbt_artifacts_run_rate_days (default 28) per ISO day-of-week + (including zero-build days) and sums the remaining calendar days' expected + values. Allowance resolves via get_smb_allowance() and applies to the 'smb' + meter only; with no plan/allowance var the breach columns are null but + projections still populate. Snowflake-only in v1 (uses Snowflake date + functions); multi-adapter support is planned. + columns: + - name: consumption_forecast_id + description: Surrogate key of the grain (hash of meter, billing_month). BI join key. + tests: + - unique + - not_null + - name: billing_month + description: First day of the calendar month (UTC). + tests: + - not_null + - name: meter + description: "Consumption meter (e.g. 'smb', 'active_target_tables')." + tests: + - not_null + - name: month_to_date_quantity + description: Sum of the meter's quantity from month start through as_of (today, clamped to month end). + tests: + - not_null + - name: allowance + description: Monthly SMB allowance (get_smb_allowance()); null for non-smb meters or when unset. + - name: daily_run_rate + description: Average daily quantity over the trailing run_rate_days window (includes zero-build days). + - name: forecast_month_end_quantity + description: month_to_date_quantity + sum of expected quantity over the remaining calendar days. + - name: forecast_exceeded_date + description: > + The day the allowance is (or is projected to be) crossed: the actual day + the running total of elapsed daily quantity first reached the allowance if + it already has (historical), otherwise the first future day the projected + cumulative (MTD + weekday-aware expected) reaches it; null when there is no + allowance, the meter is not smb, or it never crosses. + - name: pct_of_allowance_used + description: month_to_date_quantity / allowance for the smb meter; null otherwise. + tests: + - dbt_artifacts.is_between: + arguments: + min_value: 0 + - name: days_remaining + description: Calendar days remaining in the month after as_of (0 for fully-elapsed months). + - name: is_on_pace_to_exceed + description: True when forecast_month_end_quantity > allowance (smb meter); null when no allowance. diff --git a/models/fct_dbt__dag_bottlenecks.yml b/models/fct_dbt__dag_bottlenecks.yml index 87b23279..8d03cd22 100644 --- a/models/fct_dbt__dag_bottlenecks.yml +++ b/models/fct_dbt__dag_bottlenecks.yml @@ -1,38 +1,40 @@ version: 2 models: -- name: fct_dbt__dag_bottlenecks - description: > - Pinch-point mart: per parent model x UTC day, the measured wall-clock time - its children spent waiting on it. Roll-up of fct_dbt__dag_bottlenecks_detail - over rows where the parent was the binding constraint and actually gated a - child (stall_seconds > 0). Measured gating, not graph theory; descendant - counts / blocking scores are out of scope for v1. Snowflake-only. - columns: - - name: dag_bottleneck_id - description: Surrogate key of the grain (hash of date_day, parent_node_id). BI join key. - tests: - - unique - - not_null - - name: date_day - description: UTC calendar day (via cast_to_utc_date()). - tests: - - not_null - - name: parent_node_id - description: The parent model node id that gated its children. - tests: - - not_null - - name: blocked_children - description: Distinct child models for which this parent was the binding (latest-completing) parent that day. - - name: total_stall_seconds - description: Sum of measured stall seconds attributed to this parent that day. - tests: - - dbt_artifacts.is_between: - min_value: 0 - - name: max_stall_seconds - description: Largest single measured stall attributed to this parent that day. - tests: - - dbt_artifacts.is_between: - min_value: 0 - - name: invocations_observed - description: Distinct invocations in which this parent gated at least one child that day. + - name: fct_dbt__dag_bottlenecks + description: > + Pinch-point mart: per parent model x UTC day, the measured wall-clock time + its children spent waiting on it. Roll-up of fct_dbt__dag_bottlenecks_detail + over rows where the parent was the binding constraint and actually gated a + child (stall_seconds > 0). Measured gating, not graph theory; descendant + counts / blocking scores are out of scope for v1. Snowflake-only. + columns: + - name: dag_bottleneck_id + description: Surrogate key of the grain (hash of date_day, parent_node_id). BI join key. + tests: + - unique + - not_null + - name: date_day + description: UTC calendar day (via cast_to_utc_date()). + tests: + - not_null + - name: parent_node_id + description: The parent model node id that gated its children. + tests: + - not_null + - name: blocked_children + description: Distinct child models for which this parent was the binding (latest-completing) parent that day. + - name: total_stall_seconds + description: Sum of measured stall seconds attributed to this parent that day. + tests: + - dbt_artifacts.is_between: + arguments: + min_value: 0 + - name: max_stall_seconds + description: Largest single measured stall attributed to this parent that day. + tests: + - dbt_artifacts.is_between: + arguments: + min_value: 0 + - name: invocations_observed + description: Distinct invocations in which this parent gated at least one child that day. diff --git a/models/fct_dbt__dag_bottlenecks_detail.yml b/models/fct_dbt__dag_bottlenecks_detail.yml index e545ae22..548dff32 100644 --- a/models/fct_dbt__dag_bottlenecks_detail.yml +++ b/models/fct_dbt__dag_bottlenecks_detail.yml @@ -1,39 +1,40 @@ version: 2 models: -- name: fct_dbt__dag_bottlenecks_detail - description: > - One row per (invocation, child model), with the binding parent (the - latest-completing model-parent in that invocation) and the measured stall = - child.compile_started_at - binding parent.query_completed_at, clamped to >= 0. - Drill-down behind fct_dbt__dag_bottlenecks. Model parents only (seed/source - parents excluded). Snowflake-only. - columns: - - name: dag_bottleneck_detail_id - description: Surrogate key of the grain (hash of command_invocation_id, child_node_id). BI join key. - tests: - - unique - - not_null - - name: command_invocation_id - description: The invocation the child executed in. - tests: - - not_null - - name: date_day - description: UTC calendar day of the invocation (via cast_to_utc_date()). - tests: - - not_null - - name: child_node_id - description: The child model whose start was (potentially) gated. - tests: - - not_null - - name: binding_parent_node_id - description: The model-parent with the latest query_completed_at among the child's parents in the invocation. - - name: parent_query_completed_at - description: query_completed_at of the binding parent. - - name: child_compile_started_at - description: compile_started_at of the child. - - name: stall_seconds - description: max(child_compile_started_at - parent_query_completed_at, 0) in seconds. - tests: - - dbt_artifacts.is_between: - min_value: 0 + - name: fct_dbt__dag_bottlenecks_detail + description: > + One row per (invocation, child model), with the binding parent (the + latest-completing model-parent in that invocation) and the measured stall = + child.compile_started_at - binding parent.query_completed_at, clamped to >= 0. + Drill-down behind fct_dbt__dag_bottlenecks. Model parents only (seed/source + parents excluded). Snowflake-only. + columns: + - name: dag_bottleneck_detail_id + description: Surrogate key of the grain (hash of command_invocation_id, child_node_id). BI join key. + tests: + - unique + - not_null + - name: command_invocation_id + description: The invocation the child executed in. + tests: + - not_null + - name: date_day + description: UTC calendar day of the invocation (via cast_to_utc_date()). + tests: + - not_null + - name: child_node_id + description: The child model whose start was (potentially) gated. + tests: + - not_null + - name: binding_parent_node_id + description: The model-parent with the latest query_completed_at among the child's parents in the invocation. + - name: parent_query_completed_at + description: query_completed_at of the binding parent. + - name: child_compile_started_at + description: compile_started_at of the child. + - name: stall_seconds + description: max(child_compile_started_at - parent_query_completed_at, 0) in seconds. + tests: + - dbt_artifacts.is_between: + arguments: + min_value: 0 diff --git a/models/fct_dbt__flaky_tests.yml b/models/fct_dbt__flaky_tests.yml index b85e3bbf..a1695472 100644 --- a/models/fct_dbt__flaky_tests.yml +++ b/models/fct_dbt__flaky_tests.yml @@ -1,38 +1,39 @@ version: 2 models: -- name: fct_dbt__flaky_tests - description: > - Flaky-test rollup: one row per test_node_id x month. flips are genuine - fail->pass transitions with no parent rebuild in between (from - fct_dbt__flaky_tests_detail); executions is all of the test's executions - that month. is_flaky = flips >= dbt_artifacts_flaky_min_flips (default 2). - Snowflake-only. - columns: - - name: flaky_test_id - description: Surrogate key of the grain (hash of test_node_id, month). BI join key. - tests: - - unique - - not_null - - name: test_node_id - description: The test's unique node id. - tests: - - not_null - - name: month - description: First day of the calendar month (UTC). - tests: - - not_null - - name: flips - description: Number of genuine flip events for the test that month. - - name: executions - description: Total executions of the test that month. - - name: flake_rate - description: flips / executions (0..1). - tests: - - dbt_artifacts.is_between: - min_value: 0 - max_value: 1 - - name: last_flip_at - description: run_started_at of the most recent flip's pass that month (null if no flips). - - name: is_flaky - description: True when flips >= dbt_artifacts_flaky_min_flips (default 2). + - name: fct_dbt__flaky_tests + description: > + Flaky-test rollup: one row per test_node_id x month. flips are genuine + fail->pass transitions with no parent rebuild in between (from + fct_dbt__flaky_tests_detail); executions is all of the test's executions + that month. is_flaky = flips >= dbt_artifacts_flaky_min_flips (default 2). + Snowflake-only. + columns: + - name: flaky_test_id + description: Surrogate key of the grain (hash of test_node_id, month). BI join key. + tests: + - unique + - not_null + - name: test_node_id + description: The test's unique node id. + tests: + - not_null + - name: month + description: First day of the calendar month (UTC). + tests: + - not_null + - name: flips + description: Number of genuine flip events for the test that month. + - name: executions + description: Total executions of the test that month. + - name: flake_rate + description: flips / executions (0..1). + tests: + - dbt_artifacts.is_between: + arguments: + min_value: 0 + max_value: 1 + - name: last_flip_at + description: run_started_at of the most recent flip's pass that month (null if no flips). + - name: is_flaky + description: True when flips >= dbt_artifacts_flaky_min_flips (default 2). diff --git a/models/fct_dbt__model_performance.yml b/models/fct_dbt__model_performance.yml index 5bd877f3..ffe09643 100644 --- a/models/fct_dbt__model_performance.yml +++ b/models/fct_dbt__model_performance.yml @@ -1,55 +1,56 @@ version: 2 models: -- name: fct_dbt__model_performance - description: > - Per-model daily runtime stats vs a trailing weekday-aware baseline: the - "why is the 6am job slow since Tuesday" mart. One row per UTC day x node_id - (models). Runtime stats use successful, non-full-refresh executions only. - baseline_runtime is the median of the same node's same-day-of-week - median_runtime over the trailing dbt_artifacts_run_rate_days (default 28) - days, excluding the current day. is_regressed requires ratio > - dbt_artifacts_regression_threshold (default 1.5) and a baseline of at least - dbt_artifacts_regression_min_samples (default 3) same-DOW days. - columns: - - name: model_performance_id - description: Surrogate key of the grain (hash of date_day, node_id). BI join key. - tests: - - unique - - not_null - - name: date_day - description: UTC calendar day (via cast_to_utc_date()). - tests: - - not_null - - name: node_id - description: The model's unique node id. - tests: - - not_null - - name: executions - description: Total executions of the model that day (all statuses). - - name: success_count - description: Successful executions that day. - - name: full_refresh_executions - description: Executions that were full refreshes (excluded from runtime stats and baseline). - - name: median_runtime - description: Median total_node_runtime over successful non-full-refresh executions that day. - - name: p95_runtime - description: 95th-percentile total_node_runtime over successful non-full-refresh executions that day. - - name: rows_affected_sum - description: Sum of rows_affected over successful non-full-refresh executions that day. - - name: baseline_runtime + - name: fct_dbt__model_performance description: > - Trailing weekday-aware baseline: median of the same node's same-DOW - median_runtime over the prior dbt_artifacts_run_rate_days days (excludes - today). Null until enough same-DOW history exists. - - name: baseline_sample_days - description: Number of same-DOW prior days contributing to the baseline. - - name: runtime_regression_ratio - description: median_runtime / baseline_runtime; null when baseline is null or 0. - tests: - - dbt_artifacts.is_between: - min_value: 0 - - name: is_regressed - description: > - True when runtime_regression_ratio > regression_threshold and - baseline_sample_days >= regression_min_samples; else false. + Per-model daily runtime stats vs a trailing weekday-aware baseline: the + "why is the 6am job slow since Tuesday" mart. One row per UTC day x node_id + (models). Runtime stats use successful, non-full-refresh executions only. + baseline_runtime is the median of the same node's same-day-of-week + median_runtime over the trailing dbt_artifacts_run_rate_days (default 28) + days, excluding the current day. is_regressed requires ratio > + dbt_artifacts_regression_threshold (default 1.5) and a baseline of at least + dbt_artifacts_regression_min_samples (default 3) same-DOW days. + columns: + - name: model_performance_id + description: Surrogate key of the grain (hash of date_day, node_id). BI join key. + tests: + - unique + - not_null + - name: date_day + description: UTC calendar day (via cast_to_utc_date()). + tests: + - not_null + - name: node_id + description: The model's unique node id. + tests: + - not_null + - name: executions + description: Total executions of the model that day (all statuses). + - name: success_count + description: Successful executions that day. + - name: full_refresh_executions + description: Executions that were full refreshes (excluded from runtime stats and baseline). + - name: median_runtime + description: Median total_node_runtime over successful non-full-refresh executions that day. + - name: p95_runtime + description: 95th-percentile total_node_runtime over successful non-full-refresh executions that day. + - name: rows_affected_sum + description: Sum of rows_affected over successful non-full-refresh executions that day. + - name: baseline_runtime + description: > + Trailing weekday-aware baseline: median of the same node's same-DOW + median_runtime over the prior dbt_artifacts_run_rate_days days (excludes + today). Null until enough same-DOW history exists. + - name: baseline_sample_days + description: Number of same-DOW prior days contributing to the baseline. + - name: runtime_regression_ratio + description: median_runtime / baseline_runtime; null when baseline is null or 0. + tests: + - dbt_artifacts.is_between: + arguments: + min_value: 0 + - name: is_regressed + description: > + True when runtime_regression_ratio > regression_threshold and + baseline_sample_days >= regression_min_samples; else false. diff --git a/models/fct_dbt__run_health_daily.yml b/models/fct_dbt__run_health_daily.yml index cc26d535..e4ea1771 100644 --- a/models/fct_dbt__run_health_daily.yml +++ b/models/fct_dbt__run_health_daily.yml @@ -1,45 +1,46 @@ version: 2 models: -- name: fct_dbt__run_health_daily - description: > - Run-health rollup: one row per UTC day across all invocations (no billing - classification). Node counts span models + seeds + snapshots + tests. - success = status in (success, pass); failure = (fail, failure); - error = (error); skip = (skipped, skip). success_rate = - successes / (successes + failures + errors). A failed invocation has >=1 - node in error or failure. - columns: - - name: date_day - description: UTC calendar day (via cast_to_utc_date() on invocation run_started_at). - tests: - - not_null - - unique - - name: invocations - description: Distinct command_invocation_ids that day. - - name: distinct_commands - description: Distinct dbt_command values that day. - - name: node_successes - description: Successful node executions (models + seeds + snapshots + tests). - - name: node_failures - description: Failed node executions (test fails). - - name: node_errors - description: Errored node executions. - - name: node_skips - description: Skipped node executions. - - name: failed_invocations - description: Invocations with >=1 node in error or failure. - - name: success_rate - description: successes / (successes + failures + errors); null on days with no such executions. - tests: - - dbt_artifacts.is_between: - min_value: 0 - max_value: 1 - - name: total_runtime_seconds - description: Sum of total_node_runtime across all node executions that day. - - name: max_node_runtime_seconds - description: Longest single node execution that day. - - name: first_run_started_at - description: Earliest node run_started_at that day. - - name: last_run_started_at - description: Latest node run_started_at that day. + - name: fct_dbt__run_health_daily + description: > + Run-health rollup: one row per UTC day across all invocations (no billing + classification). Node counts span models + seeds + snapshots + tests. + success = status in (success, pass); failure = (fail, failure); + error = (error); skip = (skipped, skip). success_rate = + successes / (successes + failures + errors). A failed invocation has >=1 + node in error or failure. + columns: + - name: date_day + description: UTC calendar day (via cast_to_utc_date() on invocation run_started_at). + tests: + - not_null + - unique + - name: invocations + description: Distinct command_invocation_ids that day. + - name: distinct_commands + description: Distinct dbt_command values that day. + - name: node_successes + description: Successful node executions (models + seeds + snapshots + tests). + - name: node_failures + description: Failed node executions (test fails). + - name: node_errors + description: Errored node executions. + - name: node_skips + description: Skipped node executions. + - name: failed_invocations + description: Invocations with >=1 node in error or failure. + - name: success_rate + description: successes / (successes + failures + errors); null on days with no such executions. + tests: + - dbt_artifacts.is_between: + arguments: + min_value: 0 + max_value: 1 + - name: total_runtime_seconds + description: Sum of total_node_runtime across all node executions that day. + - name: max_node_runtime_seconds + description: Longest single node execution that day. + - name: first_run_started_at + description: Earliest node run_started_at that day. + - name: last_run_started_at + description: Latest node run_started_at that day. diff --git a/models/fct_dbt__run_health_daily_detail.yml b/models/fct_dbt__run_health_daily_detail.yml index a7aa0f3c..d4126e3e 100644 --- a/models/fct_dbt__run_health_daily_detail.yml +++ b/models/fct_dbt__run_health_daily_detail.yml @@ -1,48 +1,49 @@ version: 2 models: -- name: fct_dbt__run_health_daily_detail - description: > - Run-health rollup at UTC day x target_name grain (companion to - fct_dbt__run_health_daily). Same status mapping and definitions; additive - count columns sum to the daily model across targets. - columns: - - name: run_health_daily_detail_id - description: Surrogate key of the grain (hash of date_day, target_name). BI join key. - tests: - - unique - - not_null - - name: date_day - description: UTC calendar day (via cast_to_utc_date()). - tests: - - not_null - - name: target_name - description: dbt target name (target.name) of the invocations. - - name: invocations - description: Distinct command_invocation_ids that day for the target. - - name: distinct_commands - description: Distinct dbt_command values that day for the target. - - name: node_successes - description: Successful node executions for the target. - - name: node_failures - description: Failed node executions for the target. - - name: node_errors - description: Errored node executions for the target. - - name: node_skips - description: Skipped node executions for the target. - - name: failed_invocations - description: Invocations with >=1 node in error or failure for the target. - - name: success_rate - description: successes / (successes + failures + errors) for the target. - tests: - - dbt_artifacts.is_between: - min_value: 0 - max_value: 1 - - name: total_runtime_seconds - description: Sum of total_node_runtime for the target that day. - - name: max_node_runtime_seconds - description: Longest single node execution for the target that day. - - name: first_run_started_at - description: Earliest node run_started_at for the target that day. - - name: last_run_started_at - description: Latest node run_started_at for the target that day. + - name: fct_dbt__run_health_daily_detail + description: > + Run-health rollup at UTC day x target_name grain (companion to + fct_dbt__run_health_daily). Same status mapping and definitions; additive + count columns sum to the daily model across targets. + columns: + - name: run_health_daily_detail_id + description: Surrogate key of the grain (hash of date_day, target_name). BI join key. + tests: + - unique + - not_null + - name: date_day + description: UTC calendar day (via cast_to_utc_date()). + tests: + - not_null + - name: target_name + description: dbt target name (target.name) of the invocations. + - name: invocations + description: Distinct command_invocation_ids that day for the target. + - name: distinct_commands + description: Distinct dbt_command values that day for the target. + - name: node_successes + description: Successful node executions for the target. + - name: node_failures + description: Failed node executions for the target. + - name: node_errors + description: Errored node executions for the target. + - name: node_skips + description: Skipped node executions for the target. + - name: failed_invocations + description: Invocations with >=1 node in error or failure for the target. + - name: success_rate + description: successes / (successes + failures + errors) for the target. + tests: + - dbt_artifacts.is_between: + arguments: + min_value: 0 + max_value: 1 + - name: total_runtime_seconds + description: Sum of total_node_runtime for the target that day. + - name: max_node_runtime_seconds + description: Longest single node execution for the target that day. + - name: first_run_started_at + description: Earliest node run_started_at for the target that day. + - name: last_run_started_at + description: Latest node run_started_at for the target that day. From b93ed16daa8b5a75b6386b478248ea34a4396520 Mon Sep 17 00:00:00 2001 From: "Hans C. Ravache Lemm" Date: Fri, 11 Sep 2026 13:15:38 +0200 Subject: [PATCH 11/16] feat: dim_dbt__current_relations for deferral state One row per refable node describing the relation the most recent successful execution wrote, across models, seeds and snapshots. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016CNVbr2GyVityjqt3LPAnU --- .../tests/assert_current_relations_unique.sql | 24 +++ models/dim_dbt__current_relations.sql | 163 ++++++++++++++++++ models/dim_dbt__current_relations.yml | 42 +++++ models/docs.md | 6 + 4 files changed, 235 insertions(+) create mode 100644 integration_test_project/tests/assert_current_relations_unique.sql create mode 100644 models/dim_dbt__current_relations.sql create mode 100644 models/dim_dbt__current_relations.yml diff --git a/integration_test_project/tests/assert_current_relations_unique.sql b/integration_test_project/tests/assert_current_relations_unique.sql new file mode 100644 index 00000000..ed40cfbd --- /dev/null +++ b/integration_test_project/tests/assert_current_relations_unique.sql @@ -0,0 +1,24 @@ +{{ config(enabled = target.type in ["postgres", "redshift"]) }} +-- dim_dbt__current_relations must hold at most one row per node and never a +-- null relation coordinate: a consumer patches a manifest with these values, +-- so a duplicate silently picks a winner and a null produces an unusable +-- relation name. Returns rows (fails) on either condition. +with duplicates as ( + select node_id + from {{ ref("dim_dbt__current_relations") }} + group by node_id + having count(*) > 1 +), + +nulls as ( + select node_id + from {{ ref("dim_dbt__current_relations") }} + where database is null + or schema is null + or alias is null + or resource_type is null +) + +select node_id from duplicates +union all +select node_id from nulls diff --git a/models/dim_dbt__current_relations.sql b/models/dim_dbt__current_relations.sql new file mode 100644 index 00000000..5661e03a --- /dev/null +++ b/models/dim_dbt__current_relations.sql @@ -0,0 +1,163 @@ +{# + One row per refable node describing the relation production last built + successfully. Unlike dim_dbt__current_models, which scopes to the most + recent graph and covers models only, this view answers "what does the + warehouse actually hold right now" across models, seeds and snapshots, + which is what a deferral consumer needs. +#} +with + executions as ( + + select + command_invocation_id, + node_id, + 'model' as resource_type, + run_started_at, + status, + query_completed_at, + materialization, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, -- noqa + name, + alias + from {{ ref("stg_dbt__model_executions") }} + + union all + + select + command_invocation_id, + node_id, + 'seed' as resource_type, + run_started_at, + status, + query_completed_at, + materialization, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, -- noqa + name, + alias + from {{ ref("stg_dbt__seed_executions") }} + + union all + + select + command_invocation_id, + node_id, + 'snapshot' as resource_type, + run_started_at, + status, + query_completed_at, + materialization, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, -- noqa + name, + alias + from {{ ref("stg_dbt__snapshot_executions") }} + + ), + + successes as ( + + select + *, + row_number() over ( + partition by node_id order by query_completed_at desc + ) as success_idx + from executions + where status = 'success' + + ), + + latest_success as (select * from successes where success_idx = 1), + + dimensions as ( + + select + command_invocation_id, + node_id, + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, -- noqa + package_name, + checksum, + run_started_at + from {{ ref("stg_dbt__models") }} + + union all + + select + command_invocation_id, + node_id, + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, -- noqa + package_name, + checksum, + run_started_at + from {{ ref("stg_dbt__seeds") }} + + union all + + select + command_invocation_id, + node_id, + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, -- noqa + package_name, + checksum, + run_started_at + from {{ ref("stg_dbt__snapshots") }} + + ), + + latest_graph as ( + + {# Nodes present in the most recent graph, whether or not they ran. #} + select node_id + from dimensions + where run_started_at = (select max(d.run_started_at) from dimensions as d) + + ), + + invocations as ( + + select command_invocation_id, target_name + from {{ ref("stg_dbt__invocations") }} + + ), + + final as ( + + select + latest_success.node_id, + latest_success.resource_type, + latest_success.name, + dimensions.package_name, + dimensions.database, + latest_success.schema, + latest_success.alias, + latest_success.materialization, + dimensions.checksum, + latest_success.query_completed_at as last_success_at, + latest_success.command_invocation_id, + latest_success.run_started_at, + invocations.target_name, + case + when latest_graph.node_id is not null then true else false + end as in_latest_graph + from latest_success + inner join dimensions + on latest_success.command_invocation_id = dimensions.command_invocation_id + and latest_success.node_id = dimensions.node_id + left join invocations + on latest_success.command_invocation_id = invocations.command_invocation_id + left join latest_graph on latest_success.node_id = latest_graph.node_id + + ) + +select * +from final diff --git a/models/dim_dbt__current_relations.yml b/models/dim_dbt__current_relations.yml new file mode 100644 index 00000000..68647e41 --- /dev/null +++ b/models/dim_dbt__current_relations.yml @@ -0,0 +1,42 @@ +version: 2 + +models: +- name: dim_dbt__current_relations + description: | + One row per refable node (model, seed, snapshot) describing the relation + that the most recent successful execution wrote. Intended as deferral + state for CI: see "Using dbt_artifacts as deferral state" in the README. + columns: + - name: node_id + description: '{{ doc("node_id") }}' + tests: + - unique + - not_null + - name: resource_type + description: One of model, seed or snapshot. + - name: name + description: '{{ doc("name") }}' + - name: package_name + description: '{{ doc("package_name") }}' + - name: database + description: '{{ doc("database") }}' + - name: schema + description: '{{ doc("schema") }}' + - name: alias + description: '{{ doc("alias") }}' + - name: materialization + description: '{{ doc("materialization") }}' + - name: checksum + description: '{{ doc("checksum") }}' + - name: last_success_at + description: Completion time of the most recent successful execution of this node. + - name: command_invocation_id + description: '{{ doc("command_invocation_id") }}' + - name: run_started_at + description: '{{ doc("run_started_at") }}' + - name: target_name + description: '{{ doc("target_name") }}' + - name: in_latest_graph + description: | + True when the node is present in the most recent graph. False marks a + relation that still exists in the warehouse but has left the project. diff --git a/models/docs.md b/models/docs.md index 64b420cb..dfc6947d 100644 --- a/models/docs.md +++ b/models/docs.md @@ -412,3 +412,9 @@ Response provided by the adapter as JSON. All results as a JSON blob {% enddocs %} + +{% docs last_success_at %} + +Completion time of the most recent successful execution of this node. + +{% enddocs %} From 54bb9cbdd37a50e6ef43c658184f863a07a7a772 Mon Sep 17 00:00:00 2001 From: "Hans C. Ravache Lemm" Date: Fri, 11 Sep 2026 13:28:18 +0200 Subject: [PATCH 12/16] feat: export_state run-operation for deferral state Prints dim_dbt__current_relations as a versioned JSON contract, documented in the README. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016CNVbr2GyVityjqt3LPAnU --- README.md | 43 ++++++++++++++++++++ macros/_macros.yml | 27 +++++++++++++ macros/export_state.sql | 87 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 157 insertions(+) create mode 100644 macros/export_state.sql diff --git a/README.md b/README.md index 09cf1bd8..df6235a8 100644 --- a/README.md +++ b/README.md @@ -239,6 +239,49 @@ An example operation is as follows: dbt run-operation migrate_from_v0_to_v1 --args '{old_database: analytics, old_schema: dbt_artifacts, new_database: analytics, new_schema: artifact_sources}' ``` +## Using dbt_artifacts as deferral state + +`dbt build --defer --state` reads a `manifest.json` from disk, and the only +thing it takes from that manifest for unselected nodes is where each relation +lives. `dim_dbt__current_relations` knows that from your production runs, so a +CI job can parse its own project for a structurally complete manifest and then +correct the relation coordinates from the warehouse, instead of shipping a +manifest between jobs. + +```bash +dbt --quiet run-operation dbt_artifacts.export_state \ + --args '{schema: dbt_artifacts, target_name: prod}' > export.json +``` + +The document is versioned; refuse a `dbt_artifacts_state_version` whose major +differs from the one you support. + +```json +{ + "dbt_artifacts_state_version": 1, + "generated_at": "2026-09-11T09:00:00+00:00", + "source": {"database": "analytics", "schema": "dbt_artifacts", "target_name": "prod"}, + "nodes": { + "model.my_project.dim_customer": { + "resource_type": "model", + "name": "dim_customer", + "package_name": "my_project", + "database": "analytics", + "schema": "marts", + "alias": "dim_customer", + "materialization": "table", + "checksum": "9f2c…", + "last_success_at": "2026-09-10T02:14:09+00:00", + "command_invocation_id": "0f0a…" + } + } +} +``` + +A node appears when production executed it successfully at least once; the row +shown is the most recent success. An empty `nodes` object is a valid document +and means the tables hold no successful executions yet. + ## Acknowledgements Thank you to [Tails.com](https://tails.com/gb/careers/) for initial development and maintenance of this package. On 2021/12/20, the repository was transferred from the Tails.com GitHub organization to Brooklyn Data Co. diff --git a/macros/_macros.yml b/macros/_macros.yml index 89506aea..c47a4dc0 100644 --- a/macros/_macros.yml +++ b/macros/_macros.yml @@ -289,3 +289,30 @@ macros: type: list[any] description: | The results object from dbt. + + ## DEFERRAL STATE ## + - name: export_state + description: | + Prints dim_dbt__current_relations as a versioned JSON document for use as + dbt deferral state. See "Using dbt_artifacts as deferral state" in the + README for the contract. + arguments: + - name: resource_types + type: list[string] + description: | + Which resource types to export. Defaults to model, seed and snapshot; + deferral resolves refable nodes only. + - name: database + type: string + description: | + Overrides the database of dim_dbt__current_relations. Needed when the + session runs on a target whose ref() would resolve elsewhere. + - name: schema + type: string + description: | + Overrides the schema of dim_dbt__current_relations. + - name: target_name + type: string + description: | + When set, exports only rows whose invocation ran with this target + name. Use it when several targets write to the same artifacts tables. diff --git a/macros/export_state.sql b/macros/export_state.sql new file mode 100644 index 00000000..a69f8d32 --- /dev/null +++ b/macros/export_state.sql @@ -0,0 +1,87 @@ +{# + Prints dim_dbt__current_relations as a versioned JSON document, for use as + dbt deferral state. Emits with print() so `dbt --quiet run-operation` gives + clean JSON on stdout. Never raises: an absent relation or an empty result + yields an empty `nodes` object and a warning, leaving the caller to decide. + + Usage: + dbt --quiet run-operation dbt_artifacts.export_state \ + --args '{schema: dbt_artifacts, target_name: betterdata_prod}' > export.json +#} +{% macro export_state(resource_types=['model', 'seed', 'snapshot'], database=none, schema=none, target_name=none) %} + + {% set state_version = 1 %} + {% set relation = ref("dim_dbt__current_relations") %} + {% set source_database = database if database is not none else relation.database %} + {% set source_schema = schema if schema is not none else relation.schema %} + {% set target_relation = api.Relation.create( + database=source_database, schema=source_schema, identifier=relation.identifier + ) %} + + {% set nodes = {} %} + + {% if execute %} + {# load_relation (a dbt-core built-in) checks existence without raising, + unlike selecting from the relation directly, which errors out when + it is absent. #} + {% if load_relation(target_relation) is none %} + {% do log("export_state: " ~ target_relation ~ " does not exist; exporting an empty document", info=False) %} + {% else %} + {% set query %} + select + node_id, + resource_type, + name, + package_name, + {% if target.type == "sqlserver" %} "database" {% else %} database {% endif %}, -- noqa + {% if target.type == "sqlserver" %} "schema" {% else %} schema {% endif %}, -- noqa + alias, + materialization, + checksum, + last_success_at, + command_invocation_id, + target_name + from {{ target_relation }} + where resource_type in ({{ "'" ~ resource_types | join("', '") ~ "'" }}) + {% if target_name is not none %} and target_name = '{{ target_name }}' {% endif %} + {% endset %} + + {% set results = run_query(query) %} + + {% for row in results.rows %} + {% do nodes.update({ + row["node_id"]: { + "resource_type": row["resource_type"], + "name": row["name"], + "package_name": row["package_name"], + "database": row["database"], + "schema": row["schema"], + "alias": row["alias"], + "materialization": row["materialization"], + "checksum": row["checksum"], + "last_success_at": row["last_success_at"] | string, + "command_invocation_id": row["command_invocation_id"], + } + }) %} + {% endfor %} + + {% if nodes | length == 0 %} + {% do log("export_state: no successful executions found in " ~ target_relation ~ "; exporting an empty document", info=False) %} + {% endif %} + {% endif %} + {% endif %} + + {% set document = { + "dbt_artifacts_state_version": state_version, + "generated_at": modules.datetime.datetime.now(modules.pytz.utc).isoformat(), + "source": { + "database": source_database, + "schema": source_schema, + "target_name": target_name, + }, + "nodes": nodes, + } %} + + {% do print(tojson(document)) %} + +{% endmacro %} From f1b2849d0005d5a3f3de7f396f3de0c01ae099c7 Mon Sep 17 00:00:00 2001 From: "Hans C. Ravache Lemm" Date: Fri, 11 Sep 2026 13:35:10 +0200 Subject: [PATCH 13/16] fix: validate and escape export_state's interpolated arguments resource_types is validated against the known set and target_name is escaped as a SQL string literal before either reaches the query text. database/schema reach the query indirectly via api.Relation.create(), whose quoting does not escape an embedded quote character either, so they get the same treatment: validated against a safe identifier pattern before being used to build the relation. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016CNVbr2GyVityjqt3LPAnU --- macros/export_state.sql | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/macros/export_state.sql b/macros/export_state.sql index a69f8d32..00f22d99 100644 --- a/macros/export_state.sql +++ b/macros/export_state.sql @@ -14,6 +14,37 @@ {% set relation = ref("dim_dbt__current_relations") %} {% set source_database = database if database is not none else relation.database %} {% set source_schema = schema if schema is not none else relation.schema %} + + {# `resource_types` is interpolated straight into an IN (...) list below, and + `source_database`/`source_schema` are interpolated indirectly: they become + the FROM-clause relation via api.Relation.create(), whose quoted() wraps + each part in the adapter's quote character but does not escape a quote + character embedded in the value, so an unvalidated value could still break + out of the quoted identifier. Validate both, ahead of any query, so a bad + value fails loudly instead of reaching the database. This is the one place + this macro may raise: validation happens before querying starts, so the + "never raises on a missing relation or zero rows" contract, which is about + behaviour once the query runs, is untouched. `target_name` is not + constrained to a known set (target names are project-specific), so it is + escaped as a SQL string literal instead, where it is used below. #} + {% set known_resource_types = ['model', 'seed', 'snapshot'] %} + {% for resource_type in resource_types %} + {% if resource_type not in known_resource_types %} + {% do exceptions.raise_compiler_error( + "export_state: unknown resource_type '" ~ resource_type ~ "'; expected one of " ~ known_resource_types | join(", ") + ) %} + {% endif %} + {% endfor %} + + {% set safe_identifier_pattern = '^[A-Za-z_][A-Za-z0-9_]*$' %} + {% for value, label in [(source_database, 'database'), (source_schema, 'schema')] %} + {% if not modules.re.fullmatch(safe_identifier_pattern, value) %} + {% do exceptions.raise_compiler_error( + "export_state: unsafe " ~ label ~ " value '" ~ value ~ "'; expected a plain SQL identifier matching " ~ safe_identifier_pattern + ) %} + {% endif %} + {% endfor %} + {% set target_relation = api.Relation.create( database=source_database, schema=source_schema, identifier=relation.identifier ) %} @@ -43,7 +74,11 @@ target_name from {{ target_relation }} where resource_type in ({{ "'" ~ resource_types | join("', '") ~ "'" }}) - {% if target_name is not none %} and target_name = '{{ target_name }}' {% endif %} + {# resource_types is validated above against a closed set, so no + escaping is needed here. target_name is project-specific and + not validated against a known set, so escape it as a SQL + string literal by doubling embedded single quotes. #} + {% if target_name is not none %} and target_name = '{{ target_name | replace("'", "''") }}' {% endif %} {% endset %} {% set results = run_query(query) %} From a455f14953be161e69368eaa571ddb76117b1a0d Mon Sep 17 00:00:00 2001 From: "Hans C. Ravache Lemm" Date: Fri, 11 Sep 2026 13:44:27 +0200 Subject: [PATCH 14/16] fix: filter target_name in Jinja and widen the identifier pattern Quote-doubling isn't a safe escape for target_name on every adapter this package supports: Snowflake, BigQuery, Spark and Databricks treat backslash as a string-literal escape character, so a value ending in an odd run of backslashes could desynchronise a doubled quote. Stop interpolating target_name into SQL entirely and filter matching rows in Jinja instead. Widen the database/schema identifier pattern to allow the characters real warehouse identifiers use (including the hyphens common in BigQuery project IDs) while still excluding quotes, semicolons, whitespace and backslashes. Treat an empty resource_types list as "export nothing" rather than building an invalid empty IN (). Make the top docstring precise about when the macro raises. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016CNVbr2GyVityjqt3LPAnU --- macros/export_state.sql | 71 ++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/macros/export_state.sql b/macros/export_state.sql index 00f22d99..58099863 100644 --- a/macros/export_state.sql +++ b/macros/export_state.sql @@ -1,8 +1,11 @@ {# Prints dim_dbt__current_relations as a versioned JSON document, for use as dbt deferral state. Emits with print() so `dbt --quiet run-operation` gives - clean JSON on stdout. Never raises: an absent relation or an empty result - yields an empty `nodes` object and a warning, leaving the caller to decide. + clean JSON on stdout. Never raises for a missing relation, an empty + resource_types list, or a zero-row result: each yields an empty `nodes` + object and a warning, leaving the caller to decide. Raises only on + invalid arguments (an unknown resource_type, or a database/schema value + that isn't a safe identifier). Usage: dbt --quiet run-operation dbt_artifacts.export_state \ @@ -25,8 +28,9 @@ this macro may raise: validation happens before querying starts, so the "never raises on a missing relation or zero rows" contract, which is about behaviour once the query runs, is untouched. `target_name` is not - constrained to a known set (target names are project-specific), so it is - escaped as a SQL string literal instead, where it is used below. #} + constrained to a known set (target names are project-specific) and is not + interpolated into SQL at all — see the row loop below — so it needs no + validation here. #} {% set known_resource_types = ['model', 'seed', 'snapshot'] %} {% for resource_type in resource_types %} {% if resource_type not in known_resource_types %} @@ -36,7 +40,13 @@ {% endif %} {% endfor %} - {% set safe_identifier_pattern = '^[A-Za-z_][A-Za-z0-9_]*$' %} + {# Not an exhaustive identifier grammar: the point is only to exclude + quotes, semicolons, whitespace and backslashes (the characters that + could break out of a quoted identifier or a SQL string literal), + while still accepting what real warehouses use here in practice — + including hyphens, which are near-universal in BigQuery project IDs + used as `database`. #} + {% set safe_identifier_pattern = '^[A-Za-z0-9_$.\-]+$' %} {% for value, label in [(source_database, 'database'), (source_schema, 'schema')] %} {% if not modules.re.fullmatch(safe_identifier_pattern, value) %} {% do exceptions.raise_compiler_error( @@ -55,7 +65,11 @@ {# load_relation (a dbt-core built-in) checks existence without raising, unlike selecting from the relation directly, which errors out when it is absent. #} - {% if load_relation(target_relation) is none %} + {% if resource_types | length == 0 %} + {# An empty IN (...) list is invalid SQL on most adapters; treat + "nothing requested" the same as "nothing found". #} + {% do log("export_state: resource_types is empty; exporting an empty document", info=False) %} + {% elif load_relation(target_relation) is none %} {% do log("export_state: " ~ target_relation ~ " does not exist; exporting an empty document", info=False) %} {% else %} {% set query %} @@ -74,30 +88,37 @@ target_name from {{ target_relation }} where resource_type in ({{ "'" ~ resource_types | join("', '") ~ "'" }}) - {# resource_types is validated above against a closed set, so no - escaping is needed here. target_name is project-specific and - not validated against a known set, so escape it as a SQL - string literal by doubling embedded single quotes. #} - {% if target_name is not none %} and target_name = '{{ target_name | replace("'", "''") }}' {% endif %} {% endset %} {% set results = run_query(query) %} + {# target_name is project-specific — not a closed set like + resource_types — so it isn't validated or interpolated into + SQL at all. Quote-doubling would not be a safe escape on + every adapter this package supports: Snowflake, BigQuery, + Spark and Databricks treat backslash as an escape character + in string literals, so a value ending in an odd run of + backslashes could desynchronise a doubled quote and reopen + the hole. Filtering here in Jinja, over a result set that is + at most hundreds of rows, avoids the per-adapter escaping + question entirely. #} {% for row in results.rows %} - {% do nodes.update({ - row["node_id"]: { - "resource_type": row["resource_type"], - "name": row["name"], - "package_name": row["package_name"], - "database": row["database"], - "schema": row["schema"], - "alias": row["alias"], - "materialization": row["materialization"], - "checksum": row["checksum"], - "last_success_at": row["last_success_at"] | string, - "command_invocation_id": row["command_invocation_id"], - } - }) %} + {% if target_name is none or row["target_name"] == target_name %} + {% do nodes.update({ + row["node_id"]: { + "resource_type": row["resource_type"], + "name": row["name"], + "package_name": row["package_name"], + "database": row["database"], + "schema": row["schema"], + "alias": row["alias"], + "materialization": row["materialization"], + "checksum": row["checksum"], + "last_success_at": row["last_success_at"] | string, + "command_invocation_id": row["command_invocation_id"], + } + }) %} + {% endif %} {% endfor %} {% if nodes | length == 0 %} From d152d7a9c3ce7b2e9f79a8cb3925a58f31e58474 Mon Sep 17 00:00:00 2001 From: "Hans C. Ravache Lemm" Date: Fri, 11 Sep 2026 14:44:05 +0200 Subject: [PATCH 15/16] fix(dim_dbt__current_relations): rank per (node_id, target_name); order nulls last Ranking success rows by node_id alone collapsed several targets writing to the same artifacts tables down to a single winner, so a node whose latest success ran on a different target than the one export_state's target_name argument requests was dropped instead of falling back to its own latest success on that target. Join stg_dbt__invocations in before ranking and partition by (node_id, target_name), making that the view's grain: one row per node per target. Add current_relation_id, a surrogate key of that grain (following the lineage_edge_id precedent) so it stays testable without a dbt_utils dependency, and update the schema tests and the raw-SQL uniqueness test to the new grain. Also order nulls last in the same ranking: NULLS FIRST is Postgres/Redshift's default for DESC, so a success with a null completion time would otherwise outrank a real one. The case-expression form is portable to every adapter this package supports, including SQL Server, which has no NULLS LAST clause. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016CNVbr2GyVityjqt3LPAnU --- .../tests/assert_current_relations_unique.sql | 11 ++-- models/dim_dbt__current_relations.sql | 64 ++++++++++++++----- models/dim_dbt__current_relations.yml | 18 ++++-- 3 files changed, 67 insertions(+), 26 deletions(-) diff --git a/integration_test_project/tests/assert_current_relations_unique.sql b/integration_test_project/tests/assert_current_relations_unique.sql index ed40cfbd..52d6e87d 100644 --- a/integration_test_project/tests/assert_current_relations_unique.sql +++ b/integration_test_project/tests/assert_current_relations_unique.sql @@ -1,12 +1,13 @@ {{ config(enabled = target.type in ["postgres", "redshift"]) }} --- dim_dbt__current_relations must hold at most one row per node and never a --- null relation coordinate: a consumer patches a manifest with these values, --- so a duplicate silently picks a winner and a null produces an unusable --- relation name. Returns rows (fails) on either condition. +-- dim_dbt__current_relations must hold at most one row per (node_id, +-- target_name) and never a null relation coordinate: a consumer patches a +-- manifest with these values, so a duplicate silently picks a winner and a +-- null produces an unusable relation name. Returns rows (fails) on either +-- condition. with duplicates as ( select node_id from {{ ref("dim_dbt__current_relations") }} - group by node_id + group by node_id, target_name having count(*) > 1 ), diff --git a/models/dim_dbt__current_relations.sql b/models/dim_dbt__current_relations.sql index 5661e03a..41af795f 100644 --- a/models/dim_dbt__current_relations.sql +++ b/models/dim_dbt__current_relations.sql @@ -1,9 +1,16 @@ {# - One row per refable node describing the relation production last built - successfully. Unlike dim_dbt__current_models, which scopes to the most - recent graph and covers models only, this view answers "what does the - warehouse actually hold right now" across models, seeds and snapshots, - which is what a deferral consumer needs. + One row per (node_id, target_name) describing the relation that target + last built successfully. Unlike dim_dbt__current_models, which scopes to + the most recent graph and covers models only, this view answers "what + does the warehouse actually hold right now, per target" across models, + seeds and snapshots, which is what a deferral consumer needs. + + The grain is per target, not per node, because several targets (e.g. a + prod and a staging environment) can write to the same artifacts tables. + Ranking node_id alone would collapse to one target's success and silently + drop any other target's latest relation. dbt_artifacts.export_state is + the layer that collapses this back to one row per node when its own + target_name argument isn't given. #} with executions as ( @@ -59,14 +66,44 @@ with ), + invocations as ( + + select command_invocation_id, target_name + from {{ ref("stg_dbt__invocations") }} + + ), + + {# target_name has to be known before ranking, not just attached to the + winner afterwards: the ranking itself must be scoped per target, or a + node whose latest success ran on a different target than the one + requested later (in export_state) is dropped instead of falling back + to its own latest success on the requested target. #} + executions_with_target as ( + + select executions.*, invocations.target_name + from executions + left join invocations + on executions.command_invocation_id = invocations.command_invocation_id + + ), + successes as ( select *, row_number() over ( - partition by node_id order by query_completed_at desc + partition by node_id, target_name + order by + {# A success with a null completion time must rank last, + not first: NULLS FIRST is Postgres/Redshift's default + for DESC, which would let a completion-time-less + success win over a real one. The case expression is + portable to every adapter this package supports, + including SQL Server, which has no NULLS LAST clause. #} + case when query_completed_at is null then 1 else 0 end, + query_completed_at desc ) as success_idx - from executions + from executions_with_target where status = 'success' ), @@ -123,16 +160,11 @@ with ), - invocations as ( - - select command_invocation_id, target_name - from {{ ref("stg_dbt__invocations") }} - - ), - final as ( select + {{ dbt_artifacts.generate_surrogate_key(["latest_success.node_id", "latest_success.target_name"]) }} + as current_relation_id, latest_success.node_id, latest_success.resource_type, latest_success.name, @@ -145,7 +177,7 @@ with latest_success.query_completed_at as last_success_at, latest_success.command_invocation_id, latest_success.run_started_at, - invocations.target_name, + latest_success.target_name, case when latest_graph.node_id is not null then true else false end as in_latest_graph @@ -153,8 +185,6 @@ with inner join dimensions on latest_success.command_invocation_id = dimensions.command_invocation_id and latest_success.node_id = dimensions.node_id - left join invocations - on latest_success.command_invocation_id = invocations.command_invocation_id left join latest_graph on latest_success.node_id = latest_graph.node_id ) diff --git a/models/dim_dbt__current_relations.yml b/models/dim_dbt__current_relations.yml index 68647e41..b940f5cb 100644 --- a/models/dim_dbt__current_relations.yml +++ b/models/dim_dbt__current_relations.yml @@ -3,14 +3,24 @@ version: 2 models: - name: dim_dbt__current_relations description: | - One row per refable node (model, seed, snapshot) describing the relation - that the most recent successful execution wrote. Intended as deferral - state for CI: see "Using dbt_artifacts as deferral state" in the README. + One row per (node_id, target_name) describing the relation that target's + most recent successful execution wrote. Several targets (e.g. prod and + staging) can write to the same artifacts tables, so the grain is per + target, not per node: dbt_artifacts.export_state is what collapses this + to one row per node, picking the freshest target, when its own + target_name argument isn't given. Intended as deferral state for CI: see + "Using dbt_artifacts as deferral state" in the README. columns: + - name: current_relation_id + description: > + Surrogate key of (node_id, target_name), the grain of this view. Added + so the grain can be tested without a dbt_utils dependency. + tests: + - unique + - not_null - name: node_id description: '{{ doc("node_id") }}' tests: - - unique - not_null - name: resource_type description: One of model, seed or snapshot. From 2179d12c42e761a2a4294d4a217bcc624131b360 Mon Sep 17 00:00:00 2001 From: "Hans C. Ravache Lemm" Date: Fri, 11 Sep 2026 14:44:14 +0200 Subject: [PATCH 16/16] fix(export_state): fall back to a node's own target; emit ISO 8601 timestamps dim_dbt__current_relations is now grained on (node_id, target_name). export_state's row loop filtered by target_name after the view had already collapsed to one row per node_id, so a node whose winning row belonged to a different target than requested was silently dropped rather than falling back to its own latest success on the requested target. With the view now returning up to one row per node per target, the loop only needs to pick the right row: a straight match when target_name is given (now correct, since every matching row is genuinely that node's row for that target), and the most recently completed success across targets when it isn't, preserving the single-target-project default of one row per node. Also fix last_success_at: `| string` produced Postgres's space-separated, offset-less format, contradicting the ISO 8601 promise in the README and the consuming repo's spec. A value with no tzinfo is UTC wall-clock time (every adapter this package writes to records query_completed_at from dbt's own UTC run-results timing), so attach UTC explicitly before calling isoformat(), which gives the `T` separator and offset uniformly across adapters. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016CNVbr2GyVityjqt3LPAnU --- macros/_macros.yml | 5 ++- macros/export_state.sql | 89 ++++++++++++++++++++++++++++++++++------- 2 files changed, 78 insertions(+), 16 deletions(-) diff --git a/macros/_macros.yml b/macros/_macros.yml index c47a4dc0..5f5bb689 100644 --- a/macros/_macros.yml +++ b/macros/_macros.yml @@ -314,5 +314,6 @@ macros: - name: target_name type: string description: | - When set, exports only rows whose invocation ran with this target - name. Use it when several targets write to the same artifacts tables. + When set, exports each node's latest success on this target only. + Use it when several targets write to the same artifacts tables. + When unset, exports each node's latest success across all targets. diff --git a/macros/export_state.sql b/macros/export_state.sql index 58099863..4014ee0c 100644 --- a/macros/export_state.sql +++ b/macros/export_state.sql @@ -7,6 +7,13 @@ invalid arguments (an unknown resource_type, or a database/schema value that isn't a safe identifier). + Emits one node per node_id. dim_dbt__current_relations is grained on + (node_id, target_name), since several targets can write to the same + artifacts tables; when target_name is given, each node's row for that + target is used, and when it isn't, each node's most recently completed + success across all targets is used, matching the single-target-project + behaviour where there is only ever one target to pick from. + Usage: dbt --quiet run-operation dbt_artifacts.export_state \ --args '{schema: dbt_artifacts, target_name: betterdata_prod}' > export.json @@ -92,6 +99,24 @@ {% set results = run_query(query) %} + {# dim_dbt__current_relations is grained on (node_id, + target_name): several targets writing to the same artifacts + tables can each have their own latest success for the same + node. Two cases: + - target_name given: at most one row per node_id already + matches it, so filtering is a straight lookup. + - target_name not given: more than one row per node_id can + come back (one per target that has ever built the node). + Collapse to one, keeping the most recently completed + success, so the single-target-project default behaviour + (one row per node) is unchanged and the multi-target case + picks the truly freshest relation instead of an arbitrary + target's. + `last_success_at_sort_keys` holds the raw (pre-formatting) + completion time used only for that comparison; it never + reaches the emitted document. #} + {% set last_success_at_sort_keys = {} %} + {# target_name is project-specific — not a closed set like resource_types — so it isn't validated or interpolated into SQL at all. Quote-doubling would not be a safe escape on @@ -104,20 +129,56 @@ question entirely. #} {% for row in results.rows %} {% if target_name is none or row["target_name"] == target_name %} - {% do nodes.update({ - row["node_id"]: { - "resource_type": row["resource_type"], - "name": row["name"], - "package_name": row["package_name"], - "database": row["database"], - "schema": row["schema"], - "alias": row["alias"], - "materialization": row["materialization"], - "checksum": row["checksum"], - "last_success_at": row["last_success_at"] | string, - "command_invocation_id": row["command_invocation_id"], - } - }) %} + {% set completed_at = row["last_success_at"] %} + {% set current_best = last_success_at_sort_keys.get(row["node_id"]) %} + {# A specific target_name already guarantees at most one + matching row per node_id, so every match wins + outright. With no target_name, only overwrite the + node's current winner when this row is strictly more + recent; a null completion time never displaces an + existing real one, but still seeds the entry the + first time a node is seen. #} + {% set wins = target_name is not none + or row["node_id"] not in nodes + or (completed_at is not none and (current_best is none or completed_at > current_best)) %} + {% if wins %} + {% set last_success_at = none %} + {% if completed_at is not none %} + {% set last_success_at = completed_at %} + {# Every adapter this package writes to records + query_completed_at from dbt's own UTC + run-results timing. A value that comes back + with no tzinfo (e.g. Postgres/Redshift's + timestamp-without-time-zone) is UTC wall-clock + time, not an unknown offset, so attaching it + explicitly turns the naive value into a real + instant. A value that already carries tzinfo + (e.g. Snowflake's TIMESTAMP_TZ) is left as-is. + isoformat() then gives the `T` separator and + explicit offset the README documents, instead + of the space-separated, offset-less string + `| string` produced. #} + {% if last_success_at.tzinfo is none %} + {% set last_success_at = last_success_at.replace(tzinfo=modules.pytz.utc) %} + {% endif %} + {% set last_success_at = last_success_at.isoformat() %} + {% endif %} + {% do nodes.update({ + row["node_id"]: { + "resource_type": row["resource_type"], + "name": row["name"], + "package_name": row["package_name"], + "database": row["database"], + "schema": row["schema"], + "alias": row["alias"], + "materialization": row["materialization"], + "checksum": row["checksum"], + "last_success_at": last_success_at, + "command_invocation_id": row["command_invocation_id"], + } + }) %} + {% do last_success_at_sort_keys.update({row["node_id"]: completed_at}) %} + {% endif %} {% endif %} {% endfor %}