diff --git a/README.md b/README.md index 45d5f3a5..df6235a8 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ The package currently supports - Postgres :white_check_mark: - SQL Server :white_check_mark: - Trino :white_check_mark: +- Redshift ✅ Models included: @@ -238,6 +239,49 @@ An example operation is as follows: dbt run-operation migrate_from_v0_to_v1 --args '{old_database: analytics, old_schema: dbt_artifacts, new_database: analytics, new_schema: artifact_sources}' ``` +## Using dbt_artifacts as deferral state + +`dbt build --defer --state` reads a `manifest.json` from disk, and the only +thing it takes from that manifest for unselected nodes is where each relation +lives. `dim_dbt__current_relations` knows that from your production runs, so a +CI job can parse its own project for a structurally complete manifest and then +correct the relation coordinates from the warehouse, instead of shipping a +manifest between jobs. + +```bash +dbt --quiet run-operation dbt_artifacts.export_state \ + --args '{schema: dbt_artifacts, target_name: prod}' > export.json +``` + +The document is versioned; refuse a `dbt_artifacts_state_version` whose major +differs from the one you support. + +```json +{ + "dbt_artifacts_state_version": 1, + "generated_at": "2026-09-11T09:00:00+00:00", + "source": {"database": "analytics", "schema": "dbt_artifacts", "target_name": "prod"}, + "nodes": { + "model.my_project.dim_customer": { + "resource_type": "model", + "name": "dim_customer", + "package_name": "my_project", + "database": "analytics", + "schema": "marts", + "alias": "dim_customer", + "materialization": "table", + "checksum": "9f2c…", + "last_success_at": "2026-09-10T02:14:09+00:00", + "command_invocation_id": "0f0a…" + } + } +} +``` + +A node appears when production executed it successfully at least once; the row +shown is the most recent success. An empty `nodes` object is a valid document +and means the tables hold no successful executions yet. + ## Acknowledgements Thank you to [Tails.com](https://tails.com/gb/careers/) for initial development and maintenance of this package. On 2021/12/20, the repository was transferred from the Tails.com GitHub organization to Brooklyn Data Co. diff --git a/integration_test_project/example-env.sh b/integration_test_project/example-env.sh index 47cb0d67..859cb0fc 100755 --- a/integration_test_project/example-env.sh +++ b/integration_test_project/example-env.sh @@ -16,6 +16,10 @@ export DBT_ENV_SECRET_DATABRICKS_TOKEN= export DBT_ENV_SECRET_GCP_PROJECT= export DBT_ENV_SPARK_DRIVER_PATH= # /Library/simba/spark/lib/libsparkodbc_sbu.dylib on a Mac export DBT_ENV_SPARK_ENDPOINT= # The endpoint ID from the Databricks HTTP path +export DBT_ENV_SECRET_REDSHIFT_HOST= +export DBT_ENV_SECRET_REDSHIFT_CLUSTER_ID= +export DBT_ENV_SECRET_REDSHIFT_DB= +export DBT_ENV_SECRET_REDSHIFT_USER= # dbt environment variables, change these export DBT_VERSION="1_5_0" diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index e12458c0..76236e02 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -66,6 +66,17 @@ dbt_artifacts: trust_cert: True Encrypt: False user: dbt + password: "123" + redshift: + type: redshift + method: iam + threads: 8 + host: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_HOST') }}" + port: 5439 + dbname: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_DB') }}" + user: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_USER') }}" + schema: dbt_artifacts_test_commit_{{ env_var('DBT_VERSION', '') }}_{{ env_var('GITHUB_SHA_OVERRIDE', '') if env_var('GITHUB_SHA_OVERRIDE', '') else env_var('GITHUB_SHA') }} + cluster_id: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_CLUSTER_ID') }}" password: "123Administrator" trino: type: trino diff --git a/integration_test_project/tests/assert_current_relations_unique.sql b/integration_test_project/tests/assert_current_relations_unique.sql new file mode 100644 index 00000000..52d6e87d --- /dev/null +++ b/integration_test_project/tests/assert_current_relations_unique.sql @@ -0,0 +1,25 @@ +{{ config(enabled = target.type in ["postgres", "redshift"]) }} +-- dim_dbt__current_relations must hold at most one row per (node_id, +-- target_name) and never a null relation coordinate: a consumer patches a +-- manifest with these values, so a duplicate silently picks a winner and a +-- null produces an unusable relation name. Returns rows (fails) on either +-- condition. +with duplicates as ( + select node_id + from {{ ref("dim_dbt__current_relations") }} + group by node_id, target_name + having count(*) > 1 +), + +nulls as ( + select node_id + from {{ ref("dim_dbt__current_relations") }} + where database is null + or schema is null + or alias is null + or resource_type is null +) + +select node_id from duplicates +union all +select node_id from nulls diff --git a/macros/_macros.yml b/macros/_macros.yml index eda54878..5f5bb689 100644 --- a/macros/_macros.yml +++ b/macros/_macros.yml @@ -57,6 +57,10 @@ macros: description: | Dependent on the adapter type, returns the native type for storing JSON. + - name: type_string + description: | + Dependent on the adapter type, returns the native type for storing a string. + ## MIGRATION ## - name: migrate_from_v0_to_v1 description: | @@ -285,3 +289,31 @@ macros: type: list[any] description: | The results object from dbt. + + ## DEFERRAL STATE ## + - name: export_state + description: | + Prints dim_dbt__current_relations as a versioned JSON document for use as + dbt deferral state. See "Using dbt_artifacts as deferral state" in the + README for the contract. + arguments: + - name: resource_types + type: list[string] + description: | + Which resource types to export. Defaults to model, seed and snapshot; + deferral resolves refable nodes only. + - name: database + type: string + description: | + Overrides the database of dim_dbt__current_relations. Needed when the + session runs on a target whose ref() would resolve elsewhere. + - name: schema + type: string + description: | + Overrides the schema of dim_dbt__current_relations. + - name: target_name + type: string + description: | + When set, exports each node's latest success on this target only. + Use it when several targets write to the same artifacts tables. + When unset, exports each node's latest success across all targets. diff --git a/macros/database_specific_helpers/type_helpers.sql b/macros/database_specific_helpers/type_helpers.sql index cbc19e5f..e553377e 100644 --- a/macros/database_specific_helpers/type_helpers.sql +++ b/macros/database_specific_helpers/type_helpers.sql @@ -26,10 +26,14 @@ json {% endmacro %} +{% macro redshift__type_json() %} + super +{% endmacro %} + {#- ARRAY -#} {% macro type_array() %} - {{ return(adapter.dispatch('type_array', 'dbt_artifacts')()) }} + {{ return(adapter.dispatch('type_array', 'dbt_artifacts')()) }} {% endmacro %} {% macro default__type_array() %} @@ -44,6 +48,22 @@ array {% endmacro %} +{% macro redshift__type_array() %} + super +{% endmacro %} + +{% macro type_string() %} + {{ return(adapter.dispatch('type_string', 'dbt_artifacts')()) }} +{% endmacro %} + +{% macro default__type_string() %} + {{ return(api.Column.translate_type("string")) }} +{% endmacro %} + +{% macro redshift__type_string() %} + varchar(max) +{% endmacro %} + {% macro trino__type_array() %} array(varchar) {% endmacro %} diff --git a/macros/export_state.sql b/macros/export_state.sql new file mode 100644 index 00000000..4014ee0c --- /dev/null +++ b/macros/export_state.sql @@ -0,0 +1,204 @@ +{# + Prints dim_dbt__current_relations as a versioned JSON document, for use as + dbt deferral state. Emits with print() so `dbt --quiet run-operation` gives + clean JSON on stdout. Never raises for a missing relation, an empty + resource_types list, or a zero-row result: each yields an empty `nodes` + object and a warning, leaving the caller to decide. Raises only on + invalid arguments (an unknown resource_type, or a database/schema value + that isn't a safe identifier). + + Emits one node per node_id. dim_dbt__current_relations is grained on + (node_id, target_name), since several targets can write to the same + artifacts tables; when target_name is given, each node's row for that + target is used, and when it isn't, each node's most recently completed + success across all targets is used, matching the single-target-project + behaviour where there is only ever one target to pick from. + + Usage: + dbt --quiet run-operation dbt_artifacts.export_state \ + --args '{schema: dbt_artifacts, target_name: betterdata_prod}' > export.json +#} +{% macro export_state(resource_types=['model', 'seed', 'snapshot'], database=none, schema=none, target_name=none) %} + + {% set state_version = 1 %} + {% set relation = ref("dim_dbt__current_relations") %} + {% set source_database = database if database is not none else relation.database %} + {% set source_schema = schema if schema is not none else relation.schema %} + + {# `resource_types` is interpolated straight into an IN (...) list below, and + `source_database`/`source_schema` are interpolated indirectly: they become + the FROM-clause relation via api.Relation.create(), whose quoted() wraps + each part in the adapter's quote character but does not escape a quote + character embedded in the value, so an unvalidated value could still break + out of the quoted identifier. Validate both, ahead of any query, so a bad + value fails loudly instead of reaching the database. This is the one place + this macro may raise: validation happens before querying starts, so the + "never raises on a missing relation or zero rows" contract, which is about + behaviour once the query runs, is untouched. `target_name` is not + constrained to a known set (target names are project-specific) and is not + interpolated into SQL at all — see the row loop below — so it needs no + validation here. #} + {% set known_resource_types = ['model', 'seed', 'snapshot'] %} + {% for resource_type in resource_types %} + {% if resource_type not in known_resource_types %} + {% do exceptions.raise_compiler_error( + "export_state: unknown resource_type '" ~ resource_type ~ "'; expected one of " ~ known_resource_types | join(", ") + ) %} + {% endif %} + {% endfor %} + + {# Not an exhaustive identifier grammar: the point is only to exclude + quotes, semicolons, whitespace and backslashes (the characters that + could break out of a quoted identifier or a SQL string literal), + while still accepting what real warehouses use here in practice — + including hyphens, which are near-universal in BigQuery project IDs + used as `database`. #} + {% set safe_identifier_pattern = '^[A-Za-z0-9_$.\-]+$' %} + {% for value, label in [(source_database, 'database'), (source_schema, 'schema')] %} + {% if not modules.re.fullmatch(safe_identifier_pattern, value) %} + {% do exceptions.raise_compiler_error( + "export_state: unsafe " ~ label ~ " value '" ~ value ~ "'; expected a plain SQL identifier matching " ~ safe_identifier_pattern + ) %} + {% endif %} + {% endfor %} + + {% set target_relation = api.Relation.create( + database=source_database, schema=source_schema, identifier=relation.identifier + ) %} + + {% set nodes = {} %} + + {% if execute %} + {# load_relation (a dbt-core built-in) checks existence without raising, + unlike selecting from the relation directly, which errors out when + it is absent. #} + {% if resource_types | length == 0 %} + {# An empty IN (...) list is invalid SQL on most adapters; treat + "nothing requested" the same as "nothing found". #} + {% do log("export_state: resource_types is empty; exporting an empty document", info=False) %} + {% elif load_relation(target_relation) is none %} + {% do log("export_state: " ~ target_relation ~ " does not exist; exporting an empty document", info=False) %} + {% else %} + {% set query %} + select + node_id, + resource_type, + name, + package_name, + {% if target.type == "sqlserver" %} "database" {% else %} database {% endif %}, -- noqa + {% if target.type == "sqlserver" %} "schema" {% else %} schema {% endif %}, -- noqa + alias, + materialization, + checksum, + last_success_at, + command_invocation_id, + target_name + from {{ target_relation }} + where resource_type in ({{ "'" ~ resource_types | join("', '") ~ "'" }}) + {% endset %} + + {% set results = run_query(query) %} + + {# dim_dbt__current_relations is grained on (node_id, + target_name): several targets writing to the same artifacts + tables can each have their own latest success for the same + node. Two cases: + - target_name given: at most one row per node_id already + matches it, so filtering is a straight lookup. + - target_name not given: more than one row per node_id can + come back (one per target that has ever built the node). + Collapse to one, keeping the most recently completed + success, so the single-target-project default behaviour + (one row per node) is unchanged and the multi-target case + picks the truly freshest relation instead of an arbitrary + target's. + `last_success_at_sort_keys` holds the raw (pre-formatting) + completion time used only for that comparison; it never + reaches the emitted document. #} + {% set last_success_at_sort_keys = {} %} + + {# target_name is project-specific — not a closed set like + resource_types — so it isn't validated or interpolated into + SQL at all. Quote-doubling would not be a safe escape on + every adapter this package supports: Snowflake, BigQuery, + Spark and Databricks treat backslash as an escape character + in string literals, so a value ending in an odd run of + backslashes could desynchronise a doubled quote and reopen + the hole. Filtering here in Jinja, over a result set that is + at most hundreds of rows, avoids the per-adapter escaping + question entirely. #} + {% for row in results.rows %} + {% if target_name is none or row["target_name"] == target_name %} + {% set completed_at = row["last_success_at"] %} + {% set current_best = last_success_at_sort_keys.get(row["node_id"]) %} + {# A specific target_name already guarantees at most one + matching row per node_id, so every match wins + outright. With no target_name, only overwrite the + node's current winner when this row is strictly more + recent; a null completion time never displaces an + existing real one, but still seeds the entry the + first time a node is seen. #} + {% set wins = target_name is not none + or row["node_id"] not in nodes + or (completed_at is not none and (current_best is none or completed_at > current_best)) %} + {% if wins %} + {% set last_success_at = none %} + {% if completed_at is not none %} + {% set last_success_at = completed_at %} + {# Every adapter this package writes to records + query_completed_at from dbt's own UTC + run-results timing. A value that comes back + with no tzinfo (e.g. Postgres/Redshift's + timestamp-without-time-zone) is UTC wall-clock + time, not an unknown offset, so attaching it + explicitly turns the naive value into a real + instant. A value that already carries tzinfo + (e.g. Snowflake's TIMESTAMP_TZ) is left as-is. + isoformat() then gives the `T` separator and + explicit offset the README documents, instead + of the space-separated, offset-less string + `| string` produced. #} + {% if last_success_at.tzinfo is none %} + {% set last_success_at = last_success_at.replace(tzinfo=modules.pytz.utc) %} + {% endif %} + {% set last_success_at = last_success_at.isoformat() %} + {% endif %} + {% do nodes.update({ + row["node_id"]: { + "resource_type": row["resource_type"], + "name": row["name"], + "package_name": row["package_name"], + "database": row["database"], + "schema": row["schema"], + "alias": row["alias"], + "materialization": row["materialization"], + "checksum": row["checksum"], + "last_success_at": last_success_at, + "command_invocation_id": row["command_invocation_id"], + } + }) %} + {% do last_success_at_sort_keys.update({row["node_id"]: completed_at}) %} + {% endif %} + {% endif %} + {% endfor %} + + {% if nodes | length == 0 %} + {% do log("export_state: no successful executions found in " ~ target_relation ~ "; exporting an empty document", info=False) %} + {% endif %} + {% endif %} + {% endif %} + + {% set document = { + "dbt_artifacts_state_version": state_version, + "generated_at": modules.datetime.datetime.now(modules.pytz.utc).isoformat(), + "source": { + "database": source_database, + "schema": source_schema, + "target_name": target_name, + }, + "nodes": nodes, + } %} + + {% do print(tojson(document)) %} + +{% endmacro %} diff --git a/models/dim_dbt__current_relations.sql b/models/dim_dbt__current_relations.sql new file mode 100644 index 00000000..41af795f --- /dev/null +++ b/models/dim_dbt__current_relations.sql @@ -0,0 +1,193 @@ +{# + One row per (node_id, target_name) describing the relation that target + last built successfully. Unlike dim_dbt__current_models, which scopes to + the most recent graph and covers models only, this view answers "what + does the warehouse actually hold right now, per target" across models, + seeds and snapshots, which is what a deferral consumer needs. + + The grain is per target, not per node, because several targets (e.g. a + prod and a staging environment) can write to the same artifacts tables. + Ranking node_id alone would collapse to one target's success and silently + drop any other target's latest relation. dbt_artifacts.export_state is + the layer that collapses this back to one row per node when its own + target_name argument isn't given. +#} +with + executions as ( + + select + command_invocation_id, + node_id, + 'model' as resource_type, + run_started_at, + status, + query_completed_at, + materialization, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, -- noqa + name, + alias + from {{ ref("stg_dbt__model_executions") }} + + union all + + select + command_invocation_id, + node_id, + 'seed' as resource_type, + run_started_at, + status, + query_completed_at, + materialization, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, -- noqa + name, + alias + from {{ ref("stg_dbt__seed_executions") }} + + union all + + select + command_invocation_id, + node_id, + 'snapshot' as resource_type, + run_started_at, + status, + query_completed_at, + materialization, + {% if target.type == "sqlserver" %} "schema" + {% else %} schema + {% endif %}, -- noqa + name, + alias + from {{ ref("stg_dbt__snapshot_executions") }} + + ), + + invocations as ( + + select command_invocation_id, target_name + from {{ ref("stg_dbt__invocations") }} + + ), + + {# target_name has to be known before ranking, not just attached to the + winner afterwards: the ranking itself must be scoped per target, or a + node whose latest success ran on a different target than the one + requested later (in export_state) is dropped instead of falling back + to its own latest success on the requested target. #} + executions_with_target as ( + + select executions.*, invocations.target_name + from executions + left join invocations + on executions.command_invocation_id = invocations.command_invocation_id + + ), + + successes as ( + + select + *, + row_number() over ( + partition by node_id, target_name + order by + {# A success with a null completion time must rank last, + not first: NULLS FIRST is Postgres/Redshift's default + for DESC, which would let a completion-time-less + success win over a real one. The case expression is + portable to every adapter this package supports, + including SQL Server, which has no NULLS LAST clause. #} + case when query_completed_at is null then 1 else 0 end, + query_completed_at desc + ) as success_idx + from executions_with_target + where status = 'success' + + ), + + latest_success as (select * from successes where success_idx = 1), + + dimensions as ( + + select + command_invocation_id, + node_id, + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, -- noqa + package_name, + checksum, + run_started_at + from {{ ref("stg_dbt__models") }} + + union all + + select + command_invocation_id, + node_id, + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, -- noqa + package_name, + checksum, + run_started_at + from {{ ref("stg_dbt__seeds") }} + + union all + + select + command_invocation_id, + node_id, + {% if target.type == "sqlserver" %} "database" + {% else %} database + {% endif %}, -- noqa + package_name, + checksum, + run_started_at + from {{ ref("stg_dbt__snapshots") }} + + ), + + latest_graph as ( + + {# Nodes present in the most recent graph, whether or not they ran. #} + select node_id + from dimensions + where run_started_at = (select max(d.run_started_at) from dimensions as d) + + ), + + final as ( + + select + {{ dbt_artifacts.generate_surrogate_key(["latest_success.node_id", "latest_success.target_name"]) }} + as current_relation_id, + latest_success.node_id, + latest_success.resource_type, + latest_success.name, + dimensions.package_name, + dimensions.database, + latest_success.schema, + latest_success.alias, + latest_success.materialization, + dimensions.checksum, + latest_success.query_completed_at as last_success_at, + latest_success.command_invocation_id, + latest_success.run_started_at, + latest_success.target_name, + case + when latest_graph.node_id is not null then true else false + end as in_latest_graph + from latest_success + inner join dimensions + on latest_success.command_invocation_id = dimensions.command_invocation_id + and latest_success.node_id = dimensions.node_id + left join latest_graph on latest_success.node_id = latest_graph.node_id + + ) + +select * +from final diff --git a/models/dim_dbt__current_relations.yml b/models/dim_dbt__current_relations.yml new file mode 100644 index 00000000..b940f5cb --- /dev/null +++ b/models/dim_dbt__current_relations.yml @@ -0,0 +1,52 @@ +version: 2 + +models: +- name: dim_dbt__current_relations + description: | + One row per (node_id, target_name) describing the relation that target's + most recent successful execution wrote. Several targets (e.g. prod and + staging) can write to the same artifacts tables, so the grain is per + target, not per node: dbt_artifacts.export_state is what collapses this + to one row per node, picking the freshest target, when its own + target_name argument isn't given. Intended as deferral state for CI: see + "Using dbt_artifacts as deferral state" in the README. + columns: + - name: current_relation_id + description: > + Surrogate key of (node_id, target_name), the grain of this view. Added + so the grain can be tested without a dbt_utils dependency. + tests: + - unique + - not_null + - name: node_id + description: '{{ doc("node_id") }}' + tests: + - not_null + - name: resource_type + description: One of model, seed or snapshot. + - name: name + description: '{{ doc("name") }}' + - name: package_name + description: '{{ doc("package_name") }}' + - name: database + description: '{{ doc("database") }}' + - name: schema + description: '{{ doc("schema") }}' + - name: alias + description: '{{ doc("alias") }}' + - name: materialization + description: '{{ doc("materialization") }}' + - name: checksum + description: '{{ doc("checksum") }}' + - name: last_success_at + description: Completion time of the most recent successful execution of this node. + - name: command_invocation_id + description: '{{ doc("command_invocation_id") }}' + - name: run_started_at + description: '{{ doc("run_started_at") }}' + - name: target_name + description: '{{ doc("target_name") }}' + - name: in_latest_graph + description: | + True when the node is present in the most recent graph. False marks a + relation that still exists in the warehouse but has left the project. diff --git a/models/docs.md b/models/docs.md index 64b420cb..dfc6947d 100644 --- a/models/docs.md +++ b/models/docs.md @@ -412,3 +412,9 @@ Response provided by the adapter as JSON. All results as a JSON blob {% enddocs %} + +{% docs last_success_at %} + +Completion time of the most recent successful execution of this node. + +{% enddocs %} diff --git a/models/fct_dbt__consumption_by_model.yml b/models/fct_dbt__consumption_by_model.yml index 6bfb192e..c366e9e5 100644 --- a/models/fct_dbt__consumption_by_model.yml +++ b/models/fct_dbt__consumption_by_model.yml @@ -1,46 +1,47 @@ version: 2 models: -- name: fct_dbt__consumption_by_model - description: > - Consumption attributed per model: one row per billing_month x node_id - (models only). Same SMB definition as fct_dbt__consumption_daily - (deployment-classed successful model builds on run/build/retry), so monthly - SMB totals reconcile across the two marts. Includes build cadence, total - runtime, and a dbt State ROI estimate. Primary use case: "top 20 models by - SMB burn." - columns: - - name: consumption_by_model_id - description: Surrogate key of the grain (hash of billing_month, node_id). BI join key. - tests: - - unique - - not_null - - name: billing_month - description: First day of the calendar month (UTC) the SMB were built in. - tests: - - not_null - - name: node_id - description: The model's unique node id. - tests: - - not_null - - name: name - description: The model's name. - - name: smb_quantity - description: Successful Models Built for this model in the month (deployment, run/build/retry). - - name: pct_of_month_smb - description: This model's share of the month's total SMB (0..1), a window over billing_month. - tests: - - dbt_artifacts.is_between: - min_value: 0 - max_value: 1 - - name: builds_per_day_avg - description: Average successful builds per active build-day (smb_quantity / distinct_days_built). - - name: distinct_days_built - description: Number of distinct UTC days on which the model was successfully built (deployment). - - name: total_runtime_seconds - description: Sum of total_node_runtime over the model's SMB executions in the month. - - name: estimated_monthly_datt_cost_if_reused + - name: fct_dbt__consumption_by_model description: > - dbt State ROI estimate = distinct_days_built x dbt_artifacts_datt_price - (default 0.094). A directional upper-bound of what dbt State could meter - for this model. + Consumption attributed per model: one row per billing_month x node_id + (models only). Same SMB definition as fct_dbt__consumption_daily + (deployment-classed successful model builds on run/build/retry), so monthly + SMB totals reconcile across the two marts. Includes build cadence, total + runtime, and a dbt State ROI estimate. Primary use case: "top 20 models by + SMB burn." + columns: + - name: consumption_by_model_id + description: Surrogate key of the grain (hash of billing_month, node_id). BI join key. + tests: + - unique + - not_null + - name: billing_month + description: First day of the calendar month (UTC) the SMB were built in. + tests: + - not_null + - name: node_id + description: The model's unique node id. + tests: + - not_null + - name: name + description: The model's name. + - name: smb_quantity + description: Successful Models Built for this model in the month (deployment, run/build/retry). + - name: pct_of_month_smb + description: This model's share of the month's total SMB (0..1), a window over billing_month. + tests: + - dbt_artifacts.is_between: + arguments: + min_value: 0 + max_value: 1 + - name: builds_per_day_avg + description: Average successful builds per active build-day (smb_quantity / distinct_days_built). + - name: distinct_days_built + description: Number of distinct UTC days on which the model was successfully built (deployment). + - name: total_runtime_seconds + description: Sum of total_node_runtime over the model's SMB executions in the month. + - name: estimated_monthly_datt_cost_if_reused + description: > + dbt State ROI estimate = distinct_days_built x dbt_artifacts_datt_price + (default 0.094). A directional upper-bound of what dbt State could meter + for this model. diff --git a/models/fct_dbt__consumption_daily.yml b/models/fct_dbt__consumption_daily.yml index 311d6ac7..6118bf4b 100644 --- a/models/fct_dbt__consumption_daily.yml +++ b/models/fct_dbt__consumption_daily.yml @@ -1,31 +1,32 @@ version: 2 models: -- name: fct_dbt__consumption_daily - description: > - Core consumption mart: one row per UTC day x meter. A strict roll-up of - fct_dbt__consumption_daily_detail (sum of quantity), so daily totals always - reconcile exactly to the detail grain. v1 meters: 'smb' (Successful Models - Built, mirroring dbt's published rules) and 'active_target_tables' (distinct - deployment-active nodes, the DATT upper-bound proxy). Consumption metrics - only count deployment-classed invocations (see classify_invocation_billing). - columns: - - name: consumption_daily_id - description: Surrogate key of the grain (hash of date_day, meter). Stable BI join key. - tests: - - unique - - not_null - - name: date_day - description: UTC calendar day of the executions (via cast_to_utc_date()). - tests: - - not_null - - name: meter - description: "Consumption meter: 'smb' or 'active_target_tables'." - tests: - - not_null - - accepted_values: - values: ['smb', 'active_target_tables'] - - name: quantity - description: Metered quantity for the day x meter (sum of the detail rows). - tests: - - not_null + - name: fct_dbt__consumption_daily + description: > + Core consumption mart: one row per UTC day x meter. A strict roll-up of + fct_dbt__consumption_daily_detail (sum of quantity), so daily totals always + reconcile exactly to the detail grain. v1 meters: 'smb' (Successful Models + Built, mirroring dbt's published rules) and 'active_target_tables' (distinct + deployment-active nodes, the DATT upper-bound proxy). Consumption metrics + only count deployment-classed invocations (see classify_invocation_billing). + columns: + - name: consumption_daily_id + description: Surrogate key of the grain (hash of date_day, meter). Stable BI join key. + tests: + - unique + - not_null + - name: date_day + description: UTC calendar day of the executions (via cast_to_utc_date()). + tests: + - not_null + - name: meter + description: "Consumption meter: 'smb' or 'active_target_tables'." + tests: + - not_null + - accepted_values: + arguments: + values: ['smb', 'active_target_tables'] + - name: quantity + description: Metered quantity for the day x meter (sum of the detail rows). + tests: + - not_null diff --git a/models/fct_dbt__consumption_daily_detail.yml b/models/fct_dbt__consumption_daily_detail.yml index 6ffdba87..cdbd2703 100644 --- a/models/fct_dbt__consumption_daily_detail.yml +++ b/models/fct_dbt__consumption_daily_detail.yml @@ -1,39 +1,40 @@ version: 2 models: -- name: fct_dbt__consumption_daily_detail - description: > - Detail (base) grain for consumption: UTC day x meter x materialization x - target_name. fct_dbt__consumption_daily is a strict roll-up of this model. - For 'smb' rows, materialization is the model's materialization; for - 'active_target_tables' rows materialization is null and the count is - distinct nodes per deployment target that day. Only deployment-classed - invocations are counted. - columns: - - name: consumption_daily_detail_id + - name: fct_dbt__consumption_daily_detail description: > - Surrogate key of the grain (hash of date_day, meter, materialization, - target_name). Stable BI join key. - tests: - - unique - - not_null - - name: date_day - description: UTC calendar day of the executions (via cast_to_utc_date()). - tests: - - not_null - - name: meter - description: "Consumption meter: 'smb' or 'active_target_tables'." - tests: - - not_null - - accepted_values: - values: ['smb', 'active_target_tables'] - - name: materialization - description: > - Model materialization for 'smb' rows; null for 'active_target_tables' - (a cross-resource distinct count with no single materialization). - - name: target_name - description: dbt target name (target.name) of the invocations contributing the rows. - - name: quantity - description: Metered quantity for this detail cell. - tests: - - not_null + Detail (base) grain for consumption: UTC day x meter x materialization x + target_name. fct_dbt__consumption_daily is a strict roll-up of this model. + For 'smb' rows, materialization is the model's materialization; for + 'active_target_tables' rows materialization is null and the count is + distinct nodes per deployment target that day. Only deployment-classed + invocations are counted. + columns: + - name: consumption_daily_detail_id + description: > + Surrogate key of the grain (hash of date_day, meter, materialization, + target_name). Stable BI join key. + tests: + - unique + - not_null + - name: date_day + description: UTC calendar day of the executions (via cast_to_utc_date()). + tests: + - not_null + - name: meter + description: "Consumption meter: 'smb' or 'active_target_tables'." + tests: + - not_null + - accepted_values: + arguments: + values: ['smb', 'active_target_tables'] + - name: materialization + description: > + Model materialization for 'smb' rows; null for 'active_target_tables' + (a cross-resource distinct count with no single materialization). + - name: target_name + description: dbt target name (target.name) of the invocations contributing the rows. + - name: quantity + description: Metered quantity for this detail cell. + tests: + - not_null diff --git a/models/fct_dbt__consumption_forecast.yml b/models/fct_dbt__consumption_forecast.yml index 0b0f030d..cb3881bd 100644 --- a/models/fct_dbt__consumption_forecast.yml +++ b/models/fct_dbt__consumption_forecast.yml @@ -1,54 +1,55 @@ version: 2 models: -- name: fct_dbt__consumption_forecast - description: > - Month-end consumption forecast: one row per billing_month x meter with - month-to-date usage, a weekday-aware projected month-end total, the projected - allowance-breach date, and % of allowance used. Projection averages the - trailing dbt_artifacts_run_rate_days (default 28) per ISO day-of-week - (including zero-build days) and sums the remaining calendar days' expected - values. Allowance resolves via get_smb_allowance() and applies to the 'smb' - meter only; with no plan/allowance var the breach columns are null but - projections still populate. Snowflake-only in v1 (uses Snowflake date - functions); multi-adapter support is planned. - columns: - - name: consumption_forecast_id - description: Surrogate key of the grain (hash of meter, billing_month). BI join key. - tests: - - unique - - not_null - - name: billing_month - description: First day of the calendar month (UTC). - tests: - - not_null - - name: meter - description: "Consumption meter (e.g. 'smb', 'active_target_tables')." - tests: - - not_null - - name: month_to_date_quantity - description: Sum of the meter's quantity from month start through as_of (today, clamped to month end). - tests: - - not_null - - name: allowance - description: Monthly SMB allowance (get_smb_allowance()); null for non-smb meters or when unset. - - name: daily_run_rate - description: Average daily quantity over the trailing run_rate_days window (includes zero-build days). - - name: forecast_month_end_quantity - description: month_to_date_quantity + sum of expected quantity over the remaining calendar days. - - name: forecast_exceeded_date + - name: fct_dbt__consumption_forecast description: > - The day the allowance is (or is projected to be) crossed: the actual day - the running total of elapsed daily quantity first reached the allowance if - it already has (historical), otherwise the first future day the projected - cumulative (MTD + weekday-aware expected) reaches it; null when there is no - allowance, the meter is not smb, or it never crosses. - - name: pct_of_allowance_used - description: month_to_date_quantity / allowance for the smb meter; null otherwise. - tests: - - dbt_artifacts.is_between: - min_value: 0 - - name: days_remaining - description: Calendar days remaining in the month after as_of (0 for fully-elapsed months). - - name: is_on_pace_to_exceed - description: True when forecast_month_end_quantity > allowance (smb meter); null when no allowance. + Month-end consumption forecast: one row per billing_month x meter with + month-to-date usage, a weekday-aware projected month-end total, the projected + allowance-breach date, and % of allowance used. Projection averages the + trailing dbt_artifacts_run_rate_days (default 28) per ISO day-of-week + (including zero-build days) and sums the remaining calendar days' expected + values. Allowance resolves via get_smb_allowance() and applies to the 'smb' + meter only; with no plan/allowance var the breach columns are null but + projections still populate. Snowflake-only in v1 (uses Snowflake date + functions); multi-adapter support is planned. + columns: + - name: consumption_forecast_id + description: Surrogate key of the grain (hash of meter, billing_month). BI join key. + tests: + - unique + - not_null + - name: billing_month + description: First day of the calendar month (UTC). + tests: + - not_null + - name: meter + description: "Consumption meter (e.g. 'smb', 'active_target_tables')." + tests: + - not_null + - name: month_to_date_quantity + description: Sum of the meter's quantity from month start through as_of (today, clamped to month end). + tests: + - not_null + - name: allowance + description: Monthly SMB allowance (get_smb_allowance()); null for non-smb meters or when unset. + - name: daily_run_rate + description: Average daily quantity over the trailing run_rate_days window (includes zero-build days). + - name: forecast_month_end_quantity + description: month_to_date_quantity + sum of expected quantity over the remaining calendar days. + - name: forecast_exceeded_date + description: > + The day the allowance is (or is projected to be) crossed: the actual day + the running total of elapsed daily quantity first reached the allowance if + it already has (historical), otherwise the first future day the projected + cumulative (MTD + weekday-aware expected) reaches it; null when there is no + allowance, the meter is not smb, or it never crosses. + - name: pct_of_allowance_used + description: month_to_date_quantity / allowance for the smb meter; null otherwise. + tests: + - dbt_artifacts.is_between: + arguments: + min_value: 0 + - name: days_remaining + description: Calendar days remaining in the month after as_of (0 for fully-elapsed months). + - name: is_on_pace_to_exceed + description: True when forecast_month_end_quantity > allowance (smb meter); null when no allowance. diff --git a/models/fct_dbt__dag_bottlenecks.yml b/models/fct_dbt__dag_bottlenecks.yml index 87b23279..8d03cd22 100644 --- a/models/fct_dbt__dag_bottlenecks.yml +++ b/models/fct_dbt__dag_bottlenecks.yml @@ -1,38 +1,40 @@ version: 2 models: -- name: fct_dbt__dag_bottlenecks - description: > - Pinch-point mart: per parent model x UTC day, the measured wall-clock time - its children spent waiting on it. Roll-up of fct_dbt__dag_bottlenecks_detail - over rows where the parent was the binding constraint and actually gated a - child (stall_seconds > 0). Measured gating, not graph theory; descendant - counts / blocking scores are out of scope for v1. Snowflake-only. - columns: - - name: dag_bottleneck_id - description: Surrogate key of the grain (hash of date_day, parent_node_id). BI join key. - tests: - - unique - - not_null - - name: date_day - description: UTC calendar day (via cast_to_utc_date()). - tests: - - not_null - - name: parent_node_id - description: The parent model node id that gated its children. - tests: - - not_null - - name: blocked_children - description: Distinct child models for which this parent was the binding (latest-completing) parent that day. - - name: total_stall_seconds - description: Sum of measured stall seconds attributed to this parent that day. - tests: - - dbt_artifacts.is_between: - min_value: 0 - - name: max_stall_seconds - description: Largest single measured stall attributed to this parent that day. - tests: - - dbt_artifacts.is_between: - min_value: 0 - - name: invocations_observed - description: Distinct invocations in which this parent gated at least one child that day. + - name: fct_dbt__dag_bottlenecks + description: > + Pinch-point mart: per parent model x UTC day, the measured wall-clock time + its children spent waiting on it. Roll-up of fct_dbt__dag_bottlenecks_detail + over rows where the parent was the binding constraint and actually gated a + child (stall_seconds > 0). Measured gating, not graph theory; descendant + counts / blocking scores are out of scope for v1. Snowflake-only. + columns: + - name: dag_bottleneck_id + description: Surrogate key of the grain (hash of date_day, parent_node_id). BI join key. + tests: + - unique + - not_null + - name: date_day + description: UTC calendar day (via cast_to_utc_date()). + tests: + - not_null + - name: parent_node_id + description: The parent model node id that gated its children. + tests: + - not_null + - name: blocked_children + description: Distinct child models for which this parent was the binding (latest-completing) parent that day. + - name: total_stall_seconds + description: Sum of measured stall seconds attributed to this parent that day. + tests: + - dbt_artifacts.is_between: + arguments: + min_value: 0 + - name: max_stall_seconds + description: Largest single measured stall attributed to this parent that day. + tests: + - dbt_artifacts.is_between: + arguments: + min_value: 0 + - name: invocations_observed + description: Distinct invocations in which this parent gated at least one child that day. diff --git a/models/fct_dbt__dag_bottlenecks_detail.yml b/models/fct_dbt__dag_bottlenecks_detail.yml index e545ae22..548dff32 100644 --- a/models/fct_dbt__dag_bottlenecks_detail.yml +++ b/models/fct_dbt__dag_bottlenecks_detail.yml @@ -1,39 +1,40 @@ version: 2 models: -- name: fct_dbt__dag_bottlenecks_detail - description: > - One row per (invocation, child model), with the binding parent (the - latest-completing model-parent in that invocation) and the measured stall = - child.compile_started_at - binding parent.query_completed_at, clamped to >= 0. - Drill-down behind fct_dbt__dag_bottlenecks. Model parents only (seed/source - parents excluded). Snowflake-only. - columns: - - name: dag_bottleneck_detail_id - description: Surrogate key of the grain (hash of command_invocation_id, child_node_id). BI join key. - tests: - - unique - - not_null - - name: command_invocation_id - description: The invocation the child executed in. - tests: - - not_null - - name: date_day - description: UTC calendar day of the invocation (via cast_to_utc_date()). - tests: - - not_null - - name: child_node_id - description: The child model whose start was (potentially) gated. - tests: - - not_null - - name: binding_parent_node_id - description: The model-parent with the latest query_completed_at among the child's parents in the invocation. - - name: parent_query_completed_at - description: query_completed_at of the binding parent. - - name: child_compile_started_at - description: compile_started_at of the child. - - name: stall_seconds - description: max(child_compile_started_at - parent_query_completed_at, 0) in seconds. - tests: - - dbt_artifacts.is_between: - min_value: 0 + - name: fct_dbt__dag_bottlenecks_detail + description: > + One row per (invocation, child model), with the binding parent (the + latest-completing model-parent in that invocation) and the measured stall = + child.compile_started_at - binding parent.query_completed_at, clamped to >= 0. + Drill-down behind fct_dbt__dag_bottlenecks. Model parents only (seed/source + parents excluded). Snowflake-only. + columns: + - name: dag_bottleneck_detail_id + description: Surrogate key of the grain (hash of command_invocation_id, child_node_id). BI join key. + tests: + - unique + - not_null + - name: command_invocation_id + description: The invocation the child executed in. + tests: + - not_null + - name: date_day + description: UTC calendar day of the invocation (via cast_to_utc_date()). + tests: + - not_null + - name: child_node_id + description: The child model whose start was (potentially) gated. + tests: + - not_null + - name: binding_parent_node_id + description: The model-parent with the latest query_completed_at among the child's parents in the invocation. + - name: parent_query_completed_at + description: query_completed_at of the binding parent. + - name: child_compile_started_at + description: compile_started_at of the child. + - name: stall_seconds + description: max(child_compile_started_at - parent_query_completed_at, 0) in seconds. + tests: + - dbt_artifacts.is_between: + arguments: + min_value: 0 diff --git a/models/fct_dbt__flaky_tests.yml b/models/fct_dbt__flaky_tests.yml index b85e3bbf..a1695472 100644 --- a/models/fct_dbt__flaky_tests.yml +++ b/models/fct_dbt__flaky_tests.yml @@ -1,38 +1,39 @@ version: 2 models: -- name: fct_dbt__flaky_tests - description: > - Flaky-test rollup: one row per test_node_id x month. flips are genuine - fail->pass transitions with no parent rebuild in between (from - fct_dbt__flaky_tests_detail); executions is all of the test's executions - that month. is_flaky = flips >= dbt_artifacts_flaky_min_flips (default 2). - Snowflake-only. - columns: - - name: flaky_test_id - description: Surrogate key of the grain (hash of test_node_id, month). BI join key. - tests: - - unique - - not_null - - name: test_node_id - description: The test's unique node id. - tests: - - not_null - - name: month - description: First day of the calendar month (UTC). - tests: - - not_null - - name: flips - description: Number of genuine flip events for the test that month. - - name: executions - description: Total executions of the test that month. - - name: flake_rate - description: flips / executions (0..1). - tests: - - dbt_artifacts.is_between: - min_value: 0 - max_value: 1 - - name: last_flip_at - description: run_started_at of the most recent flip's pass that month (null if no flips). - - name: is_flaky - description: True when flips >= dbt_artifacts_flaky_min_flips (default 2). + - name: fct_dbt__flaky_tests + description: > + Flaky-test rollup: one row per test_node_id x month. flips are genuine + fail->pass transitions with no parent rebuild in between (from + fct_dbt__flaky_tests_detail); executions is all of the test's executions + that month. is_flaky = flips >= dbt_artifacts_flaky_min_flips (default 2). + Snowflake-only. + columns: + - name: flaky_test_id + description: Surrogate key of the grain (hash of test_node_id, month). BI join key. + tests: + - unique + - not_null + - name: test_node_id + description: The test's unique node id. + tests: + - not_null + - name: month + description: First day of the calendar month (UTC). + tests: + - not_null + - name: flips + description: Number of genuine flip events for the test that month. + - name: executions + description: Total executions of the test that month. + - name: flake_rate + description: flips / executions (0..1). + tests: + - dbt_artifacts.is_between: + arguments: + min_value: 0 + max_value: 1 + - name: last_flip_at + description: run_started_at of the most recent flip's pass that month (null if no flips). + - name: is_flaky + description: True when flips >= dbt_artifacts_flaky_min_flips (default 2). diff --git a/models/fct_dbt__model_performance.yml b/models/fct_dbt__model_performance.yml index 5bd877f3..ffe09643 100644 --- a/models/fct_dbt__model_performance.yml +++ b/models/fct_dbt__model_performance.yml @@ -1,55 +1,56 @@ version: 2 models: -- name: fct_dbt__model_performance - description: > - Per-model daily runtime stats vs a trailing weekday-aware baseline: the - "why is the 6am job slow since Tuesday" mart. One row per UTC day x node_id - (models). Runtime stats use successful, non-full-refresh executions only. - baseline_runtime is the median of the same node's same-day-of-week - median_runtime over the trailing dbt_artifacts_run_rate_days (default 28) - days, excluding the current day. is_regressed requires ratio > - dbt_artifacts_regression_threshold (default 1.5) and a baseline of at least - dbt_artifacts_regression_min_samples (default 3) same-DOW days. - columns: - - name: model_performance_id - description: Surrogate key of the grain (hash of date_day, node_id). BI join key. - tests: - - unique - - not_null - - name: date_day - description: UTC calendar day (via cast_to_utc_date()). - tests: - - not_null - - name: node_id - description: The model's unique node id. - tests: - - not_null - - name: executions - description: Total executions of the model that day (all statuses). - - name: success_count - description: Successful executions that day. - - name: full_refresh_executions - description: Executions that were full refreshes (excluded from runtime stats and baseline). - - name: median_runtime - description: Median total_node_runtime over successful non-full-refresh executions that day. - - name: p95_runtime - description: 95th-percentile total_node_runtime over successful non-full-refresh executions that day. - - name: rows_affected_sum - description: Sum of rows_affected over successful non-full-refresh executions that day. - - name: baseline_runtime + - name: fct_dbt__model_performance description: > - Trailing weekday-aware baseline: median of the same node's same-DOW - median_runtime over the prior dbt_artifacts_run_rate_days days (excludes - today). Null until enough same-DOW history exists. - - name: baseline_sample_days - description: Number of same-DOW prior days contributing to the baseline. - - name: runtime_regression_ratio - description: median_runtime / baseline_runtime; null when baseline is null or 0. - tests: - - dbt_artifacts.is_between: - min_value: 0 - - name: is_regressed - description: > - True when runtime_regression_ratio > regression_threshold and - baseline_sample_days >= regression_min_samples; else false. + Per-model daily runtime stats vs a trailing weekday-aware baseline: the + "why is the 6am job slow since Tuesday" mart. One row per UTC day x node_id + (models). Runtime stats use successful, non-full-refresh executions only. + baseline_runtime is the median of the same node's same-day-of-week + median_runtime over the trailing dbt_artifacts_run_rate_days (default 28) + days, excluding the current day. is_regressed requires ratio > + dbt_artifacts_regression_threshold (default 1.5) and a baseline of at least + dbt_artifacts_regression_min_samples (default 3) same-DOW days. + columns: + - name: model_performance_id + description: Surrogate key of the grain (hash of date_day, node_id). BI join key. + tests: + - unique + - not_null + - name: date_day + description: UTC calendar day (via cast_to_utc_date()). + tests: + - not_null + - name: node_id + description: The model's unique node id. + tests: + - not_null + - name: executions + description: Total executions of the model that day (all statuses). + - name: success_count + description: Successful executions that day. + - name: full_refresh_executions + description: Executions that were full refreshes (excluded from runtime stats and baseline). + - name: median_runtime + description: Median total_node_runtime over successful non-full-refresh executions that day. + - name: p95_runtime + description: 95th-percentile total_node_runtime over successful non-full-refresh executions that day. + - name: rows_affected_sum + description: Sum of rows_affected over successful non-full-refresh executions that day. + - name: baseline_runtime + description: > + Trailing weekday-aware baseline: median of the same node's same-DOW + median_runtime over the prior dbt_artifacts_run_rate_days days (excludes + today). Null until enough same-DOW history exists. + - name: baseline_sample_days + description: Number of same-DOW prior days contributing to the baseline. + - name: runtime_regression_ratio + description: median_runtime / baseline_runtime; null when baseline is null or 0. + tests: + - dbt_artifacts.is_between: + arguments: + min_value: 0 + - name: is_regressed + description: > + True when runtime_regression_ratio > regression_threshold and + baseline_sample_days >= regression_min_samples; else false. diff --git a/models/fct_dbt__run_health_daily.yml b/models/fct_dbt__run_health_daily.yml index cc26d535..e4ea1771 100644 --- a/models/fct_dbt__run_health_daily.yml +++ b/models/fct_dbt__run_health_daily.yml @@ -1,45 +1,46 @@ version: 2 models: -- name: fct_dbt__run_health_daily - description: > - Run-health rollup: one row per UTC day across all invocations (no billing - classification). Node counts span models + seeds + snapshots + tests. - success = status in (success, pass); failure = (fail, failure); - error = (error); skip = (skipped, skip). success_rate = - successes / (successes + failures + errors). A failed invocation has >=1 - node in error or failure. - columns: - - name: date_day - description: UTC calendar day (via cast_to_utc_date() on invocation run_started_at). - tests: - - not_null - - unique - - name: invocations - description: Distinct command_invocation_ids that day. - - name: distinct_commands - description: Distinct dbt_command values that day. - - name: node_successes - description: Successful node executions (models + seeds + snapshots + tests). - - name: node_failures - description: Failed node executions (test fails). - - name: node_errors - description: Errored node executions. - - name: node_skips - description: Skipped node executions. - - name: failed_invocations - description: Invocations with >=1 node in error or failure. - - name: success_rate - description: successes / (successes + failures + errors); null on days with no such executions. - tests: - - dbt_artifacts.is_between: - min_value: 0 - max_value: 1 - - name: total_runtime_seconds - description: Sum of total_node_runtime across all node executions that day. - - name: max_node_runtime_seconds - description: Longest single node execution that day. - - name: first_run_started_at - description: Earliest node run_started_at that day. - - name: last_run_started_at - description: Latest node run_started_at that day. + - name: fct_dbt__run_health_daily + description: > + Run-health rollup: one row per UTC day across all invocations (no billing + classification). Node counts span models + seeds + snapshots + tests. + success = status in (success, pass); failure = (fail, failure); + error = (error); skip = (skipped, skip). success_rate = + successes / (successes + failures + errors). A failed invocation has >=1 + node in error or failure. + columns: + - name: date_day + description: UTC calendar day (via cast_to_utc_date() on invocation run_started_at). + tests: + - not_null + - unique + - name: invocations + description: Distinct command_invocation_ids that day. + - name: distinct_commands + description: Distinct dbt_command values that day. + - name: node_successes + description: Successful node executions (models + seeds + snapshots + tests). + - name: node_failures + description: Failed node executions (test fails). + - name: node_errors + description: Errored node executions. + - name: node_skips + description: Skipped node executions. + - name: failed_invocations + description: Invocations with >=1 node in error or failure. + - name: success_rate + description: successes / (successes + failures + errors); null on days with no such executions. + tests: + - dbt_artifacts.is_between: + arguments: + min_value: 0 + max_value: 1 + - name: total_runtime_seconds + description: Sum of total_node_runtime across all node executions that day. + - name: max_node_runtime_seconds + description: Longest single node execution that day. + - name: first_run_started_at + description: Earliest node run_started_at that day. + - name: last_run_started_at + description: Latest node run_started_at that day. diff --git a/models/fct_dbt__run_health_daily_detail.yml b/models/fct_dbt__run_health_daily_detail.yml index a7aa0f3c..d4126e3e 100644 --- a/models/fct_dbt__run_health_daily_detail.yml +++ b/models/fct_dbt__run_health_daily_detail.yml @@ -1,48 +1,49 @@ version: 2 models: -- name: fct_dbt__run_health_daily_detail - description: > - Run-health rollup at UTC day x target_name grain (companion to - fct_dbt__run_health_daily). Same status mapping and definitions; additive - count columns sum to the daily model across targets. - columns: - - name: run_health_daily_detail_id - description: Surrogate key of the grain (hash of date_day, target_name). BI join key. - tests: - - unique - - not_null - - name: date_day - description: UTC calendar day (via cast_to_utc_date()). - tests: - - not_null - - name: target_name - description: dbt target name (target.name) of the invocations. - - name: invocations - description: Distinct command_invocation_ids that day for the target. - - name: distinct_commands - description: Distinct dbt_command values that day for the target. - - name: node_successes - description: Successful node executions for the target. - - name: node_failures - description: Failed node executions for the target. - - name: node_errors - description: Errored node executions for the target. - - name: node_skips - description: Skipped node executions for the target. - - name: failed_invocations - description: Invocations with >=1 node in error or failure for the target. - - name: success_rate - description: successes / (successes + failures + errors) for the target. - tests: - - dbt_artifacts.is_between: - min_value: 0 - max_value: 1 - - name: total_runtime_seconds - description: Sum of total_node_runtime for the target that day. - - name: max_node_runtime_seconds - description: Longest single node execution for the target that day. - - name: first_run_started_at - description: Earliest node run_started_at for the target that day. - - name: last_run_started_at - description: Latest node run_started_at for the target that day. + - name: fct_dbt__run_health_daily_detail + description: > + Run-health rollup at UTC day x target_name grain (companion to + fct_dbt__run_health_daily). Same status mapping and definitions; additive + count columns sum to the daily model across targets. + columns: + - name: run_health_daily_detail_id + description: Surrogate key of the grain (hash of date_day, target_name). BI join key. + tests: + - unique + - not_null + - name: date_day + description: UTC calendar day (via cast_to_utc_date()). + tests: + - not_null + - name: target_name + description: dbt target name (target.name) of the invocations. + - name: invocations + description: Distinct command_invocation_ids that day for the target. + - name: distinct_commands + description: Distinct dbt_command values that day for the target. + - name: node_successes + description: Successful node executions for the target. + - name: node_failures + description: Failed node executions for the target. + - name: node_errors + description: Errored node executions for the target. + - name: node_skips + description: Skipped node executions for the target. + - name: failed_invocations + description: Invocations with >=1 node in error or failure for the target. + - name: success_rate + description: successes / (successes + failures + errors) for the target. + tests: + - dbt_artifacts.is_between: + arguments: + min_value: 0 + max_value: 1 + - name: total_runtime_seconds + description: Sum of total_node_runtime for the target that day. + - name: max_node_runtime_seconds + description: Longest single node execution for the target that day. + - name: first_run_started_at + description: Earliest node run_started_at for the target that day. + - name: last_run_started_at + description: Latest node run_started_at for the target that day. diff --git a/tox.ini b/tox.ini index 0b995699..8d8e4cc1 100644 --- a/tox.ini +++ b/tox.ini @@ -324,6 +324,16 @@ commands = dbt build --target bigquery --vars '"my_var": "my value"' echo "Warnings: This version will be removed from dbt_artifacts in a future release." +### Redshift ############################################################################################# +# Redshift integration test +[testenv:integration_test_redshift] +changedir = integration_test_project +deps = dbt-redshift~=1.7.0 +commands = + dbt clean + dbt deps + dbt build --target redshift --vars '"my_var": "my value"' + ### Databricks ############################################################################################# [testenv:integration_databricks] changedir = integration_test_project