diff --git a/.buildkite/scripts/run_dev_scripts_tests.sh b/.buildkite/scripts/run_dev_scripts_tests.sh index b838f4b08fc..eb484cf917c 100755 --- a/.buildkite/scripts/run_dev_scripts_tests.sh +++ b/.buildkite/scripts/run_dev_scripts_tests.sh @@ -60,7 +60,7 @@ setup_dummy_repo() { git -C "$tmpdir" add . git -C "$tmpdir" commit -q -m "Release flat_pkg 1.0.0" mkdir -p "${tmpdir}/.state" - git -C "$tmpdir" rev-parse --short HEAD > "${tmpdir}/.state/flat_pkg_1.0.0" + git -C "$tmpdir" rev-parse --short=10 HEAD > "${tmpdir}/.state/flat_pkg_1.0.0" printf 'name: flat_pkg\nversion: 1.1.0\n' > "${tmpdir}/packages/flat_pkg/manifest.yml" git -C "$tmpdir" add . @@ -70,7 +70,7 @@ setup_dummy_repo() { printf 'name: nested_pkg\nversion: 2.0.0\n' > "${tmpdir}/packages/group/nested_pkg/manifest.yml" git -C "$tmpdir" add . git -C "$tmpdir" commit -q -m "Release nested_pkg 2.0.0" - git -C "$tmpdir" rev-parse --short HEAD > "${tmpdir}/.state/nested_pkg_2.0.0" + git -C "$tmpdir" rev-parse --short=10 HEAD > "${tmpdir}/.state/nested_pkg_2.0.0" printf 'name: nested_pkg\nversion: 2.1.0\n' > "${tmpdir}/packages/group/nested_pkg/manifest.yml" git -C "$tmpdir" add . diff --git a/dev/scripts/get_release_commit.sh b/dev/scripts/get_release_commit.sh index ac1c275ac71..b3c93825845 100755 --- a/dev/scripts/get_release_commit.sh +++ b/dev/scripts/get_release_commit.sh @@ -61,10 +61,10 @@ COMMITS=() while IFS= read -r line; do [[ -n "$line" ]] && COMMITS+=("$line") done < <( - git log --oneline \ + git log --format='%H' \ -G "^version: ['\"]?${ESCAPED_VERSION}['\"]?[[:space:]]*$" \ -- "$MANIFEST" \ - | awk 'NF {print $1}' | sort -u + | sort -u ) # Among matching commits, keep only those that *added* (not removed) the version. @@ -82,4 +82,5 @@ if [[ -z "$FOUND" ]]; then exit 1 fi -echo "$FOUND" +# Fixed-length short SHA so callers/tests are not brittle to git's dynamic --abbrev. +git rev-parse --short=10 "$FOUND" diff --git a/packages/chargeback/_dev/build/docs/README.md b/packages/chargeback/_dev/build/docs/README.md index 43d18e9e420..96e9f4d487e 100755 --- a/packages/chargeback/_dev/build/docs/README.md +++ b/packages/chargeback/_dev/build/docs/README.md @@ -22,6 +22,7 @@ Chargeback costs are presented based on a configured rate and unit, used to conv **Monitoring cluster:** - Must be on Elasticsearch version **9.2.0+** due to the use of smart [ES|QL LOOKUP JOIN](https://www.elastic.co/docs/reference/query-languages/esql/esql-lookup-join) (conditional joins) in transforms and dashboard queries. +- Kibana version **9.4.0+** is required for package **0.5.0** and later. **Kibana 9.3 is not supported** — Billing and Usage dashboards fail to render (`No embeddable factory found for type: vis`) and lack GA support for ES|QL multi-select variable controls. - This is where the Chargeback integration should be installed. **Required integrations:** @@ -148,9 +149,9 @@ The integration creates eight transforms to aggregate cost and usage data: **Usage transforms** (from monitoring indices): 5. **`cluster_deployment_contribution`**: indexing, querying, and storage metrics per deployment/day. -6. **`cluster_datastream_contribution`**: same metrics split by data stream. +6. **`cluster_datastream_contribution`**: same metrics split by data stream. The usage ingest pipeline also sets `ds_type` and `ds_namespace` from the data stream name. 7. **`cluster_tier_contribution`**: same metrics split by data tier. -8. **`cluster_tier_and_ds_contribution`**: same metrics split by both tier and data stream. +8. **`cluster_tier_and_ds_contribution`**: same metrics split by both tier and data stream (includes `ds_type` and `ds_namespace`). These transforms produce lookup indices queried by the dashboards using ES|QL LOOKUP JOINs. @@ -188,6 +189,8 @@ Answers: *which data streams and tiers drive cost, and how efficiently are we us **Source:** `cluster_tier_contribution_lookup` and related usage lookups. Totals reflect the chargeable pool (allocatable data-tier ECU discounted by utilization) and will not equal the full invoice. ML, Kibana, snapshots, and data transfer are excluded. +**Controls:** deployment group, deployment name, data tier, full data stream name, **data stream type** (`ds_type`), and **data stream namespace** (`ds_namespace`). + Sections: - **Deployment cost allocation (usage-based)**: normalized cost per deployment split by data tier (usage-weighted). Shows which deployments consume the most of their chargeable pool across tiers. - **Datatiers / utilization**: provisioned capacity vs chargeable pool, utilization p95 per deployment. @@ -215,9 +218,35 @@ Do not expect the chargeable pool to equal the full deployment bill. Non-allocat When `node_stats` is missing for a deployment/day, utilization defaults to 100%. +## Shared deployments and data stream namespaces + +Document-level chargeback (for example by a field such as `application.id`) is not available: billing and monitoring allocate at deployment, tier, and data stream level. On a **shared deployment**, assign each team a unique [Fleet data stream namespace](https://www.elastic.co/docs/reference/fleet/data-streams) so ownership follows the naming scheme: + +```text +-- +``` + +Examples: `logs-nginx.access-team_a` → `ds_type=logs`, `ds_namespace=team_a`. + +Chargeback parses every usage `datastream` value into: + +| Field | Rule | +|-------|------| +| `ds_type` | First `-`-separated segment | +| `ds_namespace` | Last `-`-separated segment | +| (not a v1 control) | Middle segment(s) are the dataset | + +Names without a hyphen set both fields to `other` so panels keep working. Use the **Data stream type** and **Data stream namespace** controls on **Usage & Cost Allocation** to filter cost. A dataset control is intentionally out of scope for this release (high cardinality; weak team signal). The full data stream name control remains available separately. + +Guidance: + +1. Give each team a unique namespace (no hyphens; max 100 bytes). +2. Use separate agent policies or per-integration namespace overrides so data lands in distinct streams. +3. Filter or break down by `ds_namespace` (and optionally `ds_type`) on the Usage dashboard. + ## Deployment Groups -The integration supports organising deployments into logical groups using the `chargeback_group` tag on ESS Billing deployments. This enables cost allocation and filtering by team, project, or any organisational structure. +The integration supports organizing deployments into logical groups using the `chargeback_group` tag on ESS Billing deployments. This enables cost allocation and filtering by team, project, or any organizational structure. To assign a deployment to a chargeback group, add a tag in the Elastic Cloud console in the format: @@ -243,6 +272,19 @@ For more information, refer to the [Elastic documentation](https://www.elastic.c ## Upgrade Notes +### Upgrading to 0.5.1 + +1. Upgrade the Fleet package to **0.5.1** (Kibana **9.4.0+** required, same as 0.5.0). +2. Usage transforms that write through the usage pipeline are reinstalled with `fleet_transform_version: 0.5.1` so lookup documents pick up `ds_type` and `ds_namespace`. Reset those transforms if existing lookup docs lack the new fields. +3. The Usage & Cost Allocation dashboard gains **Data stream type** and **Data stream namespace** controls. Delete stale dashboard saved objects if duplicates appear after upgrade. + +### Upgrading to 0.5.0 + +1. Upgrade the Fleet package to **0.5.0**. +2. Upgrade Kibana to **9.4.0+** before installing 0.5.0. Kibana 9.3 cannot render the updated Billing and Usage dashboards. +3. Transforms are reinstalled with `fleet_transform_version: 0.5.0` (pipeline logic unchanged). No lookup index recreation is required. +4. The Billing and Usage dashboards are replaced with versions that use chained ES|QL variable controls instead of legacy options-list controls. Delete stale dashboard saved objects from **Stack Management > Saved Objects** if duplicates appear after upgrade. + ### Upgrading to 0.4.0 1. Upgrade the Fleet package to **0.4.0**. diff --git a/packages/chargeback/changelog.yml b/packages/chargeback/changelog.yml index 3b7f64721be..f52bf19d5ca 100644 --- a/packages/chargeback/changelog.yml +++ b/packages/chargeback/changelog.yml @@ -1,4 +1,14 @@ # newer versions go on top +- version: 0.5.1 + changes: + - description: "Parse data stream names into ds_type (first segment) and ds_namespace (last segment) on the usage path, with other fallback for non-conforming names. Add ES|QL multi-select controls on the Usage & Cost Allocation dashboard to filter by data stream type and namespace for shared-deployment FinOps. Bump transform pipeline references and fleet_transform_version to 0.5.1." + type: enhancement + link: https://github.com/elastic/elasticsearch-chargeback/issues/23 +- version: 0.5.0 + changes: + - description: "Replace legacy options-list dashboard controls with embedded ES|QL multi-select variable controls on the Billing Components Overview and Usage & Cost Allocation dashboards. Controls ship with a __chargeback_unfiltered__ sentinel default so fresh installs avoid empty multi-select param errors on Elasticsearch 9.4+. Panel queries use MV_INTERSECTS filters. Add a shared [Chargeback] Navigation links object across all three dashboards. Requires Kibana 9.4.0+. Bump transform pipeline references and fleet_transform_version to 0.5.0 (pipeline logic unchanged)." + type: enhancement + link: https://github.com/elastic/elasticsearch-chargeback/issues/8 - version: 0.4.0 changes: - description: "Split monolithic dashboard into three focused dashboards: '[Chargeback] Billing Components Overview' (full invoice by deployment group and billing component), '[Chargeback] Usage & Cost Allocation' (usage-based chargeable pool allocation by tier and data stream), and '[Chargeback] Configuration' (rate, weights, and date-window reference). Resolves elasticsearch-chargeback#8." diff --git a/packages/chargeback/docs/README.md b/packages/chargeback/docs/README.md index 43d18e9e420..96e9f4d487e 100644 --- a/packages/chargeback/docs/README.md +++ b/packages/chargeback/docs/README.md @@ -22,6 +22,7 @@ Chargeback costs are presented based on a configured rate and unit, used to conv **Monitoring cluster:** - Must be on Elasticsearch version **9.2.0+** due to the use of smart [ES|QL LOOKUP JOIN](https://www.elastic.co/docs/reference/query-languages/esql/esql-lookup-join) (conditional joins) in transforms and dashboard queries. +- Kibana version **9.4.0+** is required for package **0.5.0** and later. **Kibana 9.3 is not supported** — Billing and Usage dashboards fail to render (`No embeddable factory found for type: vis`) and lack GA support for ES|QL multi-select variable controls. - This is where the Chargeback integration should be installed. **Required integrations:** @@ -148,9 +149,9 @@ The integration creates eight transforms to aggregate cost and usage data: **Usage transforms** (from monitoring indices): 5. **`cluster_deployment_contribution`**: indexing, querying, and storage metrics per deployment/day. -6. **`cluster_datastream_contribution`**: same metrics split by data stream. +6. **`cluster_datastream_contribution`**: same metrics split by data stream. The usage ingest pipeline also sets `ds_type` and `ds_namespace` from the data stream name. 7. **`cluster_tier_contribution`**: same metrics split by data tier. -8. **`cluster_tier_and_ds_contribution`**: same metrics split by both tier and data stream. +8. **`cluster_tier_and_ds_contribution`**: same metrics split by both tier and data stream (includes `ds_type` and `ds_namespace`). These transforms produce lookup indices queried by the dashboards using ES|QL LOOKUP JOINs. @@ -188,6 +189,8 @@ Answers: *which data streams and tiers drive cost, and how efficiently are we us **Source:** `cluster_tier_contribution_lookup` and related usage lookups. Totals reflect the chargeable pool (allocatable data-tier ECU discounted by utilization) and will not equal the full invoice. ML, Kibana, snapshots, and data transfer are excluded. +**Controls:** deployment group, deployment name, data tier, full data stream name, **data stream type** (`ds_type`), and **data stream namespace** (`ds_namespace`). + Sections: - **Deployment cost allocation (usage-based)**: normalized cost per deployment split by data tier (usage-weighted). Shows which deployments consume the most of their chargeable pool across tiers. - **Datatiers / utilization**: provisioned capacity vs chargeable pool, utilization p95 per deployment. @@ -215,9 +218,35 @@ Do not expect the chargeable pool to equal the full deployment bill. Non-allocat When `node_stats` is missing for a deployment/day, utilization defaults to 100%. +## Shared deployments and data stream namespaces + +Document-level chargeback (for example by a field such as `application.id`) is not available: billing and monitoring allocate at deployment, tier, and data stream level. On a **shared deployment**, assign each team a unique [Fleet data stream namespace](https://www.elastic.co/docs/reference/fleet/data-streams) so ownership follows the naming scheme: + +```text +-- +``` + +Examples: `logs-nginx.access-team_a` → `ds_type=logs`, `ds_namespace=team_a`. + +Chargeback parses every usage `datastream` value into: + +| Field | Rule | +|-------|------| +| `ds_type` | First `-`-separated segment | +| `ds_namespace` | Last `-`-separated segment | +| (not a v1 control) | Middle segment(s) are the dataset | + +Names without a hyphen set both fields to `other` so panels keep working. Use the **Data stream type** and **Data stream namespace** controls on **Usage & Cost Allocation** to filter cost. A dataset control is intentionally out of scope for this release (high cardinality; weak team signal). The full data stream name control remains available separately. + +Guidance: + +1. Give each team a unique namespace (no hyphens; max 100 bytes). +2. Use separate agent policies or per-integration namespace overrides so data lands in distinct streams. +3. Filter or break down by `ds_namespace` (and optionally `ds_type`) on the Usage dashboard. + ## Deployment Groups -The integration supports organising deployments into logical groups using the `chargeback_group` tag on ESS Billing deployments. This enables cost allocation and filtering by team, project, or any organisational structure. +The integration supports organizing deployments into logical groups using the `chargeback_group` tag on ESS Billing deployments. This enables cost allocation and filtering by team, project, or any organizational structure. To assign a deployment to a chargeback group, add a tag in the Elastic Cloud console in the format: @@ -243,6 +272,19 @@ For more information, refer to the [Elastic documentation](https://www.elastic.c ## Upgrade Notes +### Upgrading to 0.5.1 + +1. Upgrade the Fleet package to **0.5.1** (Kibana **9.4.0+** required, same as 0.5.0). +2. Usage transforms that write through the usage pipeline are reinstalled with `fleet_transform_version: 0.5.1` so lookup documents pick up `ds_type` and `ds_namespace`. Reset those transforms if existing lookup docs lack the new fields. +3. The Usage & Cost Allocation dashboard gains **Data stream type** and **Data stream namespace** controls. Delete stale dashboard saved objects if duplicates appear after upgrade. + +### Upgrading to 0.5.0 + +1. Upgrade the Fleet package to **0.5.0**. +2. Upgrade Kibana to **9.4.0+** before installing 0.5.0. Kibana 9.3 cannot render the updated Billing and Usage dashboards. +3. Transforms are reinstalled with `fleet_transform_version: 0.5.0` (pipeline logic unchanged). No lookup index recreation is required. +4. The Billing and Usage dashboards are replaced with versions that use chained ES|QL variable controls instead of legacy options-list controls. Delete stale dashboard saved objects from **Stack Management > Saved Objects** if duplicates appear after upgrade. + ### Upgrading to 0.4.0 1. Upgrade the Fleet package to **0.4.0**. diff --git a/packages/chargeback/elasticsearch/ingest_pipeline/usage.yml b/packages/chargeback/elasticsearch/ingest_pipeline/usage.yml index 81ad97a5616..845efa95988 100644 --- a/packages/chargeback/elasticsearch/ingest_pipeline/usage.yml +++ b/packages/chargeback/elasticsearch/ingest_pipeline/usage.yml @@ -22,6 +22,22 @@ processors: ctx.composite_datastream_key = ctx.composite_key + '_' + ctx.datastream; } } + + // Parse Fleet data stream naming: -- + // first segment → ds_type; last segment → ds_namespace; else other + if (ctx.datastream != null && ctx.datastream != '') { + def parts = ctx.datastream.splitOnToken('-'); + if (parts.length >= 2) { + ctx.ds_type = parts[0]; + ctx.ds_namespace = parts[parts.length - 1]; + } else { + ctx.ds_type = 'other'; + ctx.ds_namespace = 'other'; + } + } else { + ctx.ds_type = 'other'; + ctx.ds_namespace = 'other'; + } - pipeline: description: '[Fleet] Global pipeline for all data streams' ignore_missing_pipeline: true diff --git a/packages/chargeback/elasticsearch/transform/billing_cluster_cost/transform.yml b/packages/chargeback/elasticsearch/transform/billing_cluster_cost/transform.yml index f7b7f8c4eac..083b209790b 100644 --- a/packages/chargeback/elasticsearch/transform/billing_cluster_cost/transform.yml +++ b/packages/chargeback/elasticsearch/transform/billing_cluster_cost/transform.yml @@ -30,7 +30,7 @@ source: emit(''); dest: index: billing_cluster_cost_lookup - pipeline: 0.4.0-billing + pipeline: 0.5.1-billing frequency: 60m sync: time: @@ -67,4 +67,4 @@ _meta: run_as_kibana_system: false # Bump this version to delete, reinstall, and restart the transform during package. # Version bump is needed if there is any code change in transform. - fleet_transform_version: 0.4.0 + fleet_transform_version: 0.5.1 diff --git a/packages/chargeback/elasticsearch/transform/billing_realized_pool/transform.yml b/packages/chargeback/elasticsearch/transform/billing_realized_pool/transform.yml index 2aa1134935b..23c79d29c26 100644 --- a/packages/chargeback/elasticsearch/transform/billing_realized_pool/transform.yml +++ b/packages/chargeback/elasticsearch/transform/billing_realized_pool/transform.yml @@ -55,7 +55,7 @@ source: emit(''); dest: index: billing_realized_pool_lookup - pipeline: 0.4.0-realized_pool + pipeline: 0.5.1-realized_pool frequency: 60m sync: time: @@ -92,4 +92,4 @@ settings: _meta: managed: true run_as_kibana_system: false - fleet_transform_version: 0.4.0 + fleet_transform_version: 0.5.1 diff --git a/packages/chargeback/elasticsearch/transform/chargeback_conf_lookup/transform.yml b/packages/chargeback/elasticsearch/transform/chargeback_conf_lookup/transform.yml index c4751a5c65e..c784274be02 100644 --- a/packages/chargeback/elasticsearch/transform/chargeback_conf_lookup/transform.yml +++ b/packages/chargeback/elasticsearch/transform/chargeback_conf_lookup/transform.yml @@ -57,7 +57,7 @@ source: source: "emit(50)" dest: index: chargeback_conf_lookup - pipeline: 0.4.0-chargeback_conf_lookup + pipeline: 0.5.1-chargeback_conf_lookup pivot: group_by: config_join_key: @@ -106,4 +106,4 @@ settings: _meta: managed: true run_as_kibana_system: false - fleet_transform_version: 0.4.0 + fleet_transform_version: 0.5.1 diff --git a/packages/chargeback/elasticsearch/transform/cluster_capacity_utilization/transform.yml b/packages/chargeback/elasticsearch/transform/cluster_capacity_utilization/transform.yml index 63d25a366d3..f4928fa9294 100644 --- a/packages/chargeback/elasticsearch/transform/cluster_capacity_utilization/transform.yml +++ b/packages/chargeback/elasticsearch/transform/cluster_capacity_utilization/transform.yml @@ -33,7 +33,7 @@ source: } dest: index: cluster_capacity_utilization_lookup - pipeline: 0.4.0-capacity_utilization + pipeline: 0.5.1-capacity_utilization frequency: 60m sync: time: @@ -65,4 +65,4 @@ settings: _meta: managed: true run_as_kibana_system: false - fleet_transform_version: 0.4.0 + fleet_transform_version: 0.5.1 diff --git a/packages/chargeback/elasticsearch/transform/cluster_datastream_contribution/fields/fields.yml b/packages/chargeback/elasticsearch/transform/cluster_datastream_contribution/fields/fields.yml index f489ab2bbf6..3eb864bb839 100644 --- a/packages/chargeback/elasticsearch/transform/cluster_datastream_contribution/fields/fields.yml +++ b/packages/chargeback/elasticsearch/transform/cluster_datastream_contribution/fields/fields.yml @@ -7,6 +7,12 @@ - name: datastream type: keyword description: The name of the data stream. +- name: ds_type + type: keyword + description: First segment of the data stream name (for example logs, metrics, traces). Set to other when the name does not contain a hyphen. +- name: ds_namespace + type: keyword + description: Last segment of the data stream name (team, environment, or cost-center label). Set to other when the name does not contain a hyphen. - name: datastream_sum_indexing_time type: long description: Total indexing time in milliseconds for the data stream. diff --git a/packages/chargeback/elasticsearch/transform/cluster_datastream_contribution/transform.yml b/packages/chargeback/elasticsearch/transform/cluster_datastream_contribution/transform.yml index aca2f29639f..f6a6a06c061 100644 --- a/packages/chargeback/elasticsearch/transform/cluster_datastream_contribution/transform.yml +++ b/packages/chargeback/elasticsearch/transform/cluster_datastream_contribution/transform.yml @@ -5,7 +5,7 @@ source: - "*:monitoring-indices*" dest: index: cluster_datastream_contribution_lookup - pipeline: 0.4.0-usage + pipeline: 0.5.1-usage frequency: 60m sync: time: @@ -61,4 +61,4 @@ _meta: run_as_kibana_system: false # Bump this version to delete, reinstall, and restart the transform during package. # Version bump is needed if there is any code change in transform. - fleet_transform_version: 0.4.0 + fleet_transform_version: 0.5.1 diff --git a/packages/chargeback/elasticsearch/transform/cluster_deployment_contribution/transform.yml b/packages/chargeback/elasticsearch/transform/cluster_deployment_contribution/transform.yml index 6e19d3e660c..86b31108f43 100644 --- a/packages/chargeback/elasticsearch/transform/cluster_deployment_contribution/transform.yml +++ b/packages/chargeback/elasticsearch/transform/cluster_deployment_contribution/transform.yml @@ -6,7 +6,7 @@ source: - "*:monitoring-indices*" # Same pattern on all remote clusters (CCS). dest: index: cluster_deployment_contribution_lookup - pipeline: 0.4.0-usage + pipeline: 0.5.1-usage frequency: 60m sync: time: @@ -43,4 +43,4 @@ _meta: run_as_kibana_system: false # Bump this version to delete, reinstall, and restart the transform during package. # Version bump is needed if there is any code change in transform. - fleet_transform_version: 0.4.0 \ No newline at end of file + fleet_transform_version: 0.5.1 \ No newline at end of file diff --git a/packages/chargeback/elasticsearch/transform/cluster_tier_and_ds_contribution/fields/fields.yml b/packages/chargeback/elasticsearch/transform/cluster_tier_and_ds_contribution/fields/fields.yml index 54cc1c6bb6c..51fef908649 100644 --- a/packages/chargeback/elasticsearch/transform/cluster_tier_and_ds_contribution/fields/fields.yml +++ b/packages/chargeback/elasticsearch/transform/cluster_tier_and_ds_contribution/fields/fields.yml @@ -10,6 +10,12 @@ - name: datastream type: keyword description: The name of the data stream. +- name: ds_type + type: keyword + description: First segment of the data stream name (for example logs, metrics, traces). Set to other when the name does not contain a hyphen. +- name: ds_namespace + type: keyword + description: Last segment of the data stream name (team, environment, or cost-center label). Set to other when the name does not contain a hyphen. - name: tier_and_datastream_sum_indexing_time type: long description: Total indexing time in milliseconds for the tier and data stream. diff --git a/packages/chargeback/elasticsearch/transform/cluster_tier_and_ds_contribution/transform.yml b/packages/chargeback/elasticsearch/transform/cluster_tier_and_ds_contribution/transform.yml index 056d88725c8..b9af79ac1ec 100644 --- a/packages/chargeback/elasticsearch/transform/cluster_tier_and_ds_contribution/transform.yml +++ b/packages/chargeback/elasticsearch/transform/cluster_tier_and_ds_contribution/transform.yml @@ -5,7 +5,7 @@ source: - "*:monitoring-indices*" dest: index: cluster_tier_and_datastream_contribution_lookup - pipeline: 0.4.0-usage + pipeline: 0.5.1-usage frequency: 60m sync: time: @@ -77,4 +77,4 @@ _meta: run_as_kibana_system: false # Bump this version to delete, reinstall, and restart the transform during package. # Version bump is needed if there is any code change in transform. - fleet_transform_version: 0.4.0 \ No newline at end of file + fleet_transform_version: 0.5.1 \ No newline at end of file diff --git a/packages/chargeback/elasticsearch/transform/cluster_tier_contribution/transform.yml b/packages/chargeback/elasticsearch/transform/cluster_tier_contribution/transform.yml index aa04c2a811d..e889793fbfd 100644 --- a/packages/chargeback/elasticsearch/transform/cluster_tier_contribution/transform.yml +++ b/packages/chargeback/elasticsearch/transform/cluster_tier_contribution/transform.yml @@ -5,7 +5,7 @@ source: - "*:monitoring-indices*" dest: index: cluster_tier_contribution_lookup - pipeline: 0.4.0-usage + pipeline: 0.5.1-usage frequency: 60m sync: time: @@ -59,4 +59,4 @@ _meta: run_as_kibana_system: false # Bump this version to delete, reinstall, and restart the transform during package. # Version bump is needed if there is any code change in transform. - fleet_transform_version: 0.4.0 \ No newline at end of file + fleet_transform_version: 0.5.1 \ No newline at end of file diff --git a/packages/chargeback/kibana/dashboard/chargeback-21588d0e-fb6a-4f76-ad2f-cd7b3d7a3d7c.json b/packages/chargeback/kibana/dashboard/chargeback-21588d0e-fb6a-4f76-ad2f-cd7b3d7a3d7c.json index 76c7fd817ec..caf5c1120de 100644 --- a/packages/chargeback/kibana/dashboard/chargeback-21588d0e-fb6a-4f76-ad2f-cd7b3d7a3d7c.json +++ b/packages/chargeback/kibana/dashboard/chargeback-21588d0e-fb6a-4f76-ad2f-cd7b3d7a3d7c.json @@ -1,62 +1,9 @@ { "attributes": { - "controlGroupInput": { - "chainingSystem": "HIERARCHICAL", - "controlStyle": "oneLine", - "ignoreParentSettingsJSON": { - "ignoreFilters": false, - "ignoreQuery": false, - "ignoreTimerange": false, - "ignoreValidations": false - }, - "panelsJSON": { - "2c7df183-8a05-4ee0-b7db-415e874636cd": { - "explicitInput": { - "dataViewId": "chargeback_integration", - "exclude": false, - "existsSelected": false, - "fieldName": "deployment_name", - "searchTechnique": "prefix", - "selectedOptions": [], - "sort": { - "by": "_count", - "direction": "desc" - }, - "title": "Deployment name", - "id": "2c7df183-8a05-4ee0-b7db-415e874636cd" - }, - "grow": false, - "order": 1, - "type": "optionsListControl", - "width": "medium" - }, - "b680ef70-4697-41e5-bf5f-e7e1e700ec3e": { - "explicitInput": { - "dataViewId": "chargeback_integration", - "exclude": false, - "existsSelected": false, - "fieldName": "deployment_group", - "searchTechnique": "prefix", - "selectedOptions": [], - "sort": { - "by": "_count", - "direction": "desc" - }, - "title": "Deployment group", - "id": "b680ef70-4697-41e5-bf5f-e7e1e700ec3e" - }, - "grow": false, - "order": 0, - "type": "optionsListControl", - "width": "medium" - } - }, - "showApplySelections": false - }, + "controlGroupInput": {}, "description": "Realized cost allocation per data tier and data stream. Shows utilization, top consumers, and time-series cost breakdowns.", "kibanaSavedObjectMeta": { "searchSourceJSON": { - "filter": [], "query": { "language": "kuery", "query": "" @@ -64,6 +11,8 @@ } }, "optionsJSON": { + "autoApplyFilters": true, + "hidePanelBorders": false, "hidePanelTitles": false, "syncColors": false, "syncCursor": true, @@ -73,46 +22,49 @@ "panelsJSON": [ { "embeddableConfig": { - "enhancements": {}, - "layout": "horizontal", - "links": [ - { - "destinationRefName": "link_ea0417fa-630d-c656-8f4b-a8d7b9916b29_dashboard", - "id": "ea0417fa-630d-c656-8f4b-a8d7b9916b29", - "label": "[Chargeback] Billing Components Overview", - "options": { - "openInNewTab": false, - "useCurrentDateRange": true, - "useCurrentFilters": true + "attributes": { + "layout": "horizontal", + "links": [ + { + "destinationRefName": "link_a11b22c3-d44e-455f-8660-778899001122_dashboard", + "id": "a11b22c3-d44e-455f-8660-778899001122", + "label": "[Chargeback] Billing Components Overview", + "options": { + "openInNewTab": false, + "useCurrentDateRange": true, + "useCurrentFilters": true + }, + "order": 0, + "type": "dashboardLink" }, - "order": 0, - "type": "dashboardLink" - }, - { - "destinationRefName": "link_dc802531-4211-c74b-d9db-b197b528bbc8_dashboard", - "id": "dc802531-4211-c74b-d9db-b197b528bbc8", - "label": "[Chargeback] Usage & Cost Allocation", - "options": { - "openInNewTab": false, - "useCurrentDateRange": true, - "useCurrentFilters": true + { + "destinationRefName": "link_b22c33d4-e55f-4660-9771-889900112233_dashboard", + "id": "b22c33d4-e55f-4660-9771-889900112233", + "label": "[Chargeback] Usage & Cost Allocation", + "options": { + "openInNewTab": false, + "useCurrentDateRange": true, + "useCurrentFilters": true + }, + "order": 1, + "type": "dashboardLink" }, - "order": 1, - "type": "dashboardLink" - }, - { - "destinationRefName": "link_4e06e36e-f5d7-981f-8554-57756d1c59b8_dashboard", - "id": "4e06e36e-f5d7-981f-8554-57756d1c59b8", - "label": "[Chargeback] Configuration", - "options": { - "openInNewTab": false, - "useCurrentDateRange": true, - "useCurrentFilters": true - }, - "order": 2, - "type": "dashboardLink" - } - ] + { + "destinationRefName": "link_c33d44e5-f660-4771-a882-990011223344_dashboard", + "id": "c33d44e5-f660-4771-a882-990011223344", + "label": "[Chargeback] Configuration", + "options": { + "openInNewTab": false, + "useCurrentDateRange": true, + "useCurrentFilters": true + }, + "order": 2, + "type": "dashboardLink" + } + ] + }, + "enhancements": {}, + "hide_title": true }, "gridData": { "h": 2, @@ -126,11 +78,138 @@ }, { "embeddableConfig": { - "enhancements": { - "dynamicActions": { - "events": [] - } - }, + "control_type": "VALUES_FROM_QUERY", + "esql_query": "FROM billing_realized_pool_lookup\n| WHERE deployment_group != \"\"\n| STATS COUNT(*) BY deployment_group\n| KEEP deployment_group\n| SORT deployment_group", + "selected_options": [ + "__chargeback_unfiltered__" + ], + "single_select": false, + "title": "Deployment group", + "variable_name": "dg_selected", + "variable_type": "multi_values" + }, + "gridData": { + "h": 2, + "i": "b36355dd-4157-4047-9f99-c6979674fd5a", + "w": 8, + "x": 0, + "y": 2 + }, + "panelIndex": "b36355dd-4157-4047-9f99-c6979674fd5a", + "type": "esql_control" + }, + { + "embeddableConfig": { + "control_type": "VALUES_FROM_QUERY", + "esql_query": "FROM billing_realized_pool_lookup\n| WHERE deployment_group != \"\"\n| STATS COUNT(*) BY deployment_name\n| KEEP deployment_name\n| SORT deployment_name", + "selected_options": [ + "__chargeback_unfiltered__" + ], + "single_select": false, + "title": "Deployment name", + "variable_name": "deployment_selected", + "variable_type": "multi_values" + }, + "gridData": { + "h": 2, + "i": "2900feae-7837-4c04-bb45-5c78ccc46b95", + "w": 8, + "x": 8, + "y": 2 + }, + "panelIndex": "2900feae-7837-4c04-bb45-5c78ccc46b95", + "type": "esql_control" + }, + { + "embeddableConfig": { + "control_type": "VALUES_FROM_QUERY", + "esql_query": "FROM cluster_tier_contribution_lookup\n| WHERE tier IS NOT NULL AND tier != \"unknown\"\n| STATS COUNT(*) BY tier\n| KEEP tier\n| SORT tier", + "selected_options": [ + "__chargeback_unfiltered__" + ], + "single_select": false, + "title": "Data tier", + "variable_name": "tier_selected", + "variable_type": "multi_values" + }, + "gridData": { + "h": 2, + "i": "ff353537-6404-47e4-8b77-f12580668ae9", + "w": 8, + "x": 16, + "y": 2 + }, + "panelIndex": "ff353537-6404-47e4-8b77-f12580668ae9", + "type": "esql_control" + }, + { + "embeddableConfig": { + "control_type": "VALUES_FROM_QUERY", + "esql_query": "FROM cluster_tier_and_datastream_contribution_lookup\n| WHERE datastream IS NOT NULL\n| STATS COUNT(*) BY datastream\n| KEEP datastream\n| SORT datastream", + "selected_options": [ + "__chargeback_unfiltered__" + ], + "single_select": false, + "title": "Data stream", + "variable_name": "datastream_selected", + "variable_type": "multi_values" + }, + "gridData": { + "h": 2, + "i": "2e6a43c7-b43b-46dd-b8cd-2da45eced9c7", + "w": 8, + "x": 24, + "y": 2 + }, + "panelIndex": "2e6a43c7-b43b-46dd-b8cd-2da45eced9c7", + "type": "esql_control" + }, + { + "embeddableConfig": { + "control_type": "VALUES_FROM_QUERY", + "esql_query": "FROM cluster_tier_and_datastream_contribution_lookup\n| WHERE ds_type IS NOT NULL\n| STATS COUNT(*) BY ds_type\n| KEEP ds_type\n| SORT ds_type", + "selected_options": [ + "__chargeback_unfiltered__" + ], + "single_select": false, + "title": "Data stream type", + "variable_name": "ds_type_selected", + "variable_type": "multi_values" + }, + "gridData": { + "h": 2, + "i": "a1c8e2f4-6b3d-4e9a-9c5f-1d2e3f4a5b6c", + "w": 8, + "x": 32, + "y": 2 + }, + "panelIndex": "a1c8e2f4-6b3d-4e9a-9c5f-1d2e3f4a5b6c", + "type": "esql_control" + }, + { + "embeddableConfig": { + "control_type": "VALUES_FROM_QUERY", + "esql_query": "FROM cluster_tier_and_datastream_contribution_lookup\n| WHERE ds_namespace IS NOT NULL\n| STATS COUNT(*) BY ds_namespace\n| KEEP ds_namespace\n| SORT ds_namespace", + "selected_options": [ + "__chargeback_unfiltered__" + ], + "single_select": false, + "title": "Data stream namespace", + "variable_name": "ds_namespace_selected", + "variable_type": "multi_values" + }, + "gridData": { + "h": 2, + "i": "b2d9f3e5-7c4e-5f0b-0d6a-2e3f4a5b6c7d", + "w": 8, + "x": 40, + "y": 2 + }, + "panelIndex": "b2d9f3e5-7c4e-5f0b-0d6a-2e3f4a5b6c7d", + "type": "esql_control" + }, + { + "embeddableConfig": { "savedVis": { "data": { "aggs": [], @@ -163,7 +242,7 @@ "y": 0 }, "panelIndex": "eca9d216-830e-4509-88c7-66b1dd354c71", - "type": "visualization" + "type": "legacy_vis" }, { "embeddableConfig": { @@ -209,7 +288,7 @@ }, { "columnId": "deployment", - "customLabel": false, + "customLabel": true, "fieldName": "deployment", "label": "Deployment", "meta": { @@ -237,9 +316,9 @@ } } ], - "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "index": "billing-realized-pool-dv", "query": { - "esql": "FROM billing_realized_pool_lookup\n| WHERE @timestamp IS NOT NULL AND deployment_name IS NOT NULL\n| EVAL ts = @timestamp, deployment = CASE(deployment_id IS NULL, deployment_name, CONCAT(deployment_name, \" (\", LEFT(deployment_id, 6), \")\"))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON ts >= conf_start_date AND ts <= conf_end_date\n| EVAL mem_w = COALESCE(TO_DOUBLE(conf_utilization_memory_weight), 70.0), disk_w = COALESCE(TO_DOUBLE(conf_utilization_storage_weight), 30.0)\n| EVAL heap_p = COALESCE(heap_used_pct_p95, 100) / 100, disk_p = COALESCE(disk_used_pct_p95, 100) / 100\n| EVAL util_score = (mem_w * heap_p + disk_w * disk_p) / (mem_w + disk_w), chargeable_pool = data_tier_capacity_ecu * COALESCE(util_score, 1.0), rate = COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0)\n| STATS `Normalized cost` = SUM(chargeable_pool * rate) BY ts, deployment\n| RENAME ts AS `@timestamp`\n| SORT `@timestamp` ASC\n| LIMIT 5000" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| WHERE @timestamp IS NOT NULL AND deployment_name IS NOT NULL\n| EVAL ts = @timestamp, deployment = CASE(deployment_id IS NULL, deployment_name, CONCAT(deployment_name, \" (\", LEFT(deployment_id, 6), \")\"))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON ts >= conf_start_date AND ts <= conf_end_date\n| EVAL mem_w = COALESCE(TO_DOUBLE(conf_utilization_memory_weight), 70.0), disk_w = COALESCE(TO_DOUBLE(conf_utilization_storage_weight), 30.0)\n| EVAL heap_p = COALESCE(heap_used_pct_p95, 100) / 100, disk_p = COALESCE(disk_used_pct_p95, 100) / 100\n| EVAL util_score = (mem_w * heap_p + disk_w * disk_p) / (mem_w + disk_w), chargeable_pool = data_tier_capacity_ecu * COALESCE(util_score, 1.0), rate = COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0)\n| STATS `Normalized cost` = SUM(chargeable_pool * rate) BY ts, deployment\n| RENAME ts AS `@timestamp`\n| SORT `@timestamp` ASC\n| LIMIT 5000" }, "timeField": "@timestamp" } @@ -249,7 +328,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_realized_pool_lookup\n| WHERE @timestamp IS NOT NULL AND deployment_name IS NOT NULL\n| EVAL ts = @timestamp, deployment = CASE(deployment_id IS NULL, deployment_name, CONCAT(deployment_name, \" (\", LEFT(deployment_id, 6), \")\"))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON ts >= conf_start_date AND ts <= conf_end_date\n| EVAL mem_w = COALESCE(TO_DOUBLE(conf_utilization_memory_weight), 70.0), disk_w = COALESCE(TO_DOUBLE(conf_utilization_storage_weight), 30.0)\n| EVAL heap_p = COALESCE(heap_used_pct_p95, 100) / 100, disk_p = COALESCE(disk_used_pct_p95, 100) / 100\n| EVAL util_score = (mem_w * heap_p + disk_w * disk_p) / (mem_w + disk_w), chargeable_pool = data_tier_capacity_ecu * COALESCE(util_score, 1.0), rate = COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0)\n| STATS `Normalized cost` = SUM(chargeable_pool * rate) BY ts, deployment\n| RENAME ts AS `@timestamp`\n| SORT `@timestamp` ASC\n| LIMIT 5000" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| WHERE @timestamp IS NOT NULL AND deployment_name IS NOT NULL\n| EVAL ts = @timestamp, deployment = CASE(deployment_id IS NULL, deployment_name, CONCAT(deployment_name, \" (\", LEFT(deployment_id, 6), \")\"))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON ts >= conf_start_date AND ts <= conf_end_date\n| EVAL mem_w = COALESCE(TO_DOUBLE(conf_utilization_memory_weight), 70.0), disk_w = COALESCE(TO_DOUBLE(conf_utilization_storage_weight), 30.0)\n| EVAL heap_p = COALESCE(heap_used_pct_p95, 100) / 100, disk_p = COALESCE(disk_used_pct_p95, 100) / 100\n| EVAL util_score = (mem_w * heap_p + disk_w * disk_p) / (mem_w + disk_w), chargeable_pool = data_tier_capacity_ecu * COALESCE(util_score, 1.0), rate = COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0)\n| STATS `Normalized cost` = SUM(chargeable_pool * rate) BY ts, deployment\n| RENAME ts AS `@timestamp`\n| SORT `@timestamp` ASC\n| LIMIT 5000" }, "visualization": { "axisTitlesVisibilitySettings": { @@ -296,7 +375,6 @@ "layerId": "28174526-fb0d-4645-a2b7-f5101203fb4e", "layerType": "data", "seriesType": "bar_stacked", - "splitAccessor": "deployment", "splitAccessors": [ "deployment" ], @@ -316,7 +394,7 @@ "valueLabels": "hide" } }, - "title": "Realized Pool per Deployment", + "title": "", "version": 2, "visualizationType": "lnsXY" }, @@ -333,7 +411,7 @@ "y": 3 }, "panelIndex": "0c666274-4041-44ca-8a1e-7b4affafeb5d", - "type": "lens" + "type": "vis" }, { "embeddableConfig": { @@ -397,9 +475,9 @@ } } ], - "index": "tier-breakdown-dv", + "index": "cluster-tier-contribution-dv", "query": { - "esql": "FROM cluster_tier_contribution_lookup\n| WHERE composite_key IS NOT NULL AND tier IS NOT NULL AND tier != \"unknown\"\n| EVAL ts = @timestamp\n| LOOKUP JOIN billing_realized_pool_lookup ON composite_key\n| WHERE data_tier_capacity_ecu IS NOT NULL\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON ts >= conf_start_date AND ts <= conf_end_date\n| EVAL deployment = CASE(deployment_id IS NULL, deployment_name, CONCAT(deployment_name, \" (\", LEFT(deployment_id, 6), \")\"))\n| EVAL mem_w = COALESCE(TO_DOUBLE(conf_utilization_memory_weight), 70.0), disk_w = COALESCE(TO_DOUBLE(conf_utilization_storage_weight), 30.0)\n| EVAL heap_p = COALESCE(heap_used_pct_p95, 100) / 100, disk_p = COALESCE(disk_used_pct_p95, 100) / 100\n| EVAL util_score = (mem_w * heap_p + disk_w * disk_p) / (mem_w + disk_w), chargeable_pool = data_tier_capacity_ecu * util_score, tier_label = CASE(tier IS NULL OR tier == \"unknown\", \"Unclassified\", tier)\n| EVAL idx_w = COALESCE(TO_DOUBLE(conf_indexing_weight), 20.0), qry_w = COALESCE(TO_DOUBLE(conf_query_weight), 20.0), sto_w = COALESCE(TO_DOUBLE(conf_storage_weight), 40.0), rate = COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0)\n| EVAL indexing = CASE(deployment_sum_indexing_time > 0, TO_DOUBLE(tier_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool), querying = CASE(deployment_sum_query_time > 0, TO_DOUBLE(tier_sum_query_time) / deployment_sum_query_time * chargeable_pool), store = CASE(deployment_sum_store_size > 0, TO_DOUBLE(tier_sum_store_size) / deployment_sum_store_size * chargeable_pool), data_set = CASE(deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool)\n| EVAL storage = CASE(store == 0, data_set, store), total_weight_hot = sto_w + qry_w + idx_w, total_weight_cold = sto_w + qry_w\n| EVAL blended = CASE(tier_label == \"hot/content\", (storage * sto_w + querying * qry_w + indexing * idx_w) / total_weight_hot, (storage * sto_w + querying * qry_w) / total_weight_cold) * rate\n| STATS `Normalized cost` = SUM(blended) BY tier_label\n| WHERE tier_label IS NOT NULL\n| RENAME tier_label AS `Data Tier`\n| SORT `Normalized cost` DESC\n| LIMIT 50" + "esql": "FROM cluster_tier_contribution_lookup\n| WHERE composite_key IS NOT NULL AND tier IS NOT NULL AND tier != \"unknown\"\n| EVAL ts = @timestamp\n| LOOKUP JOIN billing_realized_pool_lookup ON composite_key\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE data_tier_capacity_ecu IS NOT NULL\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON ts >= conf_start_date AND ts <= conf_end_date\n| EVAL deployment = CASE(deployment_id IS NULL, deployment_name, CONCAT(deployment_name, \" (\", LEFT(deployment_id, 6), \")\"))\n| EVAL mem_w = COALESCE(TO_DOUBLE(conf_utilization_memory_weight), 70.0), disk_w = COALESCE(TO_DOUBLE(conf_utilization_storage_weight), 30.0)\n| EVAL heap_p = COALESCE(heap_used_pct_p95, 100) / 100, disk_p = COALESCE(disk_used_pct_p95, 100) / 100\n| EVAL util_score = (mem_w * heap_p + disk_w * disk_p) / (mem_w + disk_w), chargeable_pool = data_tier_capacity_ecu * util_score, tier_label = CASE(tier IS NULL OR tier == \"unknown\", \"Unclassified\", tier)\n| EVAL idx_w = COALESCE(TO_DOUBLE(conf_indexing_weight), 20.0), qry_w = COALESCE(TO_DOUBLE(conf_query_weight), 20.0), sto_w = COALESCE(TO_DOUBLE(conf_storage_weight), 40.0), rate = COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0)\n| EVAL indexing = CASE(deployment_sum_indexing_time > 0, TO_DOUBLE(tier_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool), querying = CASE(deployment_sum_query_time > 0, TO_DOUBLE(tier_sum_query_time) / deployment_sum_query_time * chargeable_pool), store = CASE(deployment_sum_store_size > 0, TO_DOUBLE(tier_sum_store_size) / deployment_sum_store_size * chargeable_pool), data_set = CASE(deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool)\n| EVAL storage = CASE(store == 0, data_set, store), total_weight_hot = sto_w + qry_w + idx_w, total_weight_cold = sto_w + qry_w\n| EVAL blended = CASE(tier_label == \"hot/content\", (storage * sto_w + querying * qry_w + indexing * idx_w) / total_weight_hot, (storage * sto_w + querying * qry_w) / total_weight_cold) * rate\n| STATS `Normalized cost` = SUM(blended) BY tier_label\n| WHERE tier_label IS NOT NULL\n| RENAME tier_label AS `Data Tier`\n| SORT `Normalized cost` DESC\n| LIMIT 50" }, "timeField": "@timestamp" } @@ -409,7 +487,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM cluster_tier_contribution_lookup\n| WHERE composite_key IS NOT NULL AND tier IS NOT NULL AND tier != \"unknown\"\n| EVAL ts = @timestamp\n| LOOKUP JOIN billing_realized_pool_lookup ON composite_key\n| WHERE data_tier_capacity_ecu IS NOT NULL\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON ts >= conf_start_date AND ts <= conf_end_date\n| EVAL deployment = CASE(deployment_id IS NULL, deployment_name, CONCAT(deployment_name, \" (\", LEFT(deployment_id, 6), \")\"))\n| EVAL mem_w = COALESCE(TO_DOUBLE(conf_utilization_memory_weight), 70.0), disk_w = COALESCE(TO_DOUBLE(conf_utilization_storage_weight), 30.0)\n| EVAL heap_p = COALESCE(heap_used_pct_p95, 100) / 100, disk_p = COALESCE(disk_used_pct_p95, 100) / 100\n| EVAL util_score = (mem_w * heap_p + disk_w * disk_p) / (mem_w + disk_w), chargeable_pool = data_tier_capacity_ecu * util_score, tier_label = CASE(tier IS NULL OR tier == \"unknown\", \"Unclassified\", tier)\n| EVAL idx_w = COALESCE(TO_DOUBLE(conf_indexing_weight), 20.0), qry_w = COALESCE(TO_DOUBLE(conf_query_weight), 20.0), sto_w = COALESCE(TO_DOUBLE(conf_storage_weight), 40.0), rate = COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0)\n| EVAL indexing = CASE(deployment_sum_indexing_time > 0, TO_DOUBLE(tier_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool), querying = CASE(deployment_sum_query_time > 0, TO_DOUBLE(tier_sum_query_time) / deployment_sum_query_time * chargeable_pool), store = CASE(deployment_sum_store_size > 0, TO_DOUBLE(tier_sum_store_size) / deployment_sum_store_size * chargeable_pool), data_set = CASE(deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool)\n| EVAL storage = CASE(store == 0, data_set, store), total_weight_hot = sto_w + qry_w + idx_w, total_weight_cold = sto_w + qry_w\n| EVAL blended = CASE(tier_label == \"hot/content\", (storage * sto_w + querying * qry_w + indexing * idx_w) / total_weight_hot, (storage * sto_w + querying * qry_w) / total_weight_cold) * rate\n| STATS `Normalized cost` = SUM(blended) BY tier_label\n| WHERE tier_label IS NOT NULL\n| RENAME tier_label AS `Data Tier`\n| SORT `Normalized cost` DESC\n| LIMIT 50" + "esql": "FROM cluster_tier_contribution_lookup\n| WHERE composite_key IS NOT NULL AND tier IS NOT NULL AND tier != \"unknown\"\n| EVAL ts = @timestamp\n| LOOKUP JOIN billing_realized_pool_lookup ON composite_key\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE data_tier_capacity_ecu IS NOT NULL\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON ts >= conf_start_date AND ts <= conf_end_date\n| EVAL deployment = CASE(deployment_id IS NULL, deployment_name, CONCAT(deployment_name, \" (\", LEFT(deployment_id, 6), \")\"))\n| EVAL mem_w = COALESCE(TO_DOUBLE(conf_utilization_memory_weight), 70.0), disk_w = COALESCE(TO_DOUBLE(conf_utilization_storage_weight), 30.0)\n| EVAL heap_p = COALESCE(heap_used_pct_p95, 100) / 100, disk_p = COALESCE(disk_used_pct_p95, 100) / 100\n| EVAL util_score = (mem_w * heap_p + disk_w * disk_p) / (mem_w + disk_w), chargeable_pool = data_tier_capacity_ecu * util_score, tier_label = CASE(tier IS NULL OR tier == \"unknown\", \"Unclassified\", tier)\n| EVAL idx_w = COALESCE(TO_DOUBLE(conf_indexing_weight), 20.0), qry_w = COALESCE(TO_DOUBLE(conf_query_weight), 20.0), sto_w = COALESCE(TO_DOUBLE(conf_storage_weight), 40.0), rate = COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0)\n| EVAL indexing = CASE(deployment_sum_indexing_time > 0, TO_DOUBLE(tier_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool), querying = CASE(deployment_sum_query_time > 0, TO_DOUBLE(tier_sum_query_time) / deployment_sum_query_time * chargeable_pool), store = CASE(deployment_sum_store_size > 0, TO_DOUBLE(tier_sum_store_size) / deployment_sum_store_size * chargeable_pool), data_set = CASE(deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool)\n| EVAL storage = CASE(store == 0, data_set, store), total_weight_hot = sto_w + qry_w + idx_w, total_weight_cold = sto_w + qry_w\n| EVAL blended = CASE(tier_label == \"hot/content\", (storage * sto_w + querying * qry_w + indexing * idx_w) / total_weight_hot, (storage * sto_w + querying * qry_w) / total_weight_cold) * rate\n| STATS `Normalized cost` = SUM(blended) BY tier_label\n| WHERE tier_label IS NOT NULL\n| RENAME tier_label AS `Data Tier`\n| SORT `Normalized cost` DESC\n| LIMIT 50" }, "visualization": { "columns": [ @@ -429,12 +507,8 @@ "layerType": "data" } }, - "title": "Chargeable Pool by Data Tier", + "title": "", "version": 2, - "visualization": { - "layerId": "tier-breakdown-dv", - "layerType": "data" - }, "visualizationType": "lnsDatatable" }, "description": "Full deployment bill: the sum of ALL billing SKUs (data tiers, ML, Kibana, snapshots, data transfer and on-prem nodes) converted at the configured rate. This is the highest-level total. Tier and data-stream panels show only the allocatable data-tier pool, so they sum to less than this figure.", @@ -450,7 +524,7 @@ "y": 3 }, "panelIndex": "bb5dac10-896d-4312-a0db-4aa663d0d88b", - "type": "lens" + "type": "vis" }, { "embeddableConfig": { @@ -571,9 +645,9 @@ } } ], - "index": "chargeback_integration", + "index": "billing-realized-pool-dv", "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), tier_label = CASE(tier IS NULL OR tier == \"unknown\", \"Unclassified\", tier)\n| EVAL indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate), querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate), data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool), store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_sum_store_size) / deployment_sum_store_size * chargeable_pool), storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate), total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight, total_weight_cold = conf_storage_weight + conf_query_weight, blended = CASE (tier_label == \"hot/content\", ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight) + (indexing / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_indexing_weight)) / total_weight_hot, ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight)) / total_weight_cold) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS Indexing = SUM(indexing), Querying = SUM(querying), Storage = SUM(storage), Blended = SUM(blended) BY tier_label\n| WHERE Blended > 0\n| RENAME tier_label AS `Data Tier`\n| SORT Blended DESC" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), tier_label = CASE(tier IS NULL OR tier == \"unknown\", \"Unclassified\", tier)\n| EVAL indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate), querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate), data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool), store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_sum_store_size) / deployment_sum_store_size * chargeable_pool), storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate), total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight, total_weight_cold = conf_storage_weight + conf_query_weight, blended = CASE (tier_label == \"hot/content\", ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight) + (indexing / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_indexing_weight)) / total_weight_hot, ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight)) / total_weight_cold) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS Indexing = SUM(indexing), Querying = SUM(querying), Storage = SUM(storage), Blended = SUM(blended) BY tier_label\n| WHERE Blended > 0\n| RENAME tier_label AS `Data Tier`\n| SORT Blended DESC" }, "timeField": "@timestamp" } @@ -583,7 +657,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), tier_label = CASE(tier IS NULL OR tier == \"unknown\", \"Unclassified\", tier)\n| EVAL indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate), querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate), data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool), store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_sum_store_size) / deployment_sum_store_size * chargeable_pool), storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate), total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight, total_weight_cold = conf_storage_weight + conf_query_weight, blended = CASE (tier_label == \"hot/content\", ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight) + (indexing / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_indexing_weight)) / total_weight_hot, ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight)) / total_weight_cold) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS Indexing = SUM(indexing), Querying = SUM(querying), Storage = SUM(storage), Blended = SUM(blended) BY tier_label\n| WHERE Blended > 0\n| RENAME tier_label AS `Data Tier`\n| SORT Blended DESC" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), tier_label = CASE(tier IS NULL OR tier == \"unknown\", \"Unclassified\", tier)\n| EVAL indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate), querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate), data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool), store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_sum_store_size) / deployment_sum_store_size * chargeable_pool), storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate), total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight, total_weight_cold = conf_storage_weight + conf_query_weight, blended = CASE (tier_label == \"hot/content\", ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight) + (indexing / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_indexing_weight)) / total_weight_hot, ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight)) / total_weight_cold) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS Indexing = SUM(indexing), Querying = SUM(querying), Storage = SUM(storage), Blended = SUM(blended) BY tier_label\n| WHERE Blended > 0\n| RENAME tier_label AS `Data Tier`\n| SORT Blended DESC" }, "visualization": { "columns": [ @@ -621,7 +695,7 @@ "layerType": "data" } }, - "title": "Workload Mix by Data Tier", + "title": "", "version": 2, "visualizationType": "lnsDatatable" }, @@ -638,7 +712,7 @@ "y": 3 }, "panelIndex": "b6c32121-c93d-4a4c-a850-7f486a476d81", - "type": "lens" + "type": "vis" }, { "embeddableConfig": { @@ -674,7 +748,7 @@ "y": 13 }, "panelIndex": "41c6c963-16be-4b21-b59c-3c123f84ad57", - "type": "visualization" + "type": "legacy_vis" }, { "embeddableConfig": { @@ -710,7 +784,7 @@ "y": 13 }, "panelIndex": "1ad2d1fd-04c2-4d2b-ad5d-15ffb0176cc1", - "type": "visualization" + "type": "legacy_vis" }, { "embeddableConfig": { @@ -746,7 +820,7 @@ "y": 13 }, "panelIndex": "2753c162-7420-4b8d-971f-4bda81d7be2b", - "type": "visualization" + "type": "legacy_vis" }, { "embeddableConfig": { @@ -782,16 +856,37 @@ "y": 13 }, "panelIndex": "c83bffe6-c501-430a-8769-a964b240acf5", - "type": "visualization" + "type": "legacy_vis" }, { "embeddableConfig": { "attributes": { "references": [], "state": { + "adHocDataViews": { + "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "managed": false, + "name": "billing_realized_pool_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_realized_pool_lookup", + "type": "esql" + } + }, "datasourceStates": { "textBased": { - "indexPatternRefs": [], + "indexPatternRefs": [ + { + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "timeField": "@timestamp", + "title": "billing_realized_pool_lookup" + } + ], "layers": { "0f6d5278-b325-49bc-9eb7-2febd2a5de0f": { "columns": [ @@ -825,9 +920,9 @@ } } ], - "index": "chargeback_integration", + "index": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_indexing = sum(indexing)\n BY \n datastream\n| WHERE agg_indexing > 0\n| SORT agg_indexing DESC\n| Limit 20" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_and_datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_indexing = sum(indexing)\n BY \n datastream\n| WHERE agg_indexing > 0\n| SORT agg_indexing DESC\n| Limit 20" }, "timeField": "@timestamp" } @@ -837,7 +932,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_indexing = sum(indexing)\n BY \n datastream\n| WHERE agg_indexing > 0\n| SORT agg_indexing DESC\n| Limit 20" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_and_datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_indexing = sum(indexing)\n BY \n datastream\n| WHERE agg_indexing > 0\n| SORT agg_indexing DESC\n| Limit 20" }, "visualization": { "layers": [ @@ -880,10 +975,11 @@ "shape": "donut" } }, - "title": "Top 20 Data Streams \u2014 Indexing Cost", + "title": "", "version": 2, "visualizationType": "lnsPie" }, + "drilldowns": [], "title": "Top 20 Data Streams \u2014 Indexing Cost" }, "gridData": { @@ -895,16 +991,37 @@ "y": 16 }, "panelIndex": "b349f8c6-846f-40d8-83e7-0b8365a7988e", - "type": "lens" + "type": "vis" }, { "embeddableConfig": { "attributes": { "references": [], "state": { + "adHocDataViews": { + "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "managed": false, + "name": "billing_realized_pool_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_realized_pool_lookup", + "type": "esql" + } + }, "datasourceStates": { "textBased": { - "indexPatternRefs": [], + "indexPatternRefs": [ + { + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "timeField": "@timestamp", + "title": "billing_realized_pool_lookup" + } + ], "layers": { "3b03b6a8-35ae-4554-8bd6-a887d8b36748": { "columns": [ @@ -938,9 +1055,9 @@ } } ], - "index": "chargeback_integration", + "index": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_querying = sum(querying)\n BY \n datastream\n| WHERE agg_querying > 0\n| SORT agg_querying DESC\n| Limit 20" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_and_datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_querying = sum(querying)\n BY \n datastream\n| WHERE agg_querying > 0\n| SORT agg_querying DESC\n| Limit 20" }, "timeField": "@timestamp" } @@ -950,7 +1067,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_querying = sum(querying)\n BY \n datastream\n| WHERE agg_querying > 0\n| SORT agg_querying DESC\n| Limit 20" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_and_datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_querying = sum(querying)\n BY \n datastream\n| WHERE agg_querying > 0\n| SORT agg_querying DESC\n| Limit 20" }, "visualization": { "layers": [ @@ -993,10 +1110,11 @@ "shape": "donut" } }, - "title": "Top 20 Data Streams \u2014 Query Cost", + "title": "", "version": 2, "visualizationType": "lnsPie" }, + "drilldowns": [], "title": "Top 20 Data Streams \u2014 Query Cost" }, "gridData": { @@ -1008,16 +1126,37 @@ "y": 16 }, "panelIndex": "cd86b33a-2af1-4dbc-b296-f3e5a3ee5970", - "type": "lens" + "type": "vis" }, { "embeddableConfig": { "attributes": { "references": [], "state": { + "adHocDataViews": { + "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "managed": false, + "name": "billing_realized_pool_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_realized_pool_lookup", + "type": "esql" + } + }, "datasourceStates": { "textBased": { - "indexPatternRefs": [], + "indexPatternRefs": [ + { + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "timeField": "@timestamp", + "title": "billing_realized_pool_lookup" + } + ], "layers": { "2813628b-2a20-484c-9a19-11955d86e9ff": { "columns": [ @@ -1051,9 +1190,9 @@ } } ], - "index": "chargeback_integration", + "index": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_storage = sum(storage)\n BY \n datastream\n| WHERE agg_storage > 0\n| SORT agg_storage DESC\n| limit 20" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_storage = sum(storage)\n BY \n datastream\n| WHERE agg_storage > 0\n| SORT agg_storage DESC\n| limit 20" }, "timeField": "@timestamp" } @@ -1063,7 +1202,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_storage = sum(storage)\n BY \n datastream\n| WHERE agg_storage > 0\n| SORT agg_storage DESC\n| limit 20" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_storage = sum(storage)\n BY \n datastream\n| WHERE agg_storage > 0\n| SORT agg_storage DESC\n| limit 20" }, "visualization": { "layers": [ @@ -1106,10 +1245,11 @@ "shape": "donut" } }, - "title": "Top 20 Data Streams \u2014 Storage Cost", + "title": "", "version": 2, "visualizationType": "lnsPie" }, + "drilldowns": [], "title": "Top 20 Data Streams \u2014 Storage Cost" }, "gridData": { @@ -1121,25 +1261,46 @@ "y": 16 }, "panelIndex": "c3afc6bc-09c4-4f60-a038-163d535a7d32", - "type": "lens" + "type": "vis" }, { "embeddableConfig": { "attributes": { "references": [], "state": { + "adHocDataViews": { + "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "managed": false, + "name": "billing_realized_pool_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_realized_pool_lookup", + "type": "esql" + } + }, "datasourceStates": { "textBased": { - "indexPatternRefs": [], + "indexPatternRefs": [ + { + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "timeField": "@timestamp", + "title": "billing_realized_pool_lookup" + } + ], "layers": { "ee58e17d-eba7-48a7-bae8-2df3607d9653": { "columns": [ { "columnId": "agg_blended", - "customLabel": false, + "customLabel": true, "fieldName": "agg_blended", "inMetricDimension": true, - "label": "agg_blended", + "label": "Blended", "meta": { "esType": "double", "type": "number" @@ -1156,9 +1317,9 @@ } } ], - "index": "chargeback_integration", + "index": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_and_datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool),\n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_and_datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool),\n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n TO_DOUBLE(((storage * conf_storage_weight) + (querying * conf_query_weight) + (indexing * conf_indexing_weight))) / total_weight_hot,\n TO_DOUBLE(((storage * conf_storage_weight) + (querying * conf_query_weight))) / total_weight_cold\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS\n agg_blended = sum(blended)\n BY\n datastream\n| WHERE agg_blended > 0\n| SORT agg_blended DESC\n| limit 20" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_and_datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool),\n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_and_datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool),\n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n TO_DOUBLE(((storage * conf_storage_weight) + (querying * conf_query_weight) + (indexing * conf_indexing_weight))) / total_weight_hot,\n TO_DOUBLE(((storage * conf_storage_weight) + (querying * conf_query_weight))) / total_weight_cold\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS\n agg_blended = sum(blended)\n BY\n datastream\n| WHERE agg_blended > 0\n| SORT agg_blended DESC\n| limit 20" }, "timeField": "@timestamp" } @@ -1168,7 +1329,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_and_datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool),\n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_and_datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool),\n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n TO_DOUBLE(((storage * conf_storage_weight) + (querying * conf_query_weight) + (indexing * conf_indexing_weight))) / total_weight_hot,\n TO_DOUBLE(((storage * conf_storage_weight) + (querying * conf_query_weight))) / total_weight_cold\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS\n agg_blended = sum(blended)\n BY\n datastream\n| WHERE agg_blended > 0\n| SORT agg_blended DESC\n| limit 20" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_and_datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool),\n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_and_datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool),\n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n TO_DOUBLE(((storage * conf_storage_weight) + (querying * conf_query_weight) + (indexing * conf_indexing_weight))) / total_weight_hot,\n TO_DOUBLE(((storage * conf_storage_weight) + (querying * conf_query_weight))) / total_weight_cold\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS\n agg_blended = sum(blended)\n BY\n datastream\n| WHERE agg_blended > 0\n| SORT agg_blended DESC\n| limit 20" }, "visualization": { "layers": [ @@ -1194,6 +1355,7 @@ } ] }, + "emptySizeRatio": 0.3, "layerId": "ee58e17d-eba7-48a7-bae8-2df3607d9653", "layerType": "data", "legendDisplay": "default", @@ -1207,13 +1369,14 @@ ] } ], - "shape": "pie" + "shape": "donut" } }, - "title": "Blended Cost by Data Stream", + "title": "", "version": 2, "visualizationType": "lnsPie" }, + "drilldowns": [], "title": "Blended Cost by Data Stream" }, "gridData": { @@ -1225,16 +1388,37 @@ "y": 16 }, "panelIndex": "e57765c6-40be-41cf-b3d2-e65ee8a51b67", - "type": "lens" + "type": "vis" }, { "embeddableConfig": { "attributes": { "references": [], "state": { + "adHocDataViews": { + "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "managed": false, + "name": "billing_realized_pool_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_realized_pool_lookup", + "type": "esql" + } + }, "datasourceStates": { "textBased": { - "indexPatternRefs": [], + "indexPatternRefs": [ + { + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "timeField": "@timestamp", + "title": "billing_realized_pool_lookup" + } + ], "layers": { "23693fb0-43a5-4f95-bd77-d6a54e18e34a": { "allColumns": [ @@ -1382,9 +1566,9 @@ } } ], - "index": "chargeback_integration", + "index": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| EVAL\n indexing = CASE(deployment_sum_indexing_time > 0, TO_DOUBLE(tier_and_datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n querying = CASE(deployment_sum_query_time > 0, TO_DOUBLE(tier_and_datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n data_set = CASE(deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE(deployment_sum_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE(store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE(\n tier == \"hot/content\",\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight) + (indexing / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_indexing_weight)) / total_weight_hot,\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight)) / total_weight_cold\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS\n Indexing = round(sum(indexing),2),\n Querying = round(sum(querying),2),\n Storage = round(sum(storage),2),\n Blended = round(sum(blended),2)\n BY datastream, tier\n| SORT Blended DESC\n| RENAME tier as `Data tier`, datastream as `Data stream`\n| Keep `Data stream`, `Data tier`, Indexing, Querying, Storage, Blended\n| WHERE `Data stream` is not null" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| EVAL\n indexing = CASE(deployment_sum_indexing_time > 0, TO_DOUBLE(tier_and_datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n querying = CASE(deployment_sum_query_time > 0, TO_DOUBLE(tier_and_datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n data_set = CASE(deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE(deployment_sum_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE(store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE(\n tier == \"hot/content\",\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight) + (indexing / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_indexing_weight)) / total_weight_hot,\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight)) / total_weight_cold\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS\n Indexing = round(sum(indexing),2),\n Querying = round(sum(querying),2),\n Storage = round(sum(storage),2),\n Blended = round(sum(blended),2)\n BY datastream, tier\n| SORT Blended DESC\n| RENAME tier as `Data tier`, datastream as `Data stream`\n| Keep `Data stream`, `Data tier`, Indexing, Querying, Storage, Blended\n| WHERE `Data stream` is not null" }, "timeField": "@timestamp" } @@ -1394,7 +1578,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| EVAL\n indexing = CASE(deployment_sum_indexing_time > 0, TO_DOUBLE(tier_and_datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n querying = CASE(deployment_sum_query_time > 0, TO_DOUBLE(tier_and_datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n data_set = CASE(deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE(deployment_sum_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE(store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE(\n tier == \"hot/content\",\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight) + (indexing / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_indexing_weight)) / total_weight_hot,\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight)) / total_weight_cold\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS\n Indexing = round(sum(indexing),2),\n Querying = round(sum(querying),2),\n Storage = round(sum(storage),2),\n Blended = round(sum(blended),2)\n BY datastream, tier\n| SORT Blended DESC\n| RENAME tier as `Data tier`, datastream as `Data stream`\n| Keep `Data stream`, `Data tier`, Indexing, Querying, Storage, Blended\n| WHERE `Data stream` is not null" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| EVAL\n indexing = CASE(deployment_sum_indexing_time > 0, TO_DOUBLE(tier_and_datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n querying = CASE(deployment_sum_query_time > 0, TO_DOUBLE(tier_and_datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n data_set = CASE(deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE(deployment_sum_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE(store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE(\n tier == \"hot/content\",\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight) + (indexing / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_indexing_weight)) / total_weight_hot,\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight)) / total_weight_cold\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS\n Indexing = round(sum(indexing),2),\n Querying = round(sum(querying),2),\n Storage = round(sum(storage),2),\n Blended = round(sum(blended),2)\n BY datastream, tier\n| SORT Blended DESC\n| RENAME tier as `Data tier`, datastream as `Data stream`\n| Keep `Data stream`, `Data tier`, Indexing, Querying, Storage, Blended\n| WHERE `Data stream` is not null" }, "visualization": { "columns": [ @@ -1433,22 +1617,11 @@ "layerType": "data" } }, - "title": "Workload Breakdown by Data Tier", + "title": "", "version": 2, "visualizationType": "lnsDatatable" }, - "enhancements": { - "dynamicActions": { - "events": [] - } - }, - "filters": [], - "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| EVAL\n indexing = CASE(deployment_sum_indexing_time > 0, TO_DOUBLE(tier_and_datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n querying = CASE(deployment_sum_query_time > 0, TO_DOUBLE(tier_and_datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n data_set = CASE(deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE(deployment_sum_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE(store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE(\n tier == \"hot/content\",\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight) + (indexing / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_indexing_weight)) / total_weight_hot,\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight)) / total_weight_cold\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS\n Indexing = round(sum(indexing),2),\n Querying = round(sum(querying),2),\n Storage = round(sum(storage),2),\n Blended = round(sum(blended),2)\n BY datastream, tier\n| SORT Blended DESC\n| RENAME tier as `Data tier`, datastream as `Data stream`\n| Keep `Data stream`, `Data tier`, Indexing, Querying, Storage, Blended\n| WHERE `Data stream` is not null" - }, - "syncColors": false, - "syncCursor": true, - "syncTooltips": false, + "drilldowns": [], "title": "Workload Breakdown by Data Tier" }, "gridData": { @@ -1460,15 +1633,10 @@ "y": 31 }, "panelIndex": "7766db54-cadf-46f1-8417-ebe8267a3c23", - "type": "lens" + "type": "vis" }, { "embeddableConfig": { - "enhancements": { - "dynamicActions": { - "events": [] - } - }, "savedVis": { "data": { "aggs": [], @@ -1501,16 +1669,37 @@ "y": 0 }, "panelIndex": "a22d7ea0-bd5b-4372-a1bf-c3c44af10937", - "type": "visualization" + "type": "legacy_vis" }, { "embeddableConfig": { "attributes": { "references": [], "state": { + "adHocDataViews": { + "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "managed": false, + "name": "billing_realized_pool_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_realized_pool_lookup", + "type": "esql" + } + }, "datasourceStates": { "textBased": { - "indexPatternRefs": [], + "indexPatternRefs": [ + { + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "timeField": "@timestamp", + "title": "billing_realized_pool_lookup" + } + ], "layers": { "0ee24895-1496-4b23-96ec-ec09a8f49332": { "columns": [ @@ -1554,9 +1743,9 @@ } } ], - "index": "chargeback_integration", + "index": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_indexing = sum(indexing)\n BY \n @timestamp, \n datastream\n| WHERE agg_indexing > 0\n| SORT agg_indexing DESC" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_and_datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_indexing = sum(indexing)\n BY \n @timestamp, \n datastream\n| WHERE agg_indexing > 0\n| SORT agg_indexing DESC" }, "timeField": "@timestamp" } @@ -1566,7 +1755,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_indexing = sum(indexing)\n BY \n @timestamp, \n datastream\n| WHERE agg_indexing > 0\n| SORT agg_indexing DESC" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_and_datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_indexing = sum(indexing)\n BY \n @timestamp, \n datastream\n| WHERE agg_indexing > 0\n| SORT agg_indexing DESC" }, "visualization": { "axisTitlesVisibilitySettings": { @@ -1613,7 +1802,6 @@ "layerId": "0ee24895-1496-4b23-96ec-ec09a8f49332", "layerType": "data", "seriesType": "bar_stacked", - "splitAccessor": "datastream", "splitAccessors": [ "datastream" ], @@ -1634,10 +1822,11 @@ "valueLabels": "hide" } }, - "title": "Indexing Cost by Data Stream over Time", + "title": "", "version": 2, "visualizationType": "lnsXY" }, + "drilldowns": [], "title": "Indexing Cost by Data Stream over Time" }, "gridData": { @@ -1649,16 +1838,37 @@ "y": 6 }, "panelIndex": "a85aea2f-8e25-4bef-b09a-32f0a5cb33f9", - "type": "lens" + "type": "vis" }, { "embeddableConfig": { "attributes": { "references": [], "state": { + "adHocDataViews": { + "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "managed": false, + "name": "billing_realized_pool_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_realized_pool_lookup", + "type": "esql" + } + }, "datasourceStates": { "textBased": { - "indexPatternRefs": [], + "indexPatternRefs": [ + { + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "timeField": "@timestamp", + "title": "billing_realized_pool_lookup" + } + ], "layers": { "d24af063-ec41-4736-bf4e-70f1fe4da4cf": { "columns": [ @@ -1702,9 +1912,9 @@ } } ], - "index": "chargeback_integration", + "index": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n indexing = CASE (\n deployment_sum_indexing_time > 0,\n TO_DOUBLE(tier_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_indexing = sum(indexing)\n BY \n @timestamp,\n tier\n| WHERE agg_indexing > 0" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n indexing = CASE (\n deployment_sum_indexing_time > 0,\n TO_DOUBLE(tier_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_indexing = sum(indexing)\n BY \n @timestamp,\n tier\n| WHERE agg_indexing > 0" }, "timeField": "@timestamp" } @@ -1714,7 +1924,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n indexing = CASE (\n deployment_sum_indexing_time > 0,\n TO_DOUBLE(tier_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_indexing = sum(indexing)\n BY \n @timestamp,\n tier\n| WHERE agg_indexing > 0" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n indexing = CASE (\n deployment_sum_indexing_time > 0,\n TO_DOUBLE(tier_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_indexing = sum(indexing)\n BY \n @timestamp,\n tier\n| WHERE agg_indexing > 0" }, "visualization": { "axisTitlesVisibilitySettings": { @@ -1761,7 +1971,6 @@ "layerId": "d24af063-ec41-4736-bf4e-70f1fe4da4cf", "layerType": "data", "seriesType": "bar_stacked", - "splitAccessor": "tier", "splitAccessors": [ "tier" ], @@ -1782,10 +1991,11 @@ "valueLabels": "hide" } }, - "title": "Indexing Cost by Tier over Time", + "title": "", "version": 2, "visualizationType": "lnsXY" }, + "drilldowns": [], "title": "Indexing Cost by Tier over Time" }, "gridData": { @@ -1797,16 +2007,37 @@ "y": 6 }, "panelIndex": "2f961fd2-5b04-4c4d-aafb-76af69b06130", - "type": "lens" + "type": "vis" }, { "embeddableConfig": { "attributes": { "references": [], "state": { + "adHocDataViews": { + "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "managed": false, + "name": "billing_realized_pool_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_realized_pool_lookup", + "type": "esql" + } + }, "datasourceStates": { "textBased": { - "indexPatternRefs": [], + "indexPatternRefs": [ + { + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "timeField": "@timestamp", + "title": "billing_realized_pool_lookup" + } + ], "layers": { "d5520b0e-567f-4f0a-b424-fc79eb79e2cc": { "columns": [ @@ -1850,9 +2081,9 @@ } } ], - "index": "chargeback_integration", + "index": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n indexing = CASE (\n deployment_sum_indexing_time > 0,\n TO_DOUBLE(datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_indexing = sum(indexing)\n BY \n @timestamp,\n datastream\n| WHERE agg_indexing > 0\n| SORT agg_indexing DESC" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n indexing = CASE (\n deployment_sum_indexing_time > 0,\n TO_DOUBLE(tier_and_datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_indexing = sum(indexing)\n BY \n @timestamp,\n datastream\n| WHERE agg_indexing > 0\n| SORT agg_indexing DESC" }, "timeField": "@timestamp" } @@ -1862,7 +2093,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n indexing = CASE (\n deployment_sum_indexing_time > 0,\n TO_DOUBLE(datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_indexing = sum(indexing)\n BY \n @timestamp,\n datastream\n| WHERE agg_indexing > 0\n| SORT agg_indexing DESC" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n indexing = CASE (\n deployment_sum_indexing_time > 0,\n TO_DOUBLE(tier_and_datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_indexing = sum(indexing)\n BY \n @timestamp,\n datastream\n| WHERE agg_indexing > 0\n| SORT agg_indexing DESC" }, "visualization": { "axisTitlesVisibilitySettings": { @@ -1909,7 +2140,6 @@ "layerId": "d5520b0e-567f-4f0a-b424-fc79eb79e2cc", "layerType": "data", "seriesType": "bar_percentage_stacked", - "splitAccessor": "datastream", "splitAccessors": [ "datastream" ], @@ -1930,10 +2160,11 @@ "valueLabels": "hide" } }, - "title": "Indexing Cost Share by Data Stream over Time (%)", + "title": "", "version": 2, "visualizationType": "lnsXY" }, + "drilldowns": [], "title": "Indexing Cost Share by Data Stream over Time (%)" }, "gridData": { @@ -1945,16 +2176,37 @@ "y": 6 }, "panelIndex": "c895d51f-09ee-4624-8674-1d77a8d1df61", - "type": "lens" + "type": "vis" }, { "embeddableConfig": { "attributes": { "references": [], "state": { + "adHocDataViews": { + "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "managed": false, + "name": "billing_realized_pool_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_realized_pool_lookup", + "type": "esql" + } + }, "datasourceStates": { "textBased": { - "indexPatternRefs": [], + "indexPatternRefs": [ + { + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "timeField": "@timestamp", + "title": "billing_realized_pool_lookup" + } + ], "layers": { "1a2f8269-f56f-4c02-8506-76b723acc828": { "columns": [ @@ -1998,9 +2250,9 @@ } } ], - "index": "chargeback_integration", + "index": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n querying = CASE (\n deployment_sum_query_time > 0,\n TO_DOUBLE(datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_querying = sum(querying) \n BY \n @timestamp, \n datastream\n| WHERE agg_querying > 0\n| SORT agg_querying DESC" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n querying = CASE (\n deployment_sum_query_time > 0,\n TO_DOUBLE(tier_and_datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_querying = sum(querying) \n BY \n @timestamp, \n datastream\n| WHERE agg_querying > 0\n| SORT agg_querying DESC" }, "timeField": "@timestamp" } @@ -2010,7 +2262,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n querying = CASE (\n deployment_sum_query_time > 0,\n TO_DOUBLE(datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_querying = sum(querying) \n BY \n @timestamp, \n datastream\n| WHERE agg_querying > 0\n| SORT agg_querying DESC" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n querying = CASE (\n deployment_sum_query_time > 0,\n TO_DOUBLE(tier_and_datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_querying = sum(querying) \n BY \n @timestamp, \n datastream\n| WHERE agg_querying > 0\n| SORT agg_querying DESC" }, "visualization": { "axisTitlesVisibilitySettings": { @@ -2057,7 +2309,6 @@ "layerId": "1a2f8269-f56f-4c02-8506-76b723acc828", "layerType": "data", "seriesType": "bar_stacked", - "splitAccessor": "datastream", "splitAccessors": [ "datastream" ], @@ -2078,10 +2329,11 @@ "valueLabels": "hide" } }, - "title": "Query Cost by Data Stream over Time", + "title": "", "version": 2, "visualizationType": "lnsXY" }, + "drilldowns": [], "title": "Query Cost by Data Stream over Time" }, "gridData": { @@ -2093,16 +2345,37 @@ "y": 23 }, "panelIndex": "48aef73e-b6de-424a-85fa-74f46d9bded4", - "type": "lens" + "type": "vis" }, { "embeddableConfig": { "attributes": { "references": [], "state": { + "adHocDataViews": { + "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "managed": false, + "name": "billing_realized_pool_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_realized_pool_lookup", + "type": "esql" + } + }, "datasourceStates": { "textBased": { - "indexPatternRefs": [], + "indexPatternRefs": [ + { + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "timeField": "@timestamp", + "title": "billing_realized_pool_lookup" + } + ], "layers": { "61842c44-afd9-429c-bf2e-18a4a079e5e8": { "columns": [ @@ -2146,9 +2419,9 @@ } } ], - "index": "chargeback_integration", + "index": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n querying = CASE (\n deployment_sum_query_time > 0,\n TO_DOUBLE(tier_sum_query_time) / deployment_sum_query_time * chargeable_pool\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_querying = sum(querying)\n BY \n @timestamp,\n tier\n| WHERE agg_querying > 0" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n querying = CASE (\n deployment_sum_query_time > 0,\n TO_DOUBLE(tier_sum_query_time) / deployment_sum_query_time * chargeable_pool\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_querying = sum(querying)\n BY \n @timestamp,\n tier\n| WHERE agg_querying > 0" }, "timeField": "@timestamp" } @@ -2158,7 +2431,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n querying = CASE (\n deployment_sum_query_time > 0,\n TO_DOUBLE(tier_sum_query_time) / deployment_sum_query_time * chargeable_pool\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_querying = sum(querying)\n BY \n @timestamp,\n tier\n| WHERE agg_querying > 0" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n querying = CASE (\n deployment_sum_query_time > 0,\n TO_DOUBLE(tier_sum_query_time) / deployment_sum_query_time * chargeable_pool\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_querying = sum(querying)\n BY \n @timestamp,\n tier\n| WHERE agg_querying > 0" }, "visualization": { "axisTitlesVisibilitySettings": { @@ -2205,7 +2478,6 @@ "layerId": "61842c44-afd9-429c-bf2e-18a4a079e5e8", "layerType": "data", "seriesType": "bar_stacked", - "splitAccessor": "tier", "splitAccessors": [ "tier" ], @@ -2226,10 +2498,11 @@ "valueLabels": "hide" } }, - "title": "Query Cost by Tier over Time", + "title": "", "version": 2, "visualizationType": "lnsXY" }, + "drilldowns": [], "title": "Query Cost by Tier over Time" }, "gridData": { @@ -2241,16 +2514,37 @@ "y": 23 }, "panelIndex": "fb59beb6-bf05-4ce0-90a3-d8b434d300aa", - "type": "lens" + "type": "vis" }, { "embeddableConfig": { "attributes": { "references": [], "state": { + "adHocDataViews": { + "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "managed": false, + "name": "billing_realized_pool_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_realized_pool_lookup", + "type": "esql" + } + }, "datasourceStates": { "textBased": { - "indexPatternRefs": [], + "indexPatternRefs": [ + { + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "timeField": "@timestamp", + "title": "billing_realized_pool_lookup" + } + ], "layers": { "91bcc41e-3a2c-434d-8df7-29c01340e712": { "columns": [ @@ -2294,9 +2588,9 @@ } } ], - "index": "chargeback_integration", + "index": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_querying = sum(querying)\n BY \n @timestamp, \n datastream\n| WHERE agg_querying > 0\n| SORT agg_querying DESC" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_and_datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_querying = sum(querying)\n BY \n @timestamp, \n datastream\n| WHERE agg_querying > 0\n| SORT agg_querying DESC" }, "timeField": "@timestamp" } @@ -2306,7 +2600,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_querying = sum(querying)\n BY \n @timestamp, \n datastream\n| WHERE agg_querying > 0\n| SORT agg_querying DESC" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_and_datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_querying = sum(querying)\n BY \n @timestamp, \n datastream\n| WHERE agg_querying > 0\n| SORT agg_querying DESC" }, "visualization": { "axisTitlesVisibilitySettings": { @@ -2353,7 +2647,6 @@ "layerId": "91bcc41e-3a2c-434d-8df7-29c01340e712", "layerType": "data", "seriesType": "bar_percentage_stacked", - "splitAccessor": "datastream", "splitAccessors": [ "datastream" ], @@ -2374,10 +2667,11 @@ "valueLabels": "hide" } }, - "title": "Query Cost Share by Data Stream over Time (%)", + "title": "", "version": 2, "visualizationType": "lnsXY" }, + "drilldowns": [], "title": "Query Cost Share by Data Stream over Time (%)" }, "gridData": { @@ -2389,16 +2683,37 @@ "y": 23 }, "panelIndex": "6fb65bcc-0568-4c8d-a833-c87cf8f8bccd", - "type": "lens" + "type": "vis" }, { "embeddableConfig": { "attributes": { "references": [], "state": { + "adHocDataViews": { + "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "managed": false, + "name": "billing_realized_pool_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_realized_pool_lookup", + "type": "esql" + } + }, "datasourceStates": { "textBased": { - "indexPatternRefs": [], + "indexPatternRefs": [ + { + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "timeField": "@timestamp", + "title": "billing_realized_pool_lookup" + } + ], "layers": { "b72a0ab5-dbf5-4fd2-bdef-c855fe354bee": { "columns": [ @@ -2442,9 +2757,9 @@ } } ], - "index": "chargeback_integration", + "index": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_storage = sum(storage)\n BY \n @timestamp, \n datastream\n| WHERE agg_storage > 0\n| SORT agg_storage DESC" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_storage = sum(storage)\n BY \n @timestamp, \n datastream\n| WHERE agg_storage > 0\n| SORT agg_storage DESC" }, "timeField": "@timestamp" } @@ -2454,7 +2769,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_storage = sum(storage)\n BY \n @timestamp, \n datastream\n| WHERE agg_storage > 0\n| SORT agg_storage DESC" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_storage = sum(storage)\n BY \n @timestamp, \n datastream\n| WHERE agg_storage > 0\n| SORT agg_storage DESC" }, "visualization": { "axisTitlesVisibilitySettings": { @@ -2501,7 +2816,6 @@ "layerId": "b72a0ab5-dbf5-4fd2-bdef-c855fe354bee", "layerType": "data", "seriesType": "bar_stacked", - "splitAccessor": "datastream", "splitAccessors": [ "datastream" ], @@ -2522,10 +2836,11 @@ "valueLabels": "hide" } }, - "title": "Storage Cost by Data Stream over Time", + "title": "", "version": 2, "visualizationType": "lnsXY" }, + "drilldowns": [], "title": "Storage Cost by Data Stream over Time" }, "gridData": { @@ -2537,16 +2852,37 @@ "y": 40 }, "panelIndex": "5bdd9079-cb0e-453e-835b-afcb5d72f06b", - "type": "lens" + "type": "vis" }, { "embeddableConfig": { "attributes": { "references": [], "state": { + "adHocDataViews": { + "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "managed": false, + "name": "billing_realized_pool_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_realized_pool_lookup", + "type": "esql" + } + }, "datasourceStates": { "textBased": { - "indexPatternRefs": [], + "indexPatternRefs": [ + { + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "timeField": "@timestamp", + "title": "billing_realized_pool_lookup" + } + ], "layers": { "93fd5f12-463f-4e8e-a9b8-0ac77d0522a9": { "columns": [ @@ -2590,9 +2926,9 @@ } } ], - "index": "chargeback_integration", + "index": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_storage = sum(storage)\n BY \n @timestamp,\n tier\n| WHERE agg_storage > 0" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_storage = sum(storage)\n BY \n @timestamp,\n tier\n| WHERE agg_storage > 0" }, "timeField": "@timestamp" } @@ -2602,7 +2938,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_storage = sum(storage)\n BY \n @timestamp,\n tier\n| WHERE agg_storage > 0" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_storage = sum(storage)\n BY \n @timestamp,\n tier\n| WHERE agg_storage > 0" }, "visualization": { "axisTitlesVisibilitySettings": { @@ -2649,7 +2985,6 @@ "layerId": "93fd5f12-463f-4e8e-a9b8-0ac77d0522a9", "layerType": "data", "seriesType": "bar_stacked", - "splitAccessor": "tier", "splitAccessors": [ "tier" ], @@ -2670,10 +3005,11 @@ "valueLabels": "hide" } }, - "title": "Storage Cost by Tier over Time", + "title": "", "version": 2, "visualizationType": "lnsXY" }, + "drilldowns": [], "title": "Storage Cost by Tier over Time" }, "gridData": { @@ -2685,16 +3021,37 @@ "y": 40 }, "panelIndex": "5ac2dcd4-8775-4c15-8e79-346e0a62dd4d", - "type": "lens" + "type": "vis" }, { "embeddableConfig": { "attributes": { "references": [], "state": { + "adHocDataViews": { + "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "managed": false, + "name": "billing_realized_pool_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_realized_pool_lookup", + "type": "esql" + } + }, "datasourceStates": { "textBased": { - "indexPatternRefs": [], + "indexPatternRefs": [ + { + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "timeField": "@timestamp", + "title": "billing_realized_pool_lookup" + } + ], "layers": { "b72a0ab5-dbf5-4fd2-bdef-c855fe354bee": { "columns": [ @@ -2738,9 +3095,9 @@ } } ], - "index": "chargeback_integration", + "index": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_storage = sum(storage)\n BY \n @timestamp, \n datastream\n| WHERE agg_storage > 0\n| SORT agg_storage DESC" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_storage = sum(storage)\n BY \n @timestamp, \n datastream\n| WHERE agg_storage > 0\n| SORT agg_storage DESC" }, "timeField": "@timestamp" } @@ -2750,7 +3107,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_storage = sum(storage)\n BY \n @timestamp, \n datastream\n| WHERE agg_storage > 0\n| SORT agg_storage DESC" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS \n agg_storage = sum(storage)\n BY \n @timestamp, \n datastream\n| WHERE agg_storage > 0\n| SORT agg_storage DESC" }, "visualization": { "axisTitlesVisibilitySettings": { @@ -2797,7 +3154,6 @@ "layerId": "b72a0ab5-dbf5-4fd2-bdef-c855fe354bee", "layerType": "data", "seriesType": "bar_percentage_stacked", - "splitAccessor": "datastream", "splitAccessors": [ "datastream" ], @@ -2818,10 +3174,11 @@ "valueLabels": "hide" } }, - "title": "Storage Cost Share by Data Stream over Time (%)", + "title": "", "version": 2, "visualizationType": "lnsXY" }, + "drilldowns": [], "title": "Storage Cost Share by Data Stream over Time (%)" }, "gridData": { @@ -2833,16 +3190,37 @@ "y": 40 }, "panelIndex": "5d96941f-c4e3-4833-b716-192d75d6ae0d", - "type": "lens" + "type": "vis" }, { "embeddableConfig": { "attributes": { "references": [], "state": { + "adHocDataViews": { + "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "managed": false, + "name": "billing_realized_pool_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_realized_pool_lookup", + "type": "esql" + } + }, "datasourceStates": { "textBased": { - "indexPatternRefs": [], + "indexPatternRefs": [ + { + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "timeField": "@timestamp", + "title": "billing_realized_pool_lookup" + } + ], "layers": { "52906f25-a114-4d45-b425-f0ff154f4ac1": { "columns": [ @@ -2877,18 +3255,18 @@ }, { "columnId": "datastream", - "customLabel": false, + "customLabel": true, "fieldName": "datastream", - "label": "datastream", + "label": "Datastream", "meta": { "esType": "keyword", "type": "string" } } ], - "index": "chargeback_integration", + "index": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool),\n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool),\n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n blended = TO_DOUBLE(((storage * conf_storage_weight) + (querying * conf_query_weight) + (indexing * conf_indexing_weight))) / total_weight_hot * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS\n agg_blended = sum(blended)\n BY\n @timestamp,\n datastream\n| WHERE agg_blended > 0\n| SORT agg_blended DESC" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_and_datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool),\n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_and_datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool),\n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n TO_DOUBLE(((storage * conf_storage_weight) + (querying * conf_query_weight) + (indexing * conf_indexing_weight))) / total_weight_hot,\n TO_DOUBLE(((storage * conf_storage_weight) + (querying * conf_query_weight))) / total_weight_cold\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS\n agg_blended = sum(blended)\n BY\n @timestamp,\n datastream\n| WHERE agg_blended > 0\n| SORT agg_blended DESC" }, "timeField": "@timestamp" } @@ -2898,7 +3276,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool),\n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool),\n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n blended = TO_DOUBLE(((storage * conf_storage_weight) + (querying * conf_query_weight) + (indexing * conf_indexing_weight))) / total_weight_hot * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS\n agg_blended = sum(blended)\n BY\n @timestamp,\n datastream\n| WHERE agg_blended > 0\n| SORT agg_blended DESC" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_and_datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool),\n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_and_datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool),\n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n TO_DOUBLE(((storage * conf_storage_weight) + (querying * conf_query_weight) + (indexing * conf_indexing_weight))) / total_weight_hot,\n TO_DOUBLE(((storage * conf_storage_weight) + (querying * conf_query_weight))) / total_weight_cold\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS\n agg_blended = sum(blended)\n BY\n @timestamp,\n datastream\n| WHERE agg_blended > 0\n| SORT agg_blended DESC" }, "visualization": { "axisTitlesVisibilitySettings": { @@ -2945,7 +3323,6 @@ "layerId": "52906f25-a114-4d45-b425-f0ff154f4ac1", "layerType": "data", "seriesType": "bar_stacked", - "splitAccessor": "datastream", "splitAccessors": [ "datastream" ], @@ -2966,10 +3343,11 @@ "valueLabels": "hide" } }, - "title": "Blended Cost by Data Stream over Time", + "title": "", "version": 2, "visualizationType": "lnsXY" }, + "drilldowns": [], "title": "Blended Cost by Data Stream over Time" }, "gridData": { @@ -2981,16 +3359,37 @@ "y": 58 }, "panelIndex": "0f25691b-8bc9-4dd2-94be-893a6e64a195", - "type": "lens" + "type": "vis" }, { "embeddableConfig": { "attributes": { "references": [], "state": { + "adHocDataViews": { + "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "managed": false, + "name": "billing_realized_pool_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_realized_pool_lookup", + "type": "esql" + } + }, "datasourceStates": { "textBased": { - "indexPatternRefs": [], + "indexPatternRefs": [ + { + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "timeField": "@timestamp", + "title": "billing_realized_pool_lookup" + } + ], "layers": { "43f9cfe6-d828-4773-ab65-129f92cf7f4d": { "columns": [ @@ -3026,9 +3425,9 @@ } } ], - "index": "chargeback_integration", + "index": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight) + (indexing / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_indexing_weight)) / total_weight_hot,\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight)) / total_weight_cold\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS\n `Normalized cost` = SUM(blended) BY @timestamp, tier\n| WHERE `Normalized cost` > 0\n| RENAME tier AS `Data Tier`\n| SORT @timestamp ASC" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight) + (indexing / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_indexing_weight)) / total_weight_hot,\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight)) / total_weight_cold\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS\n `Normalized cost` = SUM(blended) BY @timestamp, tier\n| WHERE `Normalized cost` > 0\n| RENAME tier AS `Data Tier`\n| SORT @timestamp ASC" }, "timeField": "@timestamp" } @@ -3038,7 +3437,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight) + (indexing / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_indexing_weight)) / total_weight_hot,\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight)) / total_weight_cold\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS\n `Normalized cost` = SUM(blended) BY @timestamp, tier\n| WHERE `Normalized cost` > 0\n| RENAME tier AS `Data Tier`\n| SORT @timestamp ASC" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight) + (indexing / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_indexing_weight)) / total_weight_hot,\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight)) / total_weight_cold\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS\n `Normalized cost` = SUM(blended) BY @timestamp, tier\n| WHERE `Normalized cost` > 0\n| RENAME tier AS `Data Tier`\n| SORT @timestamp ASC" }, "visualization": { "axisTitlesVisibilitySettings": { @@ -3085,7 +3484,6 @@ "layerId": "43f9cfe6-d828-4773-ab65-129f92cf7f4d", "layerType": "data", "seriesType": "bar_stacked", - "splitAccessor": "Data Tier", "splitAccessors": [ "Data Tier" ], @@ -3106,10 +3504,11 @@ "valueLabels": "hide" } }, - "title": "Blended Cost by Tier over Time", + "title": "", "version": 2, "visualizationType": "lnsXY" }, + "drilldowns": [], "title": "Blended Cost by Tier over Time" }, "gridData": { @@ -3121,16 +3520,37 @@ "y": 58 }, "panelIndex": "b3e1e89a-a86d-40b3-97e2-ba89751330fa", - "type": "lens" + "type": "vis" }, { "embeddableConfig": { "attributes": { "references": [], "state": { + "adHocDataViews": { + "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "managed": false, + "name": "billing_realized_pool_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_realized_pool_lookup", + "type": "esql" + } + }, "datasourceStates": { "textBased": { - "indexPatternRefs": [], + "indexPatternRefs": [ + { + "id": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", + "timeField": "@timestamp", + "title": "billing_realized_pool_lookup" + } + ], "layers": { "d720a72f-fe61-4baf-bdee-8a50d4247670": { "columns": [ @@ -3157,18 +3577,18 @@ }, { "columnId": "datastream", - "customLabel": false, + "customLabel": true, "fieldName": "datastream", - "label": "datastream", + "label": "Datastream", "meta": { "esType": "keyword", "type": "string" } } ], - "index": "chargeback_integration", + "index": "79f2ef40157158682526c4b71191b878c7d88248ab4d7c5189f68bb41c1cb496", "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_and_datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_and_datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight) + (indexing / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_indexing_weight)) / total_weight_hot,\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight)) / total_weight_cold\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS\n `% Normalized cost` = sum(blended)\n BY\n @timestamp,\n datastream,\n tier\n| WHERE `% Normalized cost` > 0\n| SORT `% Normalized cost` DESC" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_and_datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_and_datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight) + (indexing / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_indexing_weight)) / total_weight_hot,\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight)) / total_weight_cold\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS\n `% Normalized cost` = sum(blended)\n BY\n @timestamp,\n datastream,\n tier\n| WHERE `% Normalized cost` > 0\n| SORT `% Normalized cost` DESC" }, "timeField": "@timestamp" } @@ -3178,7 +3598,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_realized_pool_lookup\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_and_datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_and_datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight) + (indexing / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_indexing_weight)) / total_weight_hot,\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight)) / total_weight_cold\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS\n `% Normalized cost` = sum(blended)\n BY\n @timestamp,\n datastream,\n tier\n| WHERE `% Normalized cost` > 0\n| SORT `% Normalized cost` DESC" + "esql": "FROM billing_realized_pool_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| LOOKUP JOIN cluster_capacity_utilization_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| WHERE (CASE(?tier_selected IS NULL OR MV_INTERSECTS(?tier_selected, [\"All\"]), true, MV_INTERSECTS(?tier_selected, tier)))\n| WHERE (CASE(?datastream_selected IS NULL OR MV_INTERSECTS(?datastream_selected, [\"All\"]), true, MV_INTERSECTS(?datastream_selected, datastream)))\n| WHERE (CASE(?ds_type_selected IS NULL OR MV_INTERSECTS(?ds_type_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_type_selected, ds_type)))\n| WHERE (CASE(?ds_namespace_selected IS NULL OR MV_INTERSECTS(?ds_namespace_selected, [\"__chargeback_unfiltered__\"]), true, MV_INTERSECTS(?ds_namespace_selected, ds_namespace)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL heap_util = COALESCE(heap_used_pct_p95, 100) / 100, disk_util = COALESCE(disk_used_pct_p95, 100) / 100, w_sum = conf_utilization_memory_weight + conf_utilization_storage_weight, capacity_utilization_score = CASE(w_sum > 0, (conf_utilization_memory_weight * heap_util + conf_utilization_storage_weight * disk_util) / w_sum, 1.0)\n| EVAL chargeable_pool = data_tier_capacity_ecu * COALESCE(capacity_utilization_score, 1.0), utilization_data_missing = heap_used_pct_p95 IS NULL AND disk_used_pct_p95 IS NULL\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, TO_DOUBLE(tier_and_datastream_sum_indexing_time) / deployment_sum_indexing_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n querying = CASE (deployment_sum_query_time > 0, TO_DOUBLE(tier_and_datastream_sum_query_time) / deployment_sum_query_time * chargeable_pool) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n data_set = CASE (deployment_sum_data_set_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_data_set_store_size) / deployment_sum_data_set_store_size * chargeable_pool),\n store = CASE (deployment_sum_store_size > 0, TO_DOUBLE(tier_and_datastream_sum_store_size) / deployment_sum_store_size * chargeable_pool),\n storage = CASE (store == 0, data_set, store) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight) + (indexing / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_indexing_weight)) / total_weight_hot,\n ((storage / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_storage_weight) + (querying / COALESCE(conf_chargeable_unit_rate, conf_ecu_rate) * conf_query_weight)) / total_weight_cold\n ) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate)\n| STATS\n `% Normalized cost` = sum(blended)\n BY\n @timestamp,\n datastream,\n tier\n| WHERE `% Normalized cost` > 0\n| SORT `% Normalized cost` DESC" }, "visualization": { "axisTitlesVisibilitySettings": { @@ -3225,7 +3645,6 @@ "layerId": "d720a72f-fe61-4baf-bdee-8a50d4247670", "layerType": "data", "seriesType": "bar_percentage_stacked", - "splitAccessor": "datastream", "splitAccessors": [ "datastream" ], @@ -3246,10 +3665,11 @@ "valueLabels": "hide" } }, - "title": "Blended Cost Share by Tier and Data Stream (%)", + "title": "", "version": 2, "visualizationType": "lnsXY" }, + "drilldowns": [], "title": "Blended Cost Share by Tier and Data Stream (%)" }, "gridData": { @@ -3261,15 +3681,10 @@ "y": 58 }, "panelIndex": "cefe7754-4e91-4b85-8132-512a9ca31793", - "type": "lens" + "type": "vis" }, { "embeddableConfig": { - "enhancements": { - "dynamicActions": { - "events": [] - } - }, "savedVis": { "data": { "aggs": [], @@ -3302,15 +3717,10 @@ "y": 0 }, "panelIndex": "5994f908-7b57-4bd9-8e58-0b275c4c7f8e", - "type": "visualization" + "type": "legacy_vis" }, { "embeddableConfig": { - "enhancements": { - "dynamicActions": { - "events": [] - } - }, "savedVis": { "data": { "aggs": [], @@ -3343,15 +3753,10 @@ "y": 3 }, "panelIndex": "5011c332-ce8a-4a78-a08a-c0527c374fcb", - "type": "visualization" + "type": "legacy_vis" }, { "embeddableConfig": { - "enhancements": { - "dynamicActions": { - "events": [] - } - }, "savedVis": { "data": { "aggs": [], @@ -3384,15 +3789,10 @@ "y": 20 }, "panelIndex": "e458e034-197a-4ec6-95a8-2938181a4883", - "type": "visualization" + "type": "legacy_vis" }, { "embeddableConfig": { - "enhancements": { - "dynamicActions": { - "events": [] - } - }, "savedVis": { "data": { "aggs": [], @@ -3425,15 +3825,10 @@ "y": 37 }, "panelIndex": "692b3cdd-b37f-4da9-80a6-aff7b799ecc1", - "type": "visualization" + "type": "legacy_vis" }, { "embeddableConfig": { - "enhancements": { - "dynamicActions": { - "events": [] - } - }, "savedVis": { "data": { "aggs": [], @@ -3466,7 +3861,7 @@ "y": 55 }, "panelIndex": "50e188d3-b5da-49e7-8342-247554634e9a", - "type": "visualization" + "type": "legacy_vis" } ], "refreshInterval": { @@ -3478,7 +3873,7 @@ "collapsed": false, "gridData": { "i": "0fceb22c-1e73-45e4-b634-006f6dcc5f43", - "y": 2 + "y": 4 }, "title": "Datatiers / utilization" }, @@ -3486,7 +3881,7 @@ "collapsed": false, "gridData": { "i": "d11ae80b-d7bb-474f-9082-81752ab2865d", - "y": 3 + "y": 5 }, "title": "Data tier and data stream overview" }, @@ -3494,7 +3889,7 @@ "collapsed": false, "gridData": { "i": "7f7cc9b5-13b3-4962-98f6-5b18a1d8ca80", - "y": 4 + "y": 6 }, "title": "Data tier and data stream per day" } @@ -3502,38 +3897,28 @@ "timeFrom": "now-3M", "timeRestore": true, "timeTo": "now", - "title": "[Chargeback] Usage & Cost Allocation", - "version": 1 + "title": "[Chargeback] Usage & Cost Allocation" }, + "coreMigrationVersion": "8.8.0", + "created_at": "2026-06-09T08:55:30.079Z", "id": "chargeback-21588d0e-fb6a-4f76-ad2f-cd7b3d7a3d7c", "references": [ - { - "id": "chargeback_integration", - "name": "controlGroup_b680ef70-4697-41e5-bf5f-e7e1e700ec3e:optionsListDataView", - "type": "index-pattern" - }, - { - "id": "chargeback_integration", - "name": "controlGroup_2c7df183-8a05-4ee0-b7db-415e874636cd:optionsListDataView", - "type": "index-pattern" - }, { "id": "chargeback-b6cecac0-ebe7-43de-bfc6-1ff287ad860a", - "name": "13cd2a6a-ec89-4553-ae64-cbc347770aa5:link_ea0417fa-630d-c656-8f4b-a8d7b9916b29_dashboard", + "name": "13cd2a6a-ec89-4553-ae64-cbc347770aa5:link_a11b22c3-d44e-455f-8660-778899001122_dashboard", "type": "dashboard" }, { "id": "chargeback-21588d0e-fb6a-4f76-ad2f-cd7b3d7a3d7c", - "name": "13cd2a6a-ec89-4553-ae64-cbc347770aa5:link_dc802531-4211-c74b-d9db-b197b528bbc8_dashboard", + "name": "13cd2a6a-ec89-4553-ae64-cbc347770aa5:link_b22c33d4-e55f-4660-9771-889900112233_dashboard", "type": "dashboard" }, { "id": "chargeback-7ec9743a-ccfa-43d2-8311-2f25893dadae", - "name": "13cd2a6a-ec89-4553-ae64-cbc347770aa5:link_4e06e36e-f5d7-981f-8554-57756d1c59b8_dashboard", + "name": "13cd2a6a-ec89-4553-ae64-cbc347770aa5:link_c33d44e5-f660-4771-a882-990011223344_dashboard", "type": "dashboard" } ], "type": "dashboard", - "typeMigrationVersion": "10.3.0", - "coreMigrationVersion": "8.8.0" -} \ No newline at end of file + "typeMigrationVersion": "10.3.0" +} diff --git a/packages/chargeback/kibana/dashboard/chargeback-7ec9743a-ccfa-43d2-8311-2f25893dadae.json b/packages/chargeback/kibana/dashboard/chargeback-7ec9743a-ccfa-43d2-8311-2f25893dadae.json index 392a24325e9..8f94b3b825f 100644 --- a/packages/chargeback/kibana/dashboard/chargeback-7ec9743a-ccfa-43d2-8311-2f25893dadae.json +++ b/packages/chargeback/kibana/dashboard/chargeback-7ec9743a-ccfa-43d2-8311-2f25893dadae.json @@ -63,7 +63,8 @@ } ] }, - "enhancements": {} + "enhancements": {}, + "hide_title": true }, "gridData": { "h": 2, @@ -1016,11 +1017,14 @@ "type": "visualization" } ], + "pinned_panels": null, "timeFrom": "now-3M", "timeRestore": true, "timeTo": "now", "title": "[Chargeback] Configuration" }, + "coreMigrationVersion": "8.8.0", + "created_at": "2026-06-09T08:55:30.079Z", "id": "chargeback-7ec9743a-ccfa-43d2-8311-2f25893dadae", "references": [ { @@ -1040,6 +1044,5 @@ } ], "type": "dashboard", - "typeMigrationVersion": "10.3.0", - "coreMigrationVersion": "8.8.0" -} \ No newline at end of file + "typeMigrationVersion": "10.3.0" +} diff --git a/packages/chargeback/kibana/dashboard/chargeback-b6cecac0-ebe7-43de-bfc6-1ff287ad860a.json b/packages/chargeback/kibana/dashboard/chargeback-b6cecac0-ebe7-43de-bfc6-1ff287ad860a.json index 09b2abaaba6..a80460232ec 100644 --- a/packages/chargeback/kibana/dashboard/chargeback-b6cecac0-ebe7-43de-bfc6-1ff287ad860a.json +++ b/packages/chargeback/kibana/dashboard/chargeback-b6cecac0-ebe7-43de-bfc6-1ff287ad860a.json @@ -1,102 +1,9 @@ { "attributes": { - "controlGroupInput": { - "chainingSystem": "HIERARCHICAL", - "controlStyle": "oneLine", - "ignoreParentSettingsJSON": { - "ignoreFilters": false, - "ignoreQuery": false, - "ignoreTimerange": false, - "ignoreValidations": false - }, - "panelsJSON": { - "2c7df183-8a05-4ee0-b7db-415e874636cd": { - "explicitInput": { - "dataViewId": "chargeback_integration", - "exclude": false, - "existsSelected": false, - "fieldName": "deployment_name", - "searchTechnique": "prefix", - "selectedOptions": [], - "sort": { - "by": "_count", - "direction": "desc" - }, - "title": "Deployment name", - "id": "2c7df183-8a05-4ee0-b7db-415e874636cd" - }, - "grow": false, - "order": 1, - "type": "optionsListControl", - "width": "medium" - }, - "b680ef70-4697-41e5-bf5f-e7e1e700ec3e": { - "explicitInput": { - "dataViewId": "chargeback_integration", - "exclude": false, - "existsSelected": false, - "fieldName": "deployment_group", - "searchTechnique": "prefix", - "selectedOptions": [], - "sort": { - "by": "_count", - "direction": "desc" - }, - "title": "Deployment group", - "id": "b680ef70-4697-41e5-bf5f-e7e1e700ec3e" - }, - "grow": false, - "order": 0, - "type": "optionsListControl", - "width": "medium" - }, - "929895f7-b618-ac2e-c7d7-61502d6acb50": { - "explicitInput": { - "dataViewId": "chargeback_integration", - "exclude": false, - "existsSelected": false, - "fieldName": "cost_category", - "id": "929895f7-b618-ac2e-c7d7-61502d6acb50", - "searchTechnique": "prefix", - "selectedOptions": [], - "sort": { - "by": "_count", - "direction": "desc" - }, - "title": "Cost Category" - }, - "grow": false, - "order": 2, - "type": "optionsListControl", - "width": "medium" - }, - "456d8f2e-65b3-eaab-1d98-ef80690574ad": { - "explicitInput": { - "dataViewId": "chargeback_integration", - "exclude": false, - "existsSelected": false, - "fieldName": "cost_type", - "id": "456d8f2e-65b3-eaab-1d98-ef80690574ad", - "searchTechnique": "prefix", - "selectedOptions": [], - "sort": { - "by": "_count", - "direction": "desc" - }, - "title": "Cost Type" - }, - "grow": false, - "order": 3, - "type": "optionsListControl", - "width": "medium" - } - }, - "showApplySelections": false - }, + "controlGroupInput": {}, "description": "Cost breakdown by SKU, deployment, and tier. Shows total billing, cost by component, and deployment cost comparison.", "kibanaSavedObjectMeta": { "searchSourceJSON": { - "filter": [], "query": { "language": "kuery", "query": "" @@ -104,6 +11,8 @@ } }, "optionsJSON": { + "autoApplyFilters": true, + "hidePanelBorders": false, "hidePanelTitles": false, "syncColors": false, "syncCursor": true, @@ -113,46 +22,49 @@ "panelsJSON": [ { "embeddableConfig": { - "enhancements": {}, - "layout": "horizontal", - "links": [ - { - "destinationRefName": "link_94bf4612-f92b-cf6d-5c31-0ef3eaa2fc0d_dashboard", - "id": "94bf4612-f92b-cf6d-5c31-0ef3eaa2fc0d", - "label": "[Chargeback] Billing Components Overview", - "options": { - "openInNewTab": false, - "useCurrentDateRange": true, - "useCurrentFilters": true - }, - "order": 0, - "type": "dashboardLink" - }, - { - "destinationRefName": "link_2652d69f-1f3e-7afa-e44e-08697f8e318e_dashboard", - "id": "2652d69f-1f3e-7afa-e44e-08697f8e318e", - "label": "[Chargeback] Usage & Cost Allocation", - "options": { - "openInNewTab": false, - "useCurrentDateRange": true, - "useCurrentFilters": true + "attributes": { + "layout": "horizontal", + "links": [ + { + "destinationRefName": "link_cba353e8-3f35-e498-55ec-72e282a480ce_dashboard", + "id": "cba353e8-3f35-e498-55ec-72e282a480ce", + "label": "[Chargeback] Billing Components Overview", + "options": { + "openInNewTab": false, + "useCurrentDateRange": true, + "useCurrentFilters": true + }, + "order": 0, + "type": "dashboardLink" }, - "order": 1, - "type": "dashboardLink" - }, - { - "destinationRefName": "link_b22db599-5a58-6f1f-cfdb-408e4543af73_dashboard", - "id": "b22db599-5a58-6f1f-cfdb-408e4543af73", - "label": "[Chargeback] Configuration", - "options": { - "openInNewTab": false, - "useCurrentDateRange": true, - "useCurrentFilters": true + { + "destinationRefName": "link_4b7dbb50-c024-8666-14ba-f6b4487eced7_dashboard", + "id": "4b7dbb50-c024-8666-14ba-f6b4487eced7", + "label": "[Chargeback] Usage & Cost Allocation", + "options": { + "openInNewTab": false, + "useCurrentDateRange": true, + "useCurrentFilters": true + }, + "order": 1, + "type": "dashboardLink" }, - "order": 2, - "type": "dashboardLink" - } - ] + { + "destinationRefName": "link_3a5f90b9-a885-f9a9-6669-49f360b72b47_dashboard", + "id": "3a5f90b9-a885-f9a9-6669-49f360b72b47", + "label": "[Chargeback] Configuration", + "options": { + "openInNewTab": false, + "useCurrentDateRange": true, + "useCurrentFilters": true + }, + "order": 2, + "type": "dashboardLink" + } + ] + }, + "enhancements": {}, + "hide_title": true }, "gridData": { "h": 2, @@ -164,17 +76,105 @@ "panelIndex": "c39e2f18-f301-40b0-8f59-2a8547ddffbb", "type": "links" }, + { + "embeddableConfig": { + "control_type": "VALUES_FROM_QUERY", + "esql_query": "FROM billing_cluster_cost_lookup\n| WHERE deployment_group != \"\"\n| STATS COUNT(*) BY deployment_group\n| KEEP deployment_group\n| SORT deployment_group", + "selected_options": [ + "__chargeback_unfiltered__" + ], + "single_select": false, + "title": "Deployment group", + "variable_name": "dg_selected", + "variable_type": "multi_values" + }, + "gridData": { + "h": 2, + "i": "c42ec760-734f-46bb-8e0f-e52f3d4a1f4d", + "w": 12, + "x": 0, + "y": 2 + }, + "panelIndex": "c42ec760-734f-46bb-8e0f-e52f3d4a1f4d", + "type": "esql_control" + }, + { + "embeddableConfig": { + "control_type": "VALUES_FROM_QUERY", + "esql_query": "FROM billing_cluster_cost_lookup\n| WHERE deployment_group != \"\"\n| STATS COUNT(*) BY deployment_name\n| KEEP deployment_name\n| SORT deployment_name", + "selected_options": [ + "__chargeback_unfiltered__" + ], + "single_select": false, + "title": "Deployment name", + "variable_name": "deployment_selected", + "variable_type": "multi_values" + }, + "gridData": { + "h": 2, + "i": "5d11a79f-3957-4c0c-a42a-35616191a6fa", + "w": 12, + "x": 12, + "y": 2 + }, + "panelIndex": "5d11a79f-3957-4c0c-a42a-35616191a6fa", + "type": "esql_control" + }, + { + "embeddableConfig": { + "control_type": "VALUES_FROM_QUERY", + "esql_query": "FROM billing_cluster_cost_lookup\n| WHERE cost_category IS NOT NULL\n| STATS COUNT(*) BY cost_category\n| KEEP cost_category\n| SORT cost_category", + "selected_options": [ + "__chargeback_unfiltered__" + ], + "single_select": false, + "title": "Cost category", + "variable_name": "cost_category_selected", + "variable_type": "multi_values" + }, + "gridData": { + "h": 2, + "i": "2cc4273b-85a3-46d8-aebd-019a383c4b64", + "w": 12, + "x": 24, + "y": 2 + }, + "panelIndex": "2cc4273b-85a3-46d8-aebd-019a383c4b64", + "type": "esql_control" + }, + { + "embeddableConfig": { + "control_type": "VALUES_FROM_QUERY", + "esql_query": "FROM billing_cluster_cost_lookup\n| WHERE cost_type IS NOT NULL\n| STATS COUNT(*) BY cost_type\n| KEEP cost_type\n| SORT cost_type", + "selected_options": [ + "__chargeback_unfiltered__" + ], + "single_select": false, + "title": "Cost type", + "variable_name": "cost_type_selected", + "variable_type": "multi_values" + }, + "gridData": { + "h": 2, + "i": "18543502-9e06-4644-9a39-34e39d2eecd4", + "w": 12, + "x": 36, + "y": 2 + }, + "panelIndex": "18543502-9e06-4644-9a39-34e39d2eecd4", + "type": "esql_control" + }, { "embeddableConfig": { "attributes": { "references": [], "state": { "adHocDataViews": { - "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "billing_cluster_cost_lookup-e13dc6fb1b18b585207fd8e3082418a6823ac99eb799290ce43c5cb51441a2cc": { "allowHidden": false, "allowNoIndex": false, "fieldFormats": {}, - "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "id": "billing_cluster_cost_lookup-e13dc6fb1b18b585207fd8e3082418a6823ac99eb799290ce43c5cb51441a2cc", "managed": false, "name": "billing_cluster_cost_lookup", "runtimeFieldMap": {}, @@ -188,48 +188,39 @@ "textBased": { "indexPatternRefs": [ { - "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "id": "billing_cluster_cost_lookup-e13dc6fb1b18b585207fd8e3082418a6823ac99eb799290ce43c5cb51441a2cc", "timeField": "@timestamp", "title": "billing_cluster_cost_lookup" } ], "layers": { - "aa55ae31-395f-4738-b9fd-3da1d649d640": { + "bar_stacked_0": { "columns": [ { - "columnId": "@timestamp", - "customLabel": false, + "columnId": "bar_stacked_x", "fieldName": "@timestamp", - "label": "@timestamp", "meta": { - "esType": "date", "type": "date" } }, { - "columnId": "deployment", - "customLabel": false, - "fieldName": "deployment", - "label": "Deployment", + "columnId": "bar_stacked_y_0", + "fieldName": "Normalized cost", "meta": { - "esType": "keyword", - "type": "string" + "type": "number" } }, { - "columnId": "Normalized cost", - "customLabel": false, - "fieldName": "Normalized cost", - "label": "Normalized cost", + "columnId": "bar_stacked_breakdown", + "fieldName": "deployment", "meta": { - "esType": "double", - "type": "number" + "type": "string" } } ], - "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "index": "billing_cluster_cost_lookup-e13dc6fb1b18b585207fd8e3082418a6823ac99eb799290ce43c5cb51441a2cc", "query": { - "esql": "FROM billing_cluster_cost_lookup\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL total_chargeable_units_value = COALESCE(total_chargeable_units, total_ecu) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0), deployment = CONCAT(deployment_name, \" (\", LEFT(deployment_id, 6), \")\")\n| STATS `Normalized cost` = SUM(total_chargeable_units_value) BY @timestamp, deployment\n| SORT @timestamp DESC\n| LIMIT 5000" + "esql": "FROM billing_cluster_cost_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| WHERE (CASE(?cost_category_selected IS NULL OR MV_INTERSECTS(?cost_category_selected, [\"All\"]), true, MV_INTERSECTS(?cost_category_selected, cost_category)))\n| WHERE (CASE(?cost_type_selected IS NULL OR MV_INTERSECTS(?cost_type_selected, [\"All\"]), true, MV_INTERSECTS(?cost_type_selected, cost_type)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL total_chargeable_units_value = COALESCE(total_chargeable_units, total_ecu) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0),\n deployment = CONCAT(deployment_name, \" (\", LEFT(deployment_id, 6), \")\")\n| STATS `Normalized cost` = SUM(total_chargeable_units_value) BY @timestamp, deployment\n| SORT @timestamp DESC\n| LIMIT 5000" }, "timeField": "@timestamp" } @@ -239,7 +230,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_cluster_cost_lookup\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL total_chargeable_units_value = COALESCE(total_chargeable_units, total_ecu) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0), deployment = CONCAT(deployment_name, \" (\", LEFT(deployment_id, 6), \")\")\n| STATS `Normalized cost` = SUM(total_chargeable_units_value) BY @timestamp, deployment\n| SORT @timestamp DESC\n| LIMIT 5000" + "esql": "FROM billing_cluster_cost_lookup\n| WHERE (CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group)))\n| WHERE (CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name)))\n| WHERE (CASE(?cost_category_selected IS NULL OR MV_INTERSECTS(?cost_category_selected, [\"All\"]), true, MV_INTERSECTS(?cost_category_selected, cost_category)))\n| WHERE (CASE(?cost_type_selected IS NULL OR MV_INTERSECTS(?cost_type_selected, [\"All\"]), true, MV_INTERSECTS(?cost_type_selected, cost_type)))\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL total_chargeable_units_value = COALESCE(total_chargeable_units, total_ecu) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0),\n deployment = CONCAT(deployment_name, \" (\", LEFT(deployment_id, 6), \")\")\n| STATS `Normalized cost` = SUM(total_chargeable_units_value) BY @timestamp, deployment\n| SORT @timestamp DESC\n| LIMIT 5000" }, "visualization": { "axisTitlesVisibilitySettings": { @@ -247,12 +238,12 @@ "yLeft": true, "yRight": true }, - "fittingFunction": "Linear", "gridlinesVisibilitySettings": { "x": true, "yLeft": true, "yRight": true }, + "hideEndzones": true, "labelsOrientation": { "x": 0, "yLeft": 0, @@ -261,7 +252,7 @@ "layers": [ { "accessors": [ - "Normalized cost" + "bar_stacked_y_0" ], "colorMapping": { "assignments": [], @@ -283,34 +274,52 @@ } ] }, - "layerId": "aa55ae31-395f-4738-b9fd-3da1d649d640", + "layerId": "bar_stacked_0", "layerType": "data", "seriesType": "bar_stacked", - "splitAccessor": "deployment", "splitAccessors": [ - "deployment" + "bar_stacked_breakdown" ], - "xAccessor": "@timestamp" + "xAccessor": "bar_stacked_x", + "yConfig": [ + { + "axisMode": "left", + "forAccessor": "bar_stacked_y_0" + } + ] } ], "legend": { "isVisible": true, + "maxLines": 1, "position": "right" }, + "minBarHeight": 1, "preferredSeriesType": "bar_stacked", + "showCurrentTimeMarker": false, "tickLabelsVisibilitySettings": { "x": true, "yLeft": true, "yRight": true }, - "valueLabels": "hide" + "valueLabels": "hide", + "xExtent": { + "mode": "dataBounds", + "niceValues": false + }, + "yLeftExtent": { + "mode": "full", + "niceValues": true + }, + "yLeftScale": "linear" } }, - "title": "Total Cloud Billing Over Time", + "title": "", "version": 2, "visualizationType": "lnsXY" }, "description": "Full deployment bill: the sum of ALL billing SKUs (data tiers, ML, Kibana, snapshots, data transfer and on-prem nodes) converted at the configured rate. This is the highest-level total. Tier and data-stream panels show only the allocatable data-tier pool, so they sum to less than this figure.", + "drilldowns": [], "time_range": { "from": "now-6M", "to": "now" @@ -322,23 +331,22 @@ "i": "ea35e78f-9951-40cf-8164-df6e21289660", "w": 48, "x": 0, - "y": 2 + "y": 4 }, "panelIndex": "ea35e78f-9951-40cf-8164-df6e21289660", - "type": "lens" + "type": "vis" }, { "embeddableConfig": { "attributes": { - "description": "Full billing spend per chargeback group over time. Each group is defined by the chargeback_group tag set on ESS deployments. All SKUs (data tiers, ML, Kibana, snapshots, transfer) contribute \u2014 the stack totals equal the full organisation bill. FinOps value: shows which teams or business units are growing their spend fastest.", "references": [], "state": { "adHocDataViews": { - "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "billing_cluster_cost_lookup-e006199ac8613df2dbb3d0652bd84eaf8be5e014cd9a882014652dba6857df62": { "allowHidden": false, "allowNoIndex": false, "fieldFormats": {}, - "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "id": "billing_cluster_cost_lookup-e006199ac8613df2dbb3d0652bd84eaf8be5e014cd9a882014652dba6857df62", "managed": false, "name": "billing_cluster_cost_lookup", "runtimeFieldMap": {}, @@ -352,48 +360,41 @@ "textBased": { "indexPatternRefs": [ { - "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "id": "billing_cluster_cost_lookup-e006199ac8613df2dbb3d0652bd84eaf8be5e014cd9a882014652dba6857df62", "timeField": "@timestamp", "title": "billing_cluster_cost_lookup" } ], "layers": { - "c6398250-abb1-4935-a3a8-41af3c8ea057": { + "bar_stacked_0": { "columns": [ { - "columnId": "@timestamp", - "customLabel": false, + "columnId": "bar_stacked_x", "fieldName": "@timestamp", - "label": "@timestamp", "meta": { - "esType": "date", "type": "date" } }, { - "columnId": "deployment_group", - "customLabel": false, - "fieldName": "deployment_group", - "label": "Deployment Group", + "columnId": "bar_stacked_y_0", + "fieldName": "Normalized cost", "meta": { - "esType": "keyword", - "type": "string" + "type": "number" } }, { - "columnId": "Normalized cost", - "customLabel": false, - "fieldName": "Normalized cost", - "label": "Normalized cost", + "columnId": "bar_stacked_breakdown", + "customLabel": true, + "fieldName": "deployment_group", + "label": "Deployment group", "meta": { - "esType": "double", - "type": "number" + "type": "string" } } ], - "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "index": "billing_cluster_cost_lookup-e006199ac8613df2dbb3d0652bd84eaf8be5e014cd9a882014652dba6857df62", "query": { - "esql": "FROM billing_cluster_cost_lookup\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL total_chargeable_units_value = COALESCE(total_chargeable_units, total_ecu) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0)\n| STATS `Normalized cost` = SUM(total_chargeable_units_value) BY @timestamp, deployment_group\n| SORT @timestamp DESC\n| LIMIT 5000" + "esql": "FROM billing_cluster_cost_lookup\n | WHERE\n CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group))\n | WHERE\n CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name))\n | WHERE\n CASE(?cost_category_selected IS NULL OR MV_INTERSECTS(?cost_category_selected, [\"All\"]), true, MV_INTERSECTS(?cost_category_selected, cost_category))\n | WHERE\n CASE(?cost_type_selected IS NULL OR MV_INTERSECTS(?cost_type_selected, [\"All\"]), true, MV_INTERSECTS(?cost_type_selected, cost_type))\n | LOOKUP JOIN chargeback_conf_lookup\n ON\n @timestamp >= conf_start_date AND\n @timestamp <= conf_end_date\n | EVAL\n _units = CASE(\n total_chargeable_units IS NOT NULL,\n total_chargeable_units,\n total_ecu),\n _rate = CASE(\n conf_chargeable_unit_rate IS NOT NULL,\n conf_chargeable_unit_rate,\n CASE(\n conf_ecu_rate IS NOT NULL,\n conf_ecu_rate,\n 1.0)),\n total_chargeable_units_value = _units * _rate\n | STATS\n `Normalized cost` =\n SUM(total_chargeable_units_value)\n BY @timestamp, deployment_group\n | SORT @timestamp DESC\n | LIMIT 5000" }, "timeField": "@timestamp" } @@ -403,7 +404,7 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_cluster_cost_lookup\n| LOOKUP JOIN chargeback_conf_lookup ON @timestamp >= conf_start_date AND @timestamp <= conf_end_date\n| EVAL total_chargeable_units_value = COALESCE(total_chargeable_units, total_ecu) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0)\n| STATS `Normalized cost` = SUM(total_chargeable_units_value) BY @timestamp, deployment_group\n| SORT @timestamp DESC\n| LIMIT 5000" + "esql": "FROM billing_cluster_cost_lookup\n | WHERE\n CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group))\n | WHERE\n CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name))\n | WHERE\n CASE(?cost_category_selected IS NULL OR MV_INTERSECTS(?cost_category_selected, [\"All\"]), true, MV_INTERSECTS(?cost_category_selected, cost_category))\n | WHERE\n CASE(?cost_type_selected IS NULL OR MV_INTERSECTS(?cost_type_selected, [\"All\"]), true, MV_INTERSECTS(?cost_type_selected, cost_type))\n | LOOKUP JOIN chargeback_conf_lookup\n ON\n @timestamp >= conf_start_date AND\n @timestamp <= conf_end_date\n | EVAL\n _units = CASE(\n total_chargeable_units IS NOT NULL,\n total_chargeable_units,\n total_ecu),\n _rate = CASE(\n conf_chargeable_unit_rate IS NOT NULL,\n conf_chargeable_unit_rate,\n CASE(\n conf_ecu_rate IS NOT NULL,\n conf_ecu_rate,\n 1.0)),\n total_chargeable_units_value = _units * _rate\n | STATS\n `Normalized cost` =\n SUM(total_chargeable_units_value)\n BY @timestamp, deployment_group\n | SORT @timestamp DESC\n | LIMIT 5000" }, "visualization": { "axisTitlesVisibilitySettings": { @@ -411,12 +412,12 @@ "yLeft": true, "yRight": true }, - "fittingFunction": "Linear", "gridlinesVisibilitySettings": { "x": true, "yLeft": true, "yRight": true }, + "hideEndzones": true, "labelsOrientation": { "x": 0, "yLeft": 0, @@ -425,7 +426,7 @@ "layers": [ { "accessors": [ - "Normalized cost" + "bar_stacked_y_0" ], "colorMapping": { "assignments": [], @@ -447,39 +448,57 @@ } ] }, - "layerId": "c6398250-abb1-4935-a3a8-41af3c8ea057", + "layerId": "bar_stacked_0", "layerType": "data", "seriesType": "bar_stacked", - "splitAccessor": "deployment_group", "splitAccessors": [ - "deployment_group" + "bar_stacked_breakdown" ], - "xAccessor": "@timestamp" + "xAccessor": "bar_stacked_x", + "yConfig": [ + { + "axisMode": "left", + "forAccessor": "bar_stacked_y_0" + } + ] } ], "legend": { "isVisible": true, + "maxLines": 1, "position": "right" }, + "minBarHeight": 1, "preferredSeriesType": "bar_stacked", + "showCurrentTimeMarker": false, "tickLabelsVisibilitySettings": { "x": true, "yLeft": true, "yRight": true }, - "valueLabels": "hide" + "valueLabels": "hide", + "xExtent": { + "mode": "dataBounds", + "niceValues": false + }, + "yLeftExtent": { + "mode": "full", + "niceValues": true + }, + "yLeftScale": "linear" } }, - "title": "Normalized cost trend by deployment group", + "title": "", "version": 2, "visualizationType": "lnsXY" }, "description": "Full deployment bill: the sum of ALL billing SKUs (data tiers, ML, Kibana, snapshots, data transfer and on-prem nodes) converted at the configured rate. This is the highest-level total. Tier and data-stream panels show only the allocatable data-tier pool, so they sum to less than this figure.", + "drilldowns": [], "time_range": { "from": "now-6M", "to": "now" }, - "title": "Normalized Cost Trend by Deployment Group" + "title": "Normalized cost trend by deployment group" }, "gridData": { "h": 14, @@ -487,106 +506,22 @@ "sectionId": "835374ca-64a0-4fcf-9f24-e301a28ebf8c", "w": 48, "x": 0, - "y": 3 - }, - "panelIndex": "3aad5559-8340-4a36-89de-5f1f1ec9af88", - "type": "lens", - "title": "Normalized cost trend by deployment group" - }, - { - "embeddableConfig": { - "enhancements": { - "dynamicActions": { - "events": [] - } - }, - "savedVis": { - "data": { - "aggs": [], - "searchSource": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } - }, - "description": "", - "id": "", - "params": { - "fontSize": 12, - "markdown": "## Deployment group statistics", - "openLinksInNewTab": false - }, - "title": "", - "type": "markdown", - "uiState": {} - } - }, - "gridData": { - "h": 3, - "i": "5640ee53-9479-465c-be2b-3a6000c2ce26", - "sectionId": "835374ca-64a0-4fcf-9f24-e301a28ebf8c", - "w": 36, - "x": 0, "y": 0 }, - "panelIndex": "5640ee53-9479-465c-be2b-3a6000c2ce26", - "type": "visualization" - }, - { - "embeddableConfig": { - "enhancements": { - "dynamicActions": { - "events": [] - } - }, - "savedVis": { - "data": { - "aggs": [], - "searchSource": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } - }, - "description": "", - "id": "", - "params": { - "fontSize": 12, - "markdown": "**Source:** `billing_cluster_cost_lookup` \u2014 full invoice, all SKUs.\n\nCost is normalized using the configured chargeable-unit rate. Groups are derived from the `chargeback_group` tag on ESS deployments. Totals in this section match the full deployment bill shown above.", - "openLinksInNewTab": false - }, - "title": "", - "type": "markdown", - "uiState": {} - } - }, - "gridData": { - "h": 3, - "i": "f0ce2200-54b4-5994-44c1-d5ea71d8035a", - "sectionId": "835374ca-64a0-4fcf-9f24-e301a28ebf8c", - "w": 12, - "x": 36, - "y": 0 - }, - "panelIndex": "f0ce2200-54b4-5994-44c1-d5ea71d8035a", - "type": "visualization" + "panelIndex": "3aad5559-8340-4a36-89de-5f1f1ec9af88", + "type": "vis" }, { "embeddableConfig": { "attributes": { - "description": "Total normalized spend per chargeback group for the selected period, ranked highest to lowest. Covers the full invoice \u2014 all SKU families included. FinOps value: the primary allocation table for internal chargeback \u2014 each group's figure is what you charge back to that team or cost centre.", "references": [], "state": { "adHocDataViews": { - "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "billing_cluster_cost_lookup-d39a45bc1a298220fedba203727af5583d4fb9e91738059e743fffd430f7afef": { "allowHidden": false, "allowNoIndex": false, "fieldFormats": {}, - "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "id": "billing_cluster_cost_lookup-d39a45bc1a298220fedba203727af5583d4fb9e91738059e743fffd430f7afef", "managed": false, "name": "billing_cluster_cost_lookup", "runtimeFieldMap": {}, @@ -600,44 +535,33 @@ "textBased": { "indexPatternRefs": [ { - "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "id": "billing_cluster_cost_lookup-d39a45bc1a298220fedba203727af5583d4fb9e91738059e743fffd430f7afef", "timeField": "@timestamp", "title": "billing_cluster_cost_lookup" } ], "layers": { - "b9429019-7f45-4333-8551-6642636e2db9": { + "layer_0": { "columns": [ { - "columnId": "Deployment Group", - "customLabel": false, + "columnId": "datatable_accessor_row_0", "fieldName": "Deployment Group", - "label": "Deployment Group", "meta": { - "esType": "keyword", "type": "string" } }, { - "columnId": "329d9b50-9e64-4dc6-9cc2-d7b120709522", + "columnId": "datatable_accessor_metric_0", "fieldName": "Normalized cost", - "label": "Normalized cost", + "inMetricDimension": true, "meta": { - "esType": "double", - "params": { - "id": "number" - }, - "sourceParams": { - "indexPattern": "billing_cluster_cost_lookup", - "sourceField": "Normalized cost" - }, "type": "number" } } ], - "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "index": "billing_cluster_cost_lookup-d39a45bc1a298220fedba203727af5583d4fb9e91738059e743fffd430f7afef", "query": { - "esql": "FROM billing_cluster_cost_lookup\n| EVAL ts = @timestamp\n| LOOKUP JOIN chargeback_conf_lookup ON ts >= conf_start_date AND ts <= conf_end_date\n| STATS `Normalized cost` = SUM(COALESCE(total_chargeable_units, total_ecu) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0)) BY deployment_group | RENAME deployment_group AS `Deployment Group`\n| SORT `Normalized cost` DESC" + "esql": "FROM billing_cluster_cost_lookup\n | WHERE\n CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group))\n | WHERE\n CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name))\n | WHERE\n CASE(?cost_category_selected IS NULL OR MV_INTERSECTS(?cost_category_selected, [\"All\"]), true, MV_INTERSECTS(?cost_category_selected, cost_category))\n | WHERE\n CASE(?cost_type_selected IS NULL OR MV_INTERSECTS(?cost_type_selected, [\"All\"]), true, MV_INTERSECTS(?cost_type_selected, cost_type))\n | EVAL `ts` = @timestamp\n | LOOKUP JOIN chargeback_conf_lookup\n ON `ts` >= conf_start_date AND `ts` <= conf_end_date\n | EVAL\n _units = CASE(\n total_chargeable_units IS NOT NULL,\n total_chargeable_units,\n total_ecu),\n _rate = CASE(\n conf_chargeable_unit_rate IS NOT NULL,\n conf_chargeable_unit_rate,\n CASE(\n conf_ecu_rate IS NOT NULL,\n conf_ecu_rate,\n 1.0))\n | STATS\n `Normalized cost` = SUM(_units * _rate)\n BY deployment_group\n | RENAME deployment_group AS `Deployment Group`\n | SORT `Normalized cost` DESC" }, "timeField": "@timestamp" } @@ -647,43 +571,33 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_cluster_cost_lookup\n| EVAL ts = @timestamp\n| LOOKUP JOIN chargeback_conf_lookup ON ts >= conf_start_date AND ts <= conf_end_date\n| STATS `Normalized cost` = SUM(COALESCE(total_chargeable_units, total_ecu) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0)) BY deployment_group | RENAME deployment_group AS `Deployment Group`\n| SORT `Normalized cost` DESC" + "esql": "FROM billing_cluster_cost_lookup\n | WHERE\n CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group))\n | WHERE\n CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name))\n | WHERE\n CASE(?cost_category_selected IS NULL OR MV_INTERSECTS(?cost_category_selected, [\"All\"]), true, MV_INTERSECTS(?cost_category_selected, cost_category))\n | WHERE\n CASE(?cost_type_selected IS NULL OR MV_INTERSECTS(?cost_type_selected, [\"All\"]), true, MV_INTERSECTS(?cost_type_selected, cost_type))\n | EVAL `ts` = @timestamp\n | LOOKUP JOIN chargeback_conf_lookup\n ON `ts` >= conf_start_date AND `ts` <= conf_end_date\n | EVAL\n _units = CASE(\n total_chargeable_units IS NOT NULL,\n total_chargeable_units,\n total_ecu),\n _rate = CASE(\n conf_chargeable_unit_rate IS NOT NULL,\n conf_chargeable_unit_rate,\n CASE(\n conf_ecu_rate IS NOT NULL,\n conf_ecu_rate,\n 1.0))\n | STATS\n `Normalized cost` = SUM(_units * _rate)\n BY deployment_group\n | RENAME deployment_group AS `Deployment Group`\n | SORT `Normalized cost` DESC" }, "visualization": { "columns": [ { - "columnId": "Deployment Group", + "columnId": "datatable_accessor_row_0", + "isMetric": false, "isTransposed": false }, { - "columnId": "329d9b50-9e64-4dc6-9cc2-d7b120709522", + "columnId": "datatable_accessor_metric_0", "isMetric": true, "isTransposed": false, "summaryRow": "sum" } ], - "layerId": "b9429019-7f45-4333-8551-6642636e2db9", + "layerId": "layer_0", "layerType": "data" } }, - "title": "Total normalized cost by deployment group", + "title": "", "version": 2, "visualizationType": "lnsDatatable" }, "description": "Full deployment bill: the sum of ALL billing SKUs (data tiers, ML, Kibana, snapshots, data transfer and on-prem nodes) converted at the configured rate. This is the highest-level total. Tier and data-stream panels show only the allocatable data-tier pool, so they sum to less than this figure.", - "enhancements": { - "dynamicActions": { - "events": [] - } - }, - "filters": [], - "query": { - "esql": "FROM billing_cluster_cost_lookup\n| EVAL ts = @timestamp\n| LOOKUP JOIN chargeback_conf_lookup ON ts >= conf_start_date AND ts <= conf_end_date\n| STATS `Normalized cost` = SUM(COALESCE(total_chargeable_units, total_ecu) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0)) BY deployment_group | RENAME deployment_group AS `Deployment Group`\n| SORT `Normalized cost` DESC" - }, - "syncColors": false, - "syncCursor": true, - "syncTooltips": false, - "title": "Total Cost by Deployment Group" + "drilldowns": [], + "title": "Total normalized cost by deployment group" }, "gridData": { "h": 14, @@ -691,24 +605,22 @@ "sectionId": "835374ca-64a0-4fcf-9f24-e301a28ebf8c", "w": 16, "x": 0, - "y": 17 + "y": 14 }, "panelIndex": "7a41cee2-7043-44ee-b144-a0e8e92f87b2", - "type": "lens", - "title": "Total normalized cost by deployment group" + "type": "vis" }, { "embeddableConfig": { "attributes": { - "description": "Cost breakdown per chargeback group split by FinOps category (data_tier, platform, transfer, snapshot, onprem). Total per group equals that group's full invoice amount. FinOps value: reveals whether a group's spend is dominated by compute (data_tier) or operational overhead (snapshots, egress transfer) \u2014 helps prioritise where optimisation conversations should start.", "references": [], "state": { "adHocDataViews": { - "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "billing_cluster_cost_lookup-ddb7ec165589d8add4ba3c8b3be30bae3907ce0add78b2bb074d96f2e3c11494": { "allowHidden": false, "allowNoIndex": false, "fieldFormats": {}, - "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "id": "billing_cluster_cost_lookup-ddb7ec165589d8add4ba3c8b3be30bae3907ce0add78b2bb074d96f2e3c11494", "managed": false, "name": "billing_cluster_cost_lookup", "runtimeFieldMap": {}, @@ -722,60 +634,40 @@ "textBased": { "indexPatternRefs": [ { - "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "id": "billing_cluster_cost_lookup-ddb7ec165589d8add4ba3c8b3be30bae3907ce0add78b2bb074d96f2e3c11494", "timeField": "@timestamp", "title": "billing_cluster_cost_lookup" } ], "layers": { - "b410b660-726b-4ecd-943b-c66510f2ce58": { + "layer_0": { "columns": [ { - "columnId": "Deployment Group", - "customLabel": false, + "columnId": "datatable_accessor_row_0", "fieldName": "Deployment Group", - "label": "Deployment Group", "meta": { - "esType": "keyword", "type": "string" } }, { - "columnId": "2f5e2665-1370-45f6-a2cd-973250b4d9d5", + "columnId": "datatable_accessor_row_1", "fieldName": "Cost Category", - "label": "Cost Category", "meta": { - "esType": "keyword", - "params": { - "id": "string" - }, - "sourceParams": { - "indexPattern": "billing_cluster_cost_lookup", - "sourceField": "Cost Category" - }, "type": "string" } }, { - "columnId": "f789a84b-618c-4d7d-9b16-fe5c39a2cc91", + "columnId": "datatable_accessor_metric_0", "fieldName": "Normalized cost", - "label": "Normalized cost", + "inMetricDimension": true, "meta": { - "esType": "double", - "params": { - "id": "number" - }, - "sourceParams": { - "indexPattern": "billing_cluster_cost_lookup", - "sourceField": "Normalized cost" - }, "type": "number" } } ], - "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "index": "billing_cluster_cost_lookup-ddb7ec165589d8add4ba3c8b3be30bae3907ce0add78b2bb074d96f2e3c11494", "query": { - "esql": "FROM billing_cluster_cost_lookup\n | EVAL `ts` = @timestamp\n | LOOKUP JOIN chargeback_conf_lookup\n ON\n `ts` >= conf_start_date AND\n `ts` <= conf_end_date\n | STATS\n `Normalized cost` =\n SUM(\n COALESCE(total_chargeable_units, total_ecu) *\n COALESCE(\n conf_chargeable_unit_rate,\n conf_ecu_rate,\n 1.0))\n BY deployment_group, cost_category\n | RENAME deployment_group AS `Deployment Group`, cost_category as `Cost Category`\n | SORT `Normalized cost` DESC" + "esql": "FROM billing_cluster_cost_lookup\n | WHERE\n CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group))\n | WHERE\n CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name))\n | WHERE\n CASE(?cost_category_selected IS NULL OR MV_INTERSECTS(?cost_category_selected, [\"All\"]), true, MV_INTERSECTS(?cost_category_selected, cost_category))\n | WHERE\n CASE(?cost_type_selected IS NULL OR MV_INTERSECTS(?cost_type_selected, [\"All\"]), true, MV_INTERSECTS(?cost_type_selected, cost_type))\n | EVAL `ts` = @timestamp\n | LOOKUP JOIN chargeback_conf_lookup\n ON `ts` >= conf_start_date AND `ts` <= conf_end_date\n | EVAL\n _units = CASE(\n total_chargeable_units IS NOT NULL,\n total_chargeable_units,\n total_ecu),\n _rate = CASE(\n conf_chargeable_unit_rate IS NOT NULL,\n conf_chargeable_unit_rate,\n CASE(\n conf_ecu_rate IS NOT NULL,\n conf_ecu_rate,\n 1.0))\n | STATS\n `Normalized cost` = SUM(_units * _rate)\n BY deployment_group, cost_category\n | RENAME\n deployment_group AS `Deployment Group`,\n cost_category AS `Cost Category`\n | SORT `Normalized cost` DESC" }, "timeField": "@timestamp" } @@ -785,48 +677,38 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_cluster_cost_lookup\n | EVAL `ts` = @timestamp\n | LOOKUP JOIN chargeback_conf_lookup\n ON\n `ts` >= conf_start_date AND\n `ts` <= conf_end_date\n | STATS\n `Normalized cost` =\n SUM(\n COALESCE(total_chargeable_units, total_ecu) *\n COALESCE(\n conf_chargeable_unit_rate,\n conf_ecu_rate,\n 1.0))\n BY deployment_group, cost_category\n | RENAME deployment_group AS `Deployment Group`, cost_category as `Cost Category`\n | SORT `Normalized cost` DESC" + "esql": "FROM billing_cluster_cost_lookup\n | WHERE\n CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group))\n | WHERE\n CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name))\n | WHERE\n CASE(?cost_category_selected IS NULL OR MV_INTERSECTS(?cost_category_selected, [\"All\"]), true, MV_INTERSECTS(?cost_category_selected, cost_category))\n | WHERE\n CASE(?cost_type_selected IS NULL OR MV_INTERSECTS(?cost_type_selected, [\"All\"]), true, MV_INTERSECTS(?cost_type_selected, cost_type))\n | EVAL `ts` = @timestamp\n | LOOKUP JOIN chargeback_conf_lookup\n ON `ts` >= conf_start_date AND `ts` <= conf_end_date\n | EVAL\n _units = CASE(\n total_chargeable_units IS NOT NULL,\n total_chargeable_units,\n total_ecu),\n _rate = CASE(\n conf_chargeable_unit_rate IS NOT NULL,\n conf_chargeable_unit_rate,\n CASE(\n conf_ecu_rate IS NOT NULL,\n conf_ecu_rate,\n 1.0))\n | STATS\n `Normalized cost` = SUM(_units * _rate)\n BY deployment_group, cost_category\n | RENAME\n deployment_group AS `Deployment Group`,\n cost_category AS `Cost Category`\n | SORT `Normalized cost` DESC" }, "visualization": { "columns": [ { - "columnId": "Deployment Group", + "columnId": "datatable_accessor_row_0", + "isMetric": false, "isTransposed": false }, { - "columnId": "2f5e2665-1370-45f6-a2cd-973250b4d9d5", + "columnId": "datatable_accessor_row_1", "isMetric": false, "isTransposed": false }, { - "columnId": "f789a84b-618c-4d7d-9b16-fe5c39a2cc91", + "columnId": "datatable_accessor_metric_0", "isMetric": true, "isTransposed": false, "summaryRow": "sum" } ], - "layerId": "b410b660-726b-4ecd-943b-c66510f2ce58", + "layerId": "layer_0", "layerType": "data" } }, - "title": "Normalized cost by deployment group and cost category", + "title": "", "version": 2, "visualizationType": "lnsDatatable" }, "description": "Full deployment bill: the sum of ALL billing SKUs (data tiers, ML, Kibana, snapshots, data transfer and on-prem nodes) converted at the configured rate. This is the highest-level total. Tier and data-stream panels show only the allocatable data-tier pool, so they sum to less than this figure.", - "enhancements": { - "dynamicActions": { - "events": [] - } - }, - "filters": [], - "query": { - "esql": "FROM billing_cluster_cost_lookup\n | EVAL `ts` = @timestamp\n | LOOKUP JOIN chargeback_conf_lookup\n ON\n `ts` >= conf_start_date AND\n `ts` <= conf_end_date\n | STATS\n `Normalized cost` =\n SUM(\n COALESCE(total_chargeable_units, total_ecu) *\n COALESCE(\n conf_chargeable_unit_rate,\n conf_ecu_rate,\n 1.0))\n BY deployment_group, cost_category\n | RENAME deployment_group AS `Deployment Group`, cost_category as `Cost Category`\n | SORT `Normalized cost` DESC" - }, - "syncColors": false, - "syncCursor": true, - "syncTooltips": false, - "title": "Cost by Group and FinOps Category" + "drilldowns": [], + "title": "Normalized cost by deployment group and cost category" }, "gridData": { "h": 14, @@ -834,24 +716,22 @@ "sectionId": "835374ca-64a0-4fcf-9f24-e301a28ebf8c", "w": 16, "x": 16, - "y": 17 + "y": 14 }, "panelIndex": "4434fef9-3ae8-44c5-a0b2-f0096909fe13", - "type": "lens", - "title": "Normalized cost by deployment group and cost category" + "type": "vis" }, { "embeddableConfig": { "attributes": { - "description": "Proportional share of total spend, segmented by chargeback group and FinOps category. FinOps value: at-a-glance view for finance stakeholders showing which groups own which share of the bill and what type of cost it is.", "references": [], "state": { "adHocDataViews": { - "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "billing_cluster_cost_lookup-ddb7ec165589d8add4ba3c8b3be30bae3907ce0add78b2bb074d96f2e3c11494": { "allowHidden": false, "allowNoIndex": false, "fieldFormats": {}, - "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "id": "billing_cluster_cost_lookup-ddb7ec165589d8add4ba3c8b3be30bae3907ce0add78b2bb074d96f2e3c11494", "managed": false, "name": "billing_cluster_cost_lookup", "runtimeFieldMap": {}, @@ -865,63 +745,48 @@ "textBased": { "indexPatternRefs": [ { - "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "id": "billing_cluster_cost_lookup-ddb7ec165589d8add4ba3c8b3be30bae3907ce0add78b2bb074d96f2e3c11494", "timeField": "@timestamp", "title": "billing_cluster_cost_lookup" } ], "layers": { - "b410b660-726b-4ecd-943b-c66510f2ce58": { + "layer_0": { "columns": [ { - "columnId": "Normalized cost", - "customLabel": false, + "columnId": "partition_value_accessor_metric_0", "fieldName": "Normalized cost", - "inMetricDimension": true, - "label": "Normalized cost", "meta": { - "esType": "double", "type": "number" }, "params": { "format": { "id": "number", "params": { + "compact": false, "decimals": 2 } } } }, { - "columnId": "Deployment Group", - "customLabel": false, + "columnId": "partition_value_accessor_group_by_0", "fieldName": "Deployment Group", - "label": "Deployment Group", "meta": { - "esType": "keyword", "type": "string" } }, { - "columnId": "2f5e2665-1370-45f6-a2cd-973250b4d9d5", + "columnId": "partition_value_accessor_group_by_1", "fieldName": "Cost Category", - "label": "Cost Category", "meta": { - "esType": "keyword", - "params": { - "id": "string" - }, - "sourceParams": { - "indexPattern": "billing_cluster_cost_lookup", - "sourceField": "Cost Category" - }, "type": "string" } } ], - "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "index": "billing_cluster_cost_lookup-ddb7ec165589d8add4ba3c8b3be30bae3907ce0add78b2bb074d96f2e3c11494", "query": { - "esql": "FROM billing_cluster_cost_lookup\n | EVAL `ts` = @timestamp\n | LOOKUP JOIN chargeback_conf_lookup\n ON\n `ts` >= conf_start_date AND\n `ts` <= conf_end_date\n | STATS\n `Normalized cost` =\n SUM(\n COALESCE(total_chargeable_units, total_ecu) *\n COALESCE(\n conf_chargeable_unit_rate,\n conf_ecu_rate,\n 1.0))\n BY deployment_group, cost_category\n | RENAME deployment_group AS `Deployment Group`, cost_category as `Cost Category`\n | SORT `Normalized cost` DESC" + "esql": "FROM billing_cluster_cost_lookup\n | WHERE\n CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group))\n | WHERE\n CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name))\n | WHERE\n CASE(?cost_category_selected IS NULL OR MV_INTERSECTS(?cost_category_selected, [\"All\"]), true, MV_INTERSECTS(?cost_category_selected, cost_category))\n | WHERE\n CASE(?cost_type_selected IS NULL OR MV_INTERSECTS(?cost_type_selected, [\"All\"]), true, MV_INTERSECTS(?cost_type_selected, cost_type))\n | EVAL `ts` = @timestamp\n | LOOKUP JOIN chargeback_conf_lookup\n ON `ts` >= conf_start_date AND `ts` <= conf_end_date\n | EVAL\n _units = CASE(\n total_chargeable_units IS NOT NULL,\n total_chargeable_units,\n total_ecu),\n _rate = CASE(\n conf_chargeable_unit_rate IS NOT NULL,\n conf_chargeable_unit_rate,\n CASE(\n conf_ecu_rate IS NOT NULL,\n conf_ecu_rate,\n 1.0))\n | STATS\n `Normalized cost` = SUM(_units * _rate)\n BY deployment_group, cost_category\n | RENAME\n deployment_group AS `Deployment Group`,\n cost_category AS `Cost Category`\n | SORT `Normalized cost` DESC" }, "timeField": "@timestamp" } @@ -931,35 +796,59 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_cluster_cost_lookup\n | EVAL `ts` = @timestamp\n | LOOKUP JOIN chargeback_conf_lookup\n ON\n `ts` >= conf_start_date AND\n `ts` <= conf_end_date\n | STATS\n `Normalized cost` =\n SUM(\n COALESCE(total_chargeable_units, total_ecu) *\n COALESCE(\n conf_chargeable_unit_rate,\n conf_ecu_rate,\n 1.0))\n BY deployment_group, cost_category\n | RENAME deployment_group AS `Deployment Group`, cost_category as `Cost Category`\n | SORT `Normalized cost` DESC" + "esql": "FROM billing_cluster_cost_lookup\n | WHERE\n CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group))\n | WHERE\n CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name))\n | WHERE\n CASE(?cost_category_selected IS NULL OR MV_INTERSECTS(?cost_category_selected, [\"All\"]), true, MV_INTERSECTS(?cost_category_selected, cost_category))\n | WHERE\n CASE(?cost_type_selected IS NULL OR MV_INTERSECTS(?cost_type_selected, [\"All\"]), true, MV_INTERSECTS(?cost_type_selected, cost_type))\n | EVAL `ts` = @timestamp\n | LOOKUP JOIN chargeback_conf_lookup\n ON `ts` >= conf_start_date AND `ts` <= conf_end_date\n | EVAL\n _units = CASE(\n total_chargeable_units IS NOT NULL,\n total_chargeable_units,\n total_ecu),\n _rate = CASE(\n conf_chargeable_unit_rate IS NOT NULL,\n conf_chargeable_unit_rate,\n CASE(\n conf_ecu_rate IS NOT NULL,\n conf_ecu_rate,\n 1.0))\n | STATS\n `Normalized cost` = SUM(_units * _rate)\n BY deployment_group, cost_category\n | RENAME\n deployment_group AS `Deployment Group`,\n cost_category AS `Cost Category`\n | SORT `Normalized cost` DESC" }, "visualization": { "layers": [ { + "allowMultipleMetrics": false, "categoryDisplay": "default", - "layerId": "b410b660-726b-4ecd-943b-c66510f2ce58", + "collapseFns": {}, + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "default", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rules": [ + { + "type": "other" + } + ], + "touched": false + } + ] + }, + "layerId": "layer_0", "layerType": "data", "legendDisplay": "default", "metrics": [ - "Normalized cost" + "partition_value_accessor_metric_0" ], "nestedLegend": false, "numberDisplay": "percent", "primaryGroups": [ - "Deployment Group", - "2f5e2665-1370-45f6-a2cd-973250b4d9d5" - ] + "partition_value_accessor_group_by_0", + "partition_value_accessor_group_by_1" + ], + "truncateLegend": false } ], "shape": "treemap" } }, - "title": "Normalized cost share by deployment group and cost category", + "title": "", "version": 2, "visualizationType": "lnsPie" }, "description": "Full deployment bill: the sum of ALL billing SKUs (data tiers, ML, Kibana, snapshots, data transfer and on-prem nodes) converted at the configured rate. This is the highest-level total. Tier and data-stream panels show only the allocatable data-tier pool, so they sum to less than this figure.", - "title": "Cost Share by Group and Category" + "drilldowns": [], + "title": "Normalized cost share by deployment group and cost category" }, "gridData": { "h": 14, @@ -967,106 +856,22 @@ "sectionId": "835374ca-64a0-4fcf-9f24-e301a28ebf8c", "w": 16, "x": 32, - "y": 17 + "y": 14 }, "panelIndex": "9ba0ff86-0e96-4731-837f-6a86a046f229", - "type": "lens", - "title": "Normalized cost share by deployment group and cost category" - }, - { - "embeddableConfig": { - "enhancements": { - "dynamicActions": { - "events": [] - } - }, - "savedVis": { - "data": { - "aggs": [], - "searchSource": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } - }, - "description": "", - "id": "", - "params": { - "fontSize": 12, - "markdown": "## Component statistics", - "openLinksInNewTab": false - }, - "title": "", - "type": "markdown", - "uiState": {} - } - }, - "gridData": { - "h": 3, - "i": "c6cb5b7b-f118-4e20-aa27-97780caa7b9a", - "sectionId": "9c7631a4-b984-46e6-981d-d2888f50856f", - "w": 36, - "x": 0, - "y": 0 - }, - "panelIndex": "c6cb5b7b-f118-4e20-aa27-97780caa7b9a", - "type": "visualization" - }, - { - "embeddableConfig": { - "enhancements": { - "dynamicActions": { - "events": [] - } - }, - "savedVis": { - "data": { - "aggs": [], - "searchSource": { - "filter": [], - "query": { - "language": "kuery", - "query": "" - } - } - }, - "description": "", - "id": "", - "params": { - "fontSize": 12, - "markdown": "**Source:** `billing_cluster_cost_lookup` \u2014 full invoice, filtered to SKUs with a classified `cost_type`.\n\nBilling components (`cost_type`) are derived from the ESS SKU string: data tiers (datahot/datacontent, datawarm, datacold, datafrozen), platform (ml, kibana, apm), data-transfer, snapshots, and on-premises nodes. FinOps categories (`cost_category`) group these into broader buckets: data_tier, platform, transfer, snapshot, onprem, other.\n\n\u26a0\ufe0f If any SKUs could not be classified (`cost_type IS NULL`), those rows are excluded \u2014 totals here may be slightly lower than the full bill above.", - "openLinksInNewTab": false - }, - "title": "", - "type": "markdown", - "uiState": {} - } - }, - "gridData": { - "h": 3, - "i": "c4022318-f0fd-69eb-ac0a-8cf72c76c6ff", - "sectionId": "9c7631a4-b984-46e6-981d-d2888f50856f", - "w": 12, - "x": 36, - "y": 0 - }, - "panelIndex": "c4022318-f0fd-69eb-ac0a-8cf72c76c6ff", - "type": "visualization" + "type": "vis" }, { "embeddableConfig": { "attributes": { - "description": "Cost per deployment broken down by billing component (cost_type: e.g. datahot/datacontent, datawarm, ml, kibana, snapshots, data-transfer) and FinOps category (cost_category: data_tier, platform, transfer, snapshot, onprem). Filtered to SKUs where cost_type has been classified \u2014 rows where classification could not be derived from the SKU are excluded. FinOps value: the most granular billing breakdown available \u2014 use this to identify which specific infrastructure components drive cost for each deployment.", "references": [], "state": { "adHocDataViews": { - "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "billing_cluster_cost_lookup-8ce5a130c3b85a374d3cb1d16d8af6b736bfc81aacc3b3ae43c87b2deb23fbc7": { "allowHidden": false, "allowNoIndex": false, "fieldFormats": {}, - "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "id": "billing_cluster_cost_lookup-8ce5a130c3b85a374d3cb1d16d8af6b736bfc81aacc3b3ae43c87b2deb23fbc7", "managed": false, "name": "billing_cluster_cost_lookup", "runtimeFieldMap": {}, @@ -1080,136 +885,47 @@ "textBased": { "indexPatternRefs": [ { - "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "id": "billing_cluster_cost_lookup-8ce5a130c3b85a374d3cb1d16d8af6b736bfc81aacc3b3ae43c87b2deb23fbc7", "timeField": "@timestamp", "title": "billing_cluster_cost_lookup" } ], "layers": { - "30d04f87-c476-4b48-9f90-18bda3ce5ff2": { - "allColumns": [ - { - "columnId": "Normalized cost", - "customLabel": false, - "fieldName": "Normalized cost", - "inMetricDimension": true, - "label": "Normalized cost", - "meta": { - "esType": "double", - "type": "number" - }, - "params": { - "format": { - "id": "number", - "params": { - "decimals": 2 - } - } - } - }, - { - "columnId": "Deployment", - "customLabel": false, - "fieldName": "Deployment", - "label": "Deployment", - "meta": { - "esType": "keyword", - "type": "string" - } - }, - { - "columnId": "Cost Type", - "customLabel": false, - "fieldName": "Cost Type", - "label": "Cost Type", - "meta": { - "esType": "keyword", - "type": "string" - } - }, - { - "columnId": "38199b76-4208-4426-9bac-33521adc19e2", - "fieldName": "Cost Category", - "label": "Cost Category", - "meta": { - "esType": "keyword", - "params": { - "id": "string" - }, - "sourceParams": { - "indexPattern": "billing_cluster_cost_lookup", - "sourceField": "Cost Category" - }, - "type": "string" - } - }, - { - "columnId": "Cost Category", - "fieldName": "Cost Category", - "label": "Cost Category", - "meta": { - "esType": "keyword", - "type": "string" - } - } - ], + "layer_0": { "columns": [ { - "columnId": "Deployment", - "customLabel": false, + "columnId": "datatable_accessor_row_0", "fieldName": "Deployment", - "label": "Deployment", "meta": { - "esType": "keyword", "type": "string" } }, { - "columnId": "38199b76-4208-4426-9bac-33521adc19e2", + "columnId": "datatable_accessor_row_1", "fieldName": "Cost Category", - "label": "Cost Category", "meta": { - "esType": "keyword", - "params": { - "id": "string" - }, - "sourceParams": { - "indexPattern": "billing_cluster_cost_lookup", - "sourceField": "Cost Category" - }, "type": "string" } }, { - "columnId": "Cost Type", - "customLabel": false, + "columnId": "datatable_accessor_row_2", "fieldName": "Cost Type", - "label": "Cost Type", "meta": { - "esType": "keyword", "type": "string" } }, { - "columnId": "8bf8a2c9-aff9-438d-a069-a53a7b4207bb", + "columnId": "datatable_accessor_metric_0", "fieldName": "Normalized cost", - "label": "Normalized cost", + "inMetricDimension": true, "meta": { - "esType": "double", - "params": { - "id": "number" - }, - "sourceParams": { - "indexPattern": "billing_cluster_cost_lookup", - "sourceField": "Normalized cost" - }, "type": "number" } } ], - "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "index": "billing_cluster_cost_lookup-8ce5a130c3b85a374d3cb1d16d8af6b736bfc81aacc3b3ae43c87b2deb23fbc7", "query": { - "esql": "FROM billing_cluster_cost_lookup\n | WHERE cost_type IS NOT NULL\n | EVAL `ts` = @timestamp, `Cost Category` = cost_category, `Cost Type` = cost_type,\n Deployment = deployment_name\n | LOOKUP JOIN chargeback_conf_lookup\n ON `ts` >= conf_start_date AND `ts` <= conf_end_date\n | STATS\n `Normalized cost` =\n SUM(\n COALESCE(total_chargeable_units, total_ecu) *\n COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0))\n BY Deployment, `Cost Category`, `Cost Type`\n | SORT `Normalized cost` DESC" + "esql": "FROM billing_cluster_cost_lookup\n | WHERE cost_type IS NOT NULL\n | WHERE\n CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group))\n | WHERE\n CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name))\n | WHERE\n CASE(?cost_category_selected IS NULL OR MV_INTERSECTS(?cost_category_selected, [\"All\"]), true, MV_INTERSECTS(?cost_category_selected, cost_category))\n | WHERE\n CASE(?cost_type_selected IS NULL OR MV_INTERSECTS(?cost_type_selected, [\"All\"]), true, MV_INTERSECTS(?cost_type_selected, cost_type))\n | EVAL\n `ts` = @timestamp,\n `Cost Category` = cost_category,\n `Cost Type` = cost_type,\n Deployment = deployment_name\n | LOOKUP JOIN chargeback_conf_lookup\n ON `ts` >= conf_start_date AND `ts` <= conf_end_date\n | EVAL\n _units = CASE(total_chargeable_units IS NOT NULL, total_chargeable_units, total_ecu),\n _rate = CASE(conf_chargeable_unit_rate IS NOT NULL, conf_chargeable_unit_rate,\n CASE(conf_ecu_rate IS NOT NULL, conf_ecu_rate, 1.0))\n | STATS\n `Normalized cost` = SUM(_units * _rate)\n BY Deployment, `Cost Category`, `Cost Type`\n | SORT `Normalized cost` DESC" }, "timeField": "@timestamp" } @@ -1219,53 +935,44 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_cluster_cost_lookup\n | WHERE cost_type IS NOT NULL\n | EVAL `ts` = @timestamp, `Cost Category` = cost_category, `Cost Type` = cost_type,\n Deployment = deployment_name\n | LOOKUP JOIN chargeback_conf_lookup\n ON `ts` >= conf_start_date AND `ts` <= conf_end_date\n | STATS\n `Normalized cost` =\n SUM(\n COALESCE(total_chargeable_units, total_ecu) *\n COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0))\n BY Deployment, `Cost Category`, `Cost Type`\n | SORT `Normalized cost` DESC" + "esql": "FROM billing_cluster_cost_lookup\n | WHERE cost_type IS NOT NULL\n | WHERE\n CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group))\n | WHERE\n CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name))\n | WHERE\n CASE(?cost_category_selected IS NULL OR MV_INTERSECTS(?cost_category_selected, [\"All\"]), true, MV_INTERSECTS(?cost_category_selected, cost_category))\n | WHERE\n CASE(?cost_type_selected IS NULL OR MV_INTERSECTS(?cost_type_selected, [\"All\"]), true, MV_INTERSECTS(?cost_type_selected, cost_type))\n | EVAL\n `ts` = @timestamp,\n `Cost Category` = cost_category,\n `Cost Type` = cost_type,\n Deployment = deployment_name\n | LOOKUP JOIN chargeback_conf_lookup\n ON `ts` >= conf_start_date AND `ts` <= conf_end_date\n | EVAL\n _units = CASE(total_chargeable_units IS NOT NULL, total_chargeable_units, total_ecu),\n _rate = CASE(conf_chargeable_unit_rate IS NOT NULL, conf_chargeable_unit_rate,\n CASE(conf_ecu_rate IS NOT NULL, conf_ecu_rate, 1.0))\n | STATS\n `Normalized cost` = SUM(_units * _rate)\n BY Deployment, `Cost Category`, `Cost Type`\n | SORT `Normalized cost` DESC" }, "visualization": { "columns": [ { - "columnId": "Deployment", + "columnId": "datatable_accessor_row_0", + "isMetric": false, "isTransposed": false, - "width": 309.66666666666663 + "width": 218.66666666666663 }, { - "columnId": "Cost Type", - "isTransposed": false + "columnId": "datatable_accessor_row_1", + "isMetric": false, + "isTransposed": false, + "width": 229 }, { - "columnId": "38199b76-4208-4426-9bac-33521adc19e2", + "columnId": "datatable_accessor_row_2", "isMetric": false, - "isTransposed": false, - "width": 187 + "isTransposed": false }, { - "columnId": "8bf8a2c9-aff9-438d-a069-a53a7b4207bb", + "columnId": "datatable_accessor_metric_0", "isMetric": true, "isTransposed": false } ], - "layerId": "30d04f87-c476-4b48-9f90-18bda3ce5ff2", + "layerId": "layer_0", "layerType": "data" } }, - "title": "Normalized cost by billing component and deployment", + "title": "", "version": 2, "visualizationType": "lnsDatatable" }, "description": "Full deployment bill: the sum of ALL billing SKUs (data tiers, ML, Kibana, snapshots, data transfer and on-prem nodes) converted at the configured rate. This is the highest-level total. Tier and data-stream panels show only the allocatable data-tier pool, so they sum to less than this figure.", - "enhancements": { - "dynamicActions": { - "events": [] - } - }, - "filters": [], - "query": { - "esql": "FROM billing_cluster_cost_lookup\n | WHERE cost_type IS NOT NULL\n | EVAL `ts` = @timestamp, `Cost Category` = cost_category, `Cost Type` = cost_type,\n Deployment = deployment_name\n | LOOKUP JOIN chargeback_conf_lookup\n ON `ts` >= conf_start_date AND `ts` <= conf_end_date\n | STATS\n `Normalized cost` =\n SUM(\n COALESCE(total_chargeable_units, total_ecu) *\n COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0))\n BY Deployment, `Cost Category`, `Cost Type`\n | SORT `Normalized cost` DESC" - }, - "syncColors": false, - "syncCursor": true, - "syncTooltips": false, - "title": "Cost by Billing Component and Deployment" + "drilldowns": [], + "title": "Normalized cost by billing component and deployment" }, "gridData": { "h": 14, @@ -1273,24 +980,22 @@ "sectionId": "9c7631a4-b984-46e6-981d-d2888f50856f", "w": 16, "x": 0, - "y": 3 + "y": 0 }, "panelIndex": "37997267-8e2d-4b1d-8956-2cb69b4f3ccf", - "type": "lens", - "title": "Normalized cost by billing component and deployment" + "type": "vis" }, { "embeddableConfig": { "attributes": { - "description": "Proportional cost split by FinOps category per deployment \u2014 shows how much of each deployment's bill comes from data storage (data_tier) versus platform components (ML, Kibana, APM) versus operational overhead (transfer, snapshot). FinOps value: the headline for optimisation conversations \u2014 a high transfer or snapshot share signals quick-win reduction opportunities.", "references": [], "state": { "adHocDataViews": { - "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "billing_cluster_cost_lookup-0a1e7a7fe5e3cd44b1591e64354e6ad97668baa76655f9d646cabe1c95fdc4a5": { "allowHidden": false, "allowNoIndex": false, "fieldFormats": {}, - "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "id": "billing_cluster_cost_lookup-0a1e7a7fe5e3cd44b1591e64354e6ad97668baa76655f9d646cabe1c95fdc4a5", "managed": false, "name": "billing_cluster_cost_lookup", "runtimeFieldMap": {}, @@ -1304,103 +1009,39 @@ "textBased": { "indexPatternRefs": [ { - "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "id": "billing_cluster_cost_lookup-0a1e7a7fe5e3cd44b1591e64354e6ad97668baa76655f9d646cabe1c95fdc4a5", "timeField": "@timestamp", "title": "billing_cluster_cost_lookup" } ], "layers": { - "292a9a01-e99b-4699-b669-8719ce91ae57": { - "allColumns": [ - { - "columnId": "Cost Category", - "customLabel": false, - "fieldName": "Cost Category", - "label": "Cost Category", - "meta": { - "esType": "keyword", - "type": "string" - } - }, + "layer_0": { + "columns": [ { - "columnId": "Normalized cost", - "customLabel": false, + "columnId": "partition_value_accessor_metric_0", "fieldName": "Normalized cost", - "inMetricDimension": true, - "label": "Normalized cost", "meta": { - "esType": "double", "type": "number" } }, { - "columnId": "2c726374-4aa5-46d4-9a07-76ded6403153", - "fieldName": "Deployment", - "label": "Deployment", + "columnId": "partition_value_accessor_group_by_0", + "fieldName": "Cost Category", "meta": { - "esType": "keyword", - "params": { - "id": "string" - }, - "sourceParams": { - "indexPattern": "billing_cluster_cost_lookup", - "sourceField": "Deployment" - }, "type": "string" } }, { - "columnId": "Deployment", + "columnId": "partition_value_accessor_group_by_1", "fieldName": "Deployment", - "label": "Deployment", "meta": { - "esType": "keyword", "type": "string" } } ], - "columns": [ - { - "columnId": "2c726374-4aa5-46d4-9a07-76ded6403153", - "fieldName": "Deployment", - "label": "Deployment", - "meta": { - "esType": "keyword", - "params": { - "id": "string" - }, - "sourceParams": { - "indexPattern": "billing_cluster_cost_lookup", - "sourceField": "Deployment" - }, - "type": "string" - } - }, - { - "columnId": "Cost Category", - "customLabel": false, - "fieldName": "Cost Category", - "label": "Cost Category", - "meta": { - "esType": "keyword", - "type": "string" - } - }, - { - "columnId": "Normalized cost", - "customLabel": false, - "fieldName": "Normalized cost", - "inMetricDimension": true, - "label": "Normalized cost", - "meta": { - "esType": "double", - "type": "number" - } - } - ], - "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "index": "billing_cluster_cost_lookup-0a1e7a7fe5e3cd44b1591e64354e6ad97668baa76655f9d646cabe1c95fdc4a5", "query": { - "esql": "FROM billing_cluster_cost_lookup\n| WHERE cost_type IS NOT NULL\n| EVAL ts = @timestamp\n| LOOKUP JOIN chargeback_conf_lookup ON ts >= conf_start_date AND ts <= conf_end_date\n| STATS `Normalized cost` = SUM(COALESCE(total_chargeable_units, total_ecu) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0)) BY deployment_name, cost_category\n| RENAME cost_category AS `Cost Category`, deployment_name AS `Deployment`\n| SORT `Normalized cost` DESC" + "esql": "FROM billing_cluster_cost_lookup\n | WHERE cost_type IS NOT NULL\n | WHERE\n CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group))\n | WHERE\n CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name))\n | WHERE\n CASE(?cost_category_selected IS NULL OR MV_INTERSECTS(?cost_category_selected, [\"All\"]), true, MV_INTERSECTS(?cost_category_selected, cost_category))\n | WHERE\n CASE(?cost_type_selected IS NULL OR MV_INTERSECTS(?cost_type_selected, [\"All\"]), true, MV_INTERSECTS(?cost_type_selected, cost_type))\n | EVAL `ts` = @timestamp\n | LOOKUP JOIN chargeback_conf_lookup\n ON `ts` >= conf_start_date AND `ts` <= conf_end_date\n | EVAL\n _units = CASE(total_chargeable_units IS NOT NULL, total_chargeable_units, total_ecu),\n _rate = CASE(conf_chargeable_unit_rate IS NOT NULL, conf_chargeable_unit_rate,\n CASE(conf_ecu_rate IS NOT NULL, conf_ecu_rate, 1.0))\n | STATS\n `Normalized cost` = SUM(_units * _rate)\n BY deployment_name, cost_category\n | RENAME\n cost_category AS `Cost Category`,\n deployment_name AS Deployment\n | SORT `Normalized cost` DESC" }, "timeField": "@timestamp" } @@ -1410,35 +1051,59 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_cluster_cost_lookup\n| WHERE cost_type IS NOT NULL\n| EVAL ts = @timestamp\n| LOOKUP JOIN chargeback_conf_lookup ON ts >= conf_start_date AND ts <= conf_end_date\n| STATS `Normalized cost` = SUM(COALESCE(total_chargeable_units, total_ecu) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0)) BY deployment_name, cost_category\n| RENAME cost_category AS `Cost Category`, deployment_name AS `Deployment`\n| SORT `Normalized cost` DESC" + "esql": "FROM billing_cluster_cost_lookup\n | WHERE cost_type IS NOT NULL\n | WHERE\n CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group))\n | WHERE\n CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name))\n | WHERE\n CASE(?cost_category_selected IS NULL OR MV_INTERSECTS(?cost_category_selected, [\"All\"]), true, MV_INTERSECTS(?cost_category_selected, cost_category))\n | WHERE\n CASE(?cost_type_selected IS NULL OR MV_INTERSECTS(?cost_type_selected, [\"All\"]), true, MV_INTERSECTS(?cost_type_selected, cost_type))\n | EVAL `ts` = @timestamp\n | LOOKUP JOIN chargeback_conf_lookup\n ON `ts` >= conf_start_date AND `ts` <= conf_end_date\n | EVAL\n _units = CASE(total_chargeable_units IS NOT NULL, total_chargeable_units, total_ecu),\n _rate = CASE(conf_chargeable_unit_rate IS NOT NULL, conf_chargeable_unit_rate,\n CASE(conf_ecu_rate IS NOT NULL, conf_ecu_rate, 1.0))\n | STATS\n `Normalized cost` = SUM(_units * _rate)\n BY deployment_name, cost_category\n | RENAME\n cost_category AS `Cost Category`,\n deployment_name AS Deployment\n | SORT `Normalized cost` DESC" }, "visualization": { "layers": [ { + "allowMultipleMetrics": false, "categoryDisplay": "default", - "layerId": "292a9a01-e99b-4699-b669-8719ce91ae57", + "collapseFns": {}, + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "default", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rules": [ + { + "type": "other" + } + ], + "touched": false + } + ] + }, + "layerId": "layer_0", "layerType": "data", "legendDisplay": "default", "metrics": [ - "Normalized cost" + "partition_value_accessor_metric_0" ], "nestedLegend": false, "numberDisplay": "percent", "primaryGroups": [ - "Cost Category", - "2c726374-4aa5-46d4-9a07-76ded6403153" - ] + "partition_value_accessor_group_by_0", + "partition_value_accessor_group_by_1" + ], + "truncateLegend": false } ], "shape": "treemap" } }, - "title": "Normalized cost share by cost category", + "title": "", "version": 2, "visualizationType": "lnsPie" }, "description": "Cost broken down by Deployment, and FinOps roll-up group (Cost category).\nCost category groups these into broader buckets (data_tier, platform, transfer, snapshot, onprem, other).\nThis is the highest-level total. Tier and data-stream panels show only the allocatable data-tier pool, so they sum to less than this figure.", - "title": "Cost Share by FinOps Category" + "drilldowns": [], + "title": "Normalized cost share by cost category" }, "gridData": { "h": 14, @@ -1446,24 +1111,22 @@ "sectionId": "9c7631a4-b984-46e6-981d-d2888f50856f", "w": 16, "x": 16, - "y": 3 + "y": 0 }, "panelIndex": "709e2f03-a1e9-43b5-bf4e-30565cf36260", - "type": "lens", - "title": "Normalized cost share by cost category" + "type": "vis" }, { "embeddableConfig": { "attributes": { - "description": "Proportional cost split by billing component (cost_type) per deployment \u2014 the finest-grained view of what Elastic is invoicing. FinOps value: pinpoints exactly which SKU families dominate spend, e.g. whether cold-tier storage or data-transfer is the unexpected cost driver.", "references": [], "state": { "adHocDataViews": { - "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "billing_cluster_cost_lookup-969e7d41f4da54d99d110e079f5a0b1f37263f7add16a0a1242a3ee7f618b548": { "allowHidden": false, "allowNoIndex": false, "fieldFormats": {}, - "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "id": "billing_cluster_cost_lookup-969e7d41f4da54d99d110e079f5a0b1f37263f7add16a0a1242a3ee7f618b548", "managed": false, "name": "billing_cluster_cost_lookup", "runtimeFieldMap": {}, @@ -1477,103 +1140,39 @@ "textBased": { "indexPatternRefs": [ { - "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "id": "billing_cluster_cost_lookup-969e7d41f4da54d99d110e079f5a0b1f37263f7add16a0a1242a3ee7f618b548", "timeField": "@timestamp", "title": "billing_cluster_cost_lookup" } ], "layers": { - "a72c8733-c35a-4013-80dd-293f9a0e8f7b": { - "allColumns": [ - { - "columnId": "Cost Type", - "customLabel": false, - "fieldName": "Cost Type", - "label": "Cost Type", - "meta": { - "esType": "keyword", - "type": "string" - } - }, + "layer_0": { + "columns": [ { - "columnId": "Normalized cost", - "customLabel": false, + "columnId": "partition_value_accessor_metric_0", "fieldName": "Normalized cost", - "inMetricDimension": true, - "label": "Normalized cost", "meta": { - "esType": "double", "type": "number" } }, { - "columnId": "71f0415a-bb2a-4ee5-acbb-c5dc1b179bc7", - "fieldName": "Deployment", - "label": "Deployment", + "columnId": "partition_value_accessor_group_by_0", + "fieldName": "Cost Type", "meta": { - "esType": "keyword", - "params": { - "id": "string" - }, - "sourceParams": { - "indexPattern": "billing_cluster_cost_lookup", - "sourceField": "Deployment" - }, "type": "string" } }, { - "columnId": "Deployment", + "columnId": "partition_value_accessor_group_by_1", "fieldName": "Deployment", - "label": "Deployment", "meta": { - "esType": "keyword", "type": "string" } } ], - "columns": [ - { - "columnId": "71f0415a-bb2a-4ee5-acbb-c5dc1b179bc7", - "fieldName": "Deployment", - "label": "Deployment", - "meta": { - "esType": "keyword", - "params": { - "id": "string" - }, - "sourceParams": { - "indexPattern": "billing_cluster_cost_lookup", - "sourceField": "Deployment" - }, - "type": "string" - } - }, - { - "columnId": "Cost Type", - "customLabel": false, - "fieldName": "Cost Type", - "label": "Cost Type", - "meta": { - "esType": "keyword", - "type": "string" - } - }, - { - "columnId": "Normalized cost", - "customLabel": false, - "fieldName": "Normalized cost", - "inMetricDimension": true, - "label": "Normalized cost", - "meta": { - "esType": "double", - "type": "number" - } - } - ], - "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "index": "billing_cluster_cost_lookup-969e7d41f4da54d99d110e079f5a0b1f37263f7add16a0a1242a3ee7f618b548", "query": { - "esql": "FROM billing_cluster_cost_lookup\n| WHERE cost_type IS NOT NULL\n| EVAL ts = @timestamp\n| LOOKUP JOIN chargeback_conf_lookup ON ts >= conf_start_date AND ts <= conf_end_date\n| STATS `Normalized cost` = SUM(COALESCE(total_chargeable_units, total_ecu) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0)) BY deployment_name, cost_type\n| RENAME cost_type AS `Cost Type`, deployment_name as `Deployment`\n| SORT `Normalized cost` DESC" + "esql": "FROM billing_cluster_cost_lookup\n | WHERE cost_type IS NOT NULL\n | WHERE\n CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group))\n | WHERE\n CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name))\n | WHERE\n CASE(?cost_category_selected IS NULL OR MV_INTERSECTS(?cost_category_selected, [\"All\"]), true, MV_INTERSECTS(?cost_category_selected, cost_category))\n | WHERE\n CASE(?cost_type_selected IS NULL OR MV_INTERSECTS(?cost_type_selected, [\"All\"]), true, MV_INTERSECTS(?cost_type_selected, cost_type))\n | EVAL `ts` = @timestamp\n | LOOKUP JOIN chargeback_conf_lookup\n ON\n `ts` >= conf_start_date AND\n `ts` <= conf_end_date\n | EVAL\n _units =\n CASE(\n total_chargeable_units IS NOT NULL,\n total_chargeable_units,\n total_ecu),\n _rate =\n CASE(\n conf_chargeable_unit_rate IS NOT NULL,\n conf_chargeable_unit_rate,\n CASE(conf_ecu_rate IS NOT NULL, conf_ecu_rate,\n 1.0))\n | STATS `Normalized cost` = SUM(_units * _rate)\n BY deployment_name, cost_type\n | RENAME\n cost_type AS `Cost Type`,\n deployment_name AS Deployment\n | SORT `Normalized cost` DESC" }, "timeField": "@timestamp" } @@ -1583,35 +1182,59 @@ "filters": [], "needsRefresh": false, "query": { - "esql": "FROM billing_cluster_cost_lookup\n| WHERE cost_type IS NOT NULL\n| EVAL ts = @timestamp\n| LOOKUP JOIN chargeback_conf_lookup ON ts >= conf_start_date AND ts <= conf_end_date\n| STATS `Normalized cost` = SUM(COALESCE(total_chargeable_units, total_ecu) * COALESCE(conf_chargeable_unit_rate, conf_ecu_rate, 1.0)) BY deployment_name, cost_type\n| RENAME cost_type AS `Cost Type`, deployment_name as `Deployment`\n| SORT `Normalized cost` DESC" + "esql": "FROM billing_cluster_cost_lookup\n | WHERE cost_type IS NOT NULL\n | WHERE\n CASE(?dg_selected IS NULL OR MV_INTERSECTS(?dg_selected, [\"All\"]), true, MV_INTERSECTS(?dg_selected, deployment_group))\n | WHERE\n CASE(?deployment_selected IS NULL OR MV_INTERSECTS(?deployment_selected, [\"All\"]), true, MV_INTERSECTS(?deployment_selected, deployment_name))\n | WHERE\n CASE(?cost_category_selected IS NULL OR MV_INTERSECTS(?cost_category_selected, [\"All\"]), true, MV_INTERSECTS(?cost_category_selected, cost_category))\n | WHERE\n CASE(?cost_type_selected IS NULL OR MV_INTERSECTS(?cost_type_selected, [\"All\"]), true, MV_INTERSECTS(?cost_type_selected, cost_type))\n | EVAL `ts` = @timestamp\n | LOOKUP JOIN chargeback_conf_lookup\n ON\n `ts` >= conf_start_date AND\n `ts` <= conf_end_date\n | EVAL\n _units =\n CASE(\n total_chargeable_units IS NOT NULL,\n total_chargeable_units,\n total_ecu),\n _rate =\n CASE(\n conf_chargeable_unit_rate IS NOT NULL,\n conf_chargeable_unit_rate,\n CASE(conf_ecu_rate IS NOT NULL, conf_ecu_rate,\n 1.0))\n | STATS `Normalized cost` = SUM(_units * _rate)\n BY deployment_name, cost_type\n | RENAME\n cost_type AS `Cost Type`,\n deployment_name AS Deployment\n | SORT `Normalized cost` DESC" }, "visualization": { "layers": [ { + "allowMultipleMetrics": false, "categoryDisplay": "default", - "layerId": "a72c8733-c35a-4013-80dd-293f9a0e8f7b", + "collapseFns": {}, + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "default", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rules": [ + { + "type": "other" + } + ], + "touched": false + } + ] + }, + "layerId": "layer_0", "layerType": "data", "legendDisplay": "default", "metrics": [ - "Normalized cost" + "partition_value_accessor_metric_0" ], "nestedLegend": false, "numberDisplay": "percent", "primaryGroups": [ - "Cost Type", - "71f0415a-bb2a-4ee5-acbb-c5dc1b179bc7" - ] + "partition_value_accessor_group_by_0", + "partition_value_accessor_group_by_1" + ], + "truncateLegend": false } ], "shape": "treemap" } }, - "title": "Normalized cost share by cost type", + "title": "", "version": 2, "visualizationType": "lnsPie" }, "description": "Cost broken down by Deployment, and Billable component (Cost type).\nCost Type shows individual SKU families (datahot/datacontent, datawarm, ml, kibana, snapshots, \u2026).", - "title": "Cost Share by Billing Component" + "drilldowns": [], + "title": "Normalized cost share by cost type" }, "gridData": { "h": 14, @@ -1619,13 +1242,13 @@ "sectionId": "9c7631a4-b984-46e6-981d-d2888f50856f", "w": 16, "x": 32, - "y": 3 + "y": 0 }, "panelIndex": "7f1720fc-35ac-41cc-916f-d9542c4b1130", - "type": "lens", - "title": "Normalized cost share by cost type" + "type": "vis" } ], + "pinned_panels": null, "refreshInterval": { "pause": true, "value": 60000 @@ -1635,7 +1258,7 @@ "collapsed": false, "gridData": { "i": "835374ca-64a0-4fcf-9f24-e301a28ebf8c", - "y": 13 + "y": 15 }, "title": "Deployment group statistics" }, @@ -1643,7 +1266,7 @@ "collapsed": false, "gridData": { "i": "9c7631a4-b984-46e6-981d-d2888f50856f", - "y": 14 + "y": 16 }, "title": "Component statistics" } @@ -1651,48 +1274,28 @@ "timeFrom": "now-3M", "timeRestore": true, "timeTo": "now", - "title": "[Chargeback] Billing Components Overview", - "version": 1 + "title": "[Chargeback] Billing Components Overview" }, + "coreMigrationVersion": "8.8.0", + "created_at": "2026-06-09T08:55:30.079Z", "id": "chargeback-b6cecac0-ebe7-43de-bfc6-1ff287ad860a", "references": [ - { - "id": "chargeback_integration", - "name": "controlGroup_b680ef70-4697-41e5-bf5f-e7e1e700ec3e:optionsListDataView", - "type": "index-pattern" - }, - { - "id": "chargeback_integration", - "name": "controlGroup_2c7df183-8a05-4ee0-b7db-415e874636cd:optionsListDataView", - "type": "index-pattern" - }, { "id": "chargeback-b6cecac0-ebe7-43de-bfc6-1ff287ad860a", - "name": "c39e2f18-f301-40b0-8f59-2a8547ddffbb:link_94bf4612-f92b-cf6d-5c31-0ef3eaa2fc0d_dashboard", + "name": "c39e2f18-f301-40b0-8f59-2a8547ddffbb:link_cba353e8-3f35-e498-55ec-72e282a480ce_dashboard", "type": "dashboard" }, { "id": "chargeback-21588d0e-fb6a-4f76-ad2f-cd7b3d7a3d7c", - "name": "c39e2f18-f301-40b0-8f59-2a8547ddffbb:link_2652d69f-1f3e-7afa-e44e-08697f8e318e_dashboard", + "name": "c39e2f18-f301-40b0-8f59-2a8547ddffbb:link_4b7dbb50-c024-8666-14ba-f6b4487eced7_dashboard", "type": "dashboard" }, { "id": "chargeback-7ec9743a-ccfa-43d2-8311-2f25893dadae", - "name": "c39e2f18-f301-40b0-8f59-2a8547ddffbb:link_b22db599-5a58-6f1f-cfdb-408e4543af73_dashboard", + "name": "c39e2f18-f301-40b0-8f59-2a8547ddffbb:link_3a5f90b9-a885-f9a9-6669-49f360b72b47_dashboard", "type": "dashboard" - }, - { - "id": "chargeback_integration", - "name": "controlGroup_929895f7-b618-ac2e-c7d7-61502d6acb50:optionsListDataView", - "type": "index-pattern" - }, - { - "id": "chargeback_integration", - "name": "controlGroup_456d8f2e-65b3-eaab-1d98-ef80690574ad:optionsListDataView", - "type": "index-pattern" } ], "type": "dashboard", - "typeMigrationVersion": "10.3.0", - "coreMigrationVersion": "8.8.0" -} \ No newline at end of file + "typeMigrationVersion": "10.3.0" +} diff --git a/packages/chargeback/manifest.yml b/packages/chargeback/manifest.yml index e6e2228bc4d..a1df98a2b0e 100644 --- a/packages/chargeback/manifest.yml +++ b/packages/chargeback/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.5.0 name: chargeback title: "Chargeback" -version: 0.4.0 +version: 0.5.1 description: "This package calculates chargeback based on billing and consumption data" type: integration categories: @@ -9,7 +9,7 @@ categories: - custom conditions: kibana: - version: "^9.2.0" + version: "^9.4.0" elastic: subscription: "basic" screenshots: