Prometheus / OpenMetrics output

[bluecmd]

Hi,

I recently threw together this hack to output metrics from our backups to be ingested by our Prometheus monitoring.

#!/usr/bin/python3.9
import collections
import datetime as dt
import glob
import os
from libvirtnbdbackup.exceptions import RestoreError
from libvirtnbdbackup.restore import header
from libvirtnbdbackup.sparsestream import streamer
from libvirtnbdbackup.sparsestream import types
from libvirtnbdbackup import common as lib
from prometheus_client import CollectorRegistry, Gauge, write_to_textfile


def main() -> None:
    registry = CollectorRegistry()
    g_last_backup = Gauge('virtnbd_last_backup_time', 'Timestamp of last saved backup', ['backup', 'disk'], registry=registry)
    g_virtual_size = Gauge('virtnbd_virtual_size_bytes', 'Provisioned size of the disk', ['backup', 'disk'], registry=registry)
    g_last_data_size = Gauge('virtnbd_latest_data_size_bytes', 'Size of bytes included in the latest backup', ['backup', 'disk'], registry=registry)
    g_last_file_size = Gauge('virtnbd_latest_file_size_bytes', 'Size of actual bytes the latest backup is consuming', ['backup', 'disk'], registry=registry)
    g_total_file_size = Gauge('virtnbd_total_file_size_bytes', 'Size of actual bytes all checkpoints are consuming', ['backup', 'disk'], registry=registry)
    g_chkpt_count = Gauge('virtnbd_total_checkpoints_count', 'Number of checkpoints taken', ['backup', 'disk'], registry=registry)
    dirs = glob.glob(f'/srv/backup/vm/*/')
    dirs.sort()
    stream = streamer.SparseStream(types)
    for backupDir in dirs:
        dataFiles = lib.getLatest(backupDir, '*.data')
        disks = collections.defaultdict(lambda: dict({'__count': 0, '__totalFileSize': 0}))
        for dataFile in dataFiles:
            try:
                meta = header.get(dataFile, stream)
            except RestoreError as e:
                continue
            meta['date'] = dt.datetime.strptime(meta['date'], '%Y-%m-%dT%H:%M:%S.%f')
            meta['__fileSize'] = os.path.getsize(dataFile)
            meta['__totalFileSize'] = disks[meta['diskName']]['__totalFileSize'] + meta['__fileSize']
            meta['__count'] = disks[meta['diskName']]['__count'] + 1
            disks[meta['diskName']] = meta
        backup = os.path.basename(backupDir[:-1])
        for disk, data in disks.items():
            g_last_backup.labels(backup, disk).set(data['date'].timestamp())
            g_virtual_size.labels(backup, disk).set(data['virtualSize'])
            g_last_data_size.labels(backup, disk).set(data['dataSize'])
            g_last_file_size.labels(backup, disk).set(data['__fileSize'])
            g_total_file_size.labels(backup, disk).set(data['__totalFileSize'])
            g_chkpt_count.labels(backup, disk).set(data['__count'])
    write_to_textfile('/var/lib/node_exporter/virtnbdmetrics.prom', registry)

if __name__ == '__main__':
    main()

Do you think there would be a place for something like this in e.g. virtnbdrestore or some other tool? Maybe as virtnbdrestore -i /..../ -a metrics -o /var/lib/node_exporter/virtnbdmetrics.prom?

[abbbi]

hi,

thanks for your input. I dont use Prometheus anyhwere so i dont intend to ship :)
You could also just parse the json returned by the dump function i guess:

 ./virtnbdrestore -i /tmp/backup -o dump 2> /dev/null
[
    {
        "virtualSize": 52428800,
        "dataSize": 6094848,
        "date": "2024-08-13T17:06:43.450794",
        "diskName": "sda",
        "diskFormat": "qcow2",
        "checkpointName": "virtnbdbackup.0",
        "compressed": false,
        "compressionMethod": "lz4",
        "parentCheckpoint": "",
        "incremental": false,
        "streamVersion": 2
    }
]

you can also specify a sequence of files in a folder if desired, afair:

virtnbdrestore -i /tmp/backup -o dump -s file1,file2