Automatic yt-dlp updates

jobs.yaml

@@ -0,0 +1,23 @@
+- command: python utils/cleanuptasks.py
+  filelog: 'yes'
+  filelog-stderr: /data/project/video2commons/cleanup.err
+  filelog-stdout: /data/project/video2commons/cleanup.out
+  image: python3.9
+  name: cleanup
+  schedule: 23 7 * * *
+
+- command: bash cleanuploads.sh
+  filelog: 'yes'
+  filelog-stderr: /data/project/video2commons/cleanuploads.err
+  filelog-stdout: /data/project/video2commons/cleanuploads.out
+  image: bullseye
+  name: cleanuploads
+  schedule: 28 7 * * *
+
+- command: bash utils/update-yt-dlp-toolforge.sh
+  filelog: 'yes'
+  filelog-stderr: /data/project/video2commons/update-yt-dlp.err
+  filelog-stdout: /data/project/video2commons/update-yt-dlp.out
+  image: python3.11
+  name: update-yt-dlp
+  schedule: 33 7 * * *

puppet/backend.pp

@@ -344,3 +344,10 @@
     minute  => '0',
     require => Service['v2ccelery'],
 }
+cron::job { 'v2cytdlp':
+    command => '/bin/bash /srv/v2c/utils/update-yt-dlp-encoder.sh',
+    user    => 'root',
+    minute  => '32',
+    hour    => '6',
+    require => Service['v2ccelery'],
+}

utils/update-yt-dlp-encoder.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Upgrades yt-dlp in the encoder venv.
+#
+# Celery is only restarted when the installed version actually changed, and the
+# service is running (e.g. not restarting or manually stopped).
+
+set -e
+
+VENV_PIP=/srv/v2c/venv/bin/pip3
+
+installed_version() {
+    "$VENV_PIP" show yt-dlp | sed -n 's/^Version: //p'
+}
+
+before=$(installed_version)
+
+"$VENV_PIP" install -U 'yt-dlp[default]'
+
+after=$(installed_version)
+
+if [ "$before" = "$after" ]; then
+    echo "yt-dlp is already at $after, nothing to do"
+    exit 0
+fi
+
+sub_state=$(systemctl show v2ccelery.service --property=SubState --value)
+
+if [ "$sub_state" != "running" ]; then
+    echo "yt-dlp went from '$before' to '$after', but skipping restart as celery is '$sub_state'"
+    exit 0
+fi
+
+echo "yt-dlp went from '$before' to '$after', restarting celery"
+systemctl --no-block restart v2ccelery.service

utils/update-yt-dlp-toolforge.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Upgrades yt-dlp in the frontend venv.
+#
+# The webservice is only reloaded when the installed version actually changed.
+# The reload happens through uwsgi's 'touch-reload' option, since the toolforge
+# CLI is not usable from the job image.
+
+set -e
+
+VENV_PIP="$HOME/www/python/venv/bin/pip3"
+RELOAD_TRIGGER="$HOME/www/python/reload"
+
+installed_version() {
+    "$VENV_PIP" show yt-dlp | sed -n 's/^Version: //p'
+}
+
+before=$(installed_version)
+
+"$VENV_PIP" install -U 'yt-dlp[default]'
+
+after=$(installed_version)
+
+if [ "$before" = "$after" ]; then
+    echo "yt-dlp is already at $after, nothing to do"
+    exit 0
+fi
+
+echo "yt-dlp went from '$before' to '$after', reloading the webservice"
+touch "$RELOAD_TRIGGER"

www/python/uwsgi.ini

@@ -0,0 +1,5 @@
+[uwsgi]
+# 10 MiB log rotation
+log-maxsize = 10485760
+# Touched by utils/update-yt-dlp-toolforge.sh after a yt-dlp upgrade
+touch-reload = %dreload