diff --git a/jobs.yaml b/jobs.yaml new file mode 100644 index 00000000..b3dfe53e --- /dev/null +++ b/jobs.yaml @@ -0,0 +1,23 @@ +- command: python utils/cleanuptasks.py + filelog: 'yes' + filelog-stderr: /data/project/video2commons/cleanup.err + filelog-stdout: /data/project/video2commons/cleanup.out + image: python3.9 + name: cleanup + schedule: 23 7 * * * + +- command: bash cleanuploads.sh + filelog: 'yes' + filelog-stderr: /data/project/video2commons/cleanuploads.err + filelog-stdout: /data/project/video2commons/cleanuploads.out + image: bullseye + name: cleanuploads + schedule: 28 7 * * * + +- command: bash utils/update-yt-dlp-toolforge.sh + filelog: 'yes' + filelog-stderr: /data/project/video2commons/update-yt-dlp.err + filelog-stdout: /data/project/video2commons/update-yt-dlp.out + image: python3.11 + name: update-yt-dlp + schedule: 33 7 * * * diff --git a/puppet/backend.pp b/puppet/backend.pp index b19172f4..f7830fb9 100644 --- a/puppet/backend.pp +++ b/puppet/backend.pp @@ -344,3 +344,10 @@ minute => '0', require => Service['v2ccelery'], } +cron::job { 'v2cytdlp': + command => '/bin/bash /srv/v2c/utils/update-yt-dlp-encoder.sh', + user => 'root', + minute => '32', + hour => '6', + require => Service['v2ccelery'], +} diff --git a/utils/update-yt-dlp-encoder.sh b/utils/update-yt-dlp-encoder.sh new file mode 100755 index 00000000..932e0753 --- /dev/null +++ b/utils/update-yt-dlp-encoder.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Upgrades yt-dlp in the encoder venv. +# +# Celery is only restarted when the installed version actually changed, and the +# service is running (e.g. not restarting or manually stopped). + +set -e + +VENV_PIP=/srv/v2c/venv/bin/pip3 + +installed_version() { + "$VENV_PIP" show yt-dlp | sed -n 's/^Version: //p' +} + +before=$(installed_version) + +"$VENV_PIP" install -U 'yt-dlp[default]' + +after=$(installed_version) + +if [ "$before" = "$after" ]; then + echo "yt-dlp is already at $after, nothing to do" + exit 0 +fi + +sub_state=$(systemctl show v2ccelery.service --property=SubState --value) + +if [ "$sub_state" != "running" ]; then + echo "yt-dlp went from '$before' to '$after', but skipping restart as celery is '$sub_state'" + exit 0 +fi + +echo "yt-dlp went from '$before' to '$after', restarting celery" +systemctl --no-block restart v2ccelery.service diff --git a/utils/update-yt-dlp-toolforge.sh b/utils/update-yt-dlp-toolforge.sh new file mode 100755 index 00000000..717b347d --- /dev/null +++ b/utils/update-yt-dlp-toolforge.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Upgrades yt-dlp in the frontend venv. +# +# The webservice is only reloaded when the installed version actually changed. +# The reload happens through uwsgi's 'touch-reload' option, since the toolforge +# CLI is not usable from the job image. + +set -e + +VENV_PIP="$HOME/www/python/venv/bin/pip3" +RELOAD_TRIGGER="$HOME/www/python/reload" + +installed_version() { + "$VENV_PIP" show yt-dlp | sed -n 's/^Version: //p' +} + +before=$(installed_version) + +"$VENV_PIP" install -U 'yt-dlp[default]' + +after=$(installed_version) + +if [ "$before" = "$after" ]; then + echo "yt-dlp is already at $after, nothing to do" + exit 0 +fi + +echo "yt-dlp went from '$before' to '$after', reloading the webservice" +touch "$RELOAD_TRIGGER" diff --git a/www/python/reload b/www/python/reload new file mode 100644 index 00000000..e69de29b diff --git a/www/python/uwsgi.ini b/www/python/uwsgi.ini new file mode 100644 index 00000000..5e63cd23 --- /dev/null +++ b/www/python/uwsgi.ini @@ -0,0 +1,5 @@ +[uwsgi] +# 10 MiB log rotation +log-maxsize = 10485760 +# Touched by utils/update-yt-dlp-toolforge.sh after a yt-dlp upgrade +touch-reload = %dreload