diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml index 4f017acb9..2ec1ade0b 100644 --- a/.github/actions/build/action.yml +++ b/.github/actions/build/action.yml @@ -21,6 +21,8 @@ runs: using: composite steps: - uses: ./.github/actions/cleanup + - name: Install Task + uses: arduino/setup-task@b91d5d2c96a56797b48ac1e0e89220bf64044611 # v2.0.0 - name: Setup environment and build shell: bash env: @@ -35,23 +37,14 @@ runs: # Capture PHC is disciplined to TAI at runtime via phc2sys -O # , so the kernel TAI offset is left untouched. TOOLS_RUN_SET_TAI_OFFSET: '0' - run: | - bash .github/scripts/setup_environment.sh - if [ -n "${{ inputs.remote_host }}" ]; then - BRANCH="${{ inputs.remote_branch }}" - BRANCH="${BRANCH#refs/heads/}" - ENV_FILE="${{ inputs.remote_env_file }}" - ENV_FILE="${ENV_FILE/#\~/$HOME}" - { - [ -f "$ENV_FILE" ] && cat "$ENV_FILE" \ - || echo "::warning::No env file at $ENV_FILE — using defaults" - cat .github/scripts/remote_build.sh - } | ssh -o StrictHostKeyChecking=no "${{ inputs.remote_host }}" \ - bash -s -- "${{ inputs.remote_mtl_path }}" "${BRANCH}" - fi + REMOTE_HOST: ${{ inputs.remote_host }} + REMOTE_MTL_PATH: ${{ inputs.remote_mtl_path }} + REMOTE_BRANCH: ${{ inputs.remote_branch }} + REMOTE_ENV_FILE: ${{ inputs.remote_env_file }} + run: task ci:build-action - name: Upload build artifacts if: ${{ always() && inputs.remote_host == '' }} - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: build-${{ github.sha }} path: | diff --git a/.github/actions/check-ebpf/action.yml b/.github/actions/check-ebpf/action.yml new file mode 100644 index 000000000..5f276feb6 --- /dev/null +++ b/.github/actions/check-ebpf/action.yml @@ -0,0 +1,35 @@ +name: Check eBPF/XDP prerequisites +description: > + Assert that the runner can build and run MTL's eBPF/XDP paths before the job + spends any time on it. Wraps script/build_ebpf_xdp.sh --check, so a local + `task ebpf:check` and a CI run execute the same code. + + Place it as the first step of a job: a missing prerequisite fails here in + seconds instead of surfacing minutes later as an unrelated pkg-config error. + +inputs: + mode: + description: What to verify -- build, runtime or all. + required: false + default: 'all' + strict: + description: Fail the job when a required prerequisite is missing. + required: false + default: 'true' + require_xdp: + description: > + Treat xdp-tools/libbpf as mandatory and demand the exact versions pinned + in versions.env. Enable on hosts provisioned for AF_XDP. + required: false + default: 'false' + +runs: + using: composite + steps: + - name: 'ebpf: check prerequisites' + shell: bash + env: + MODE: ${{ inputs.mode }} + STRICT: ${{ inputs.strict }} + REQUIRE_XDP: ${{ inputs.require_xdp }} + run: bash "$GITHUB_ACTION_PATH/../../scripts/ci/check-ebpf-action.sh" diff --git a/.github/actions/cleanup/action.yml b/.github/actions/cleanup/action.yml index f260b4bd8..9d20b1775 100644 --- a/.github/actions/cleanup/action.yml +++ b/.github/actions/cleanup/action.yml @@ -5,16 +5,4 @@ runs: steps: - name: Cleanup processes shell: bash - run: | - echo "Killing any running DPDK or MTL related processes..." - sudo killall -SIGKILL gtest.sh || true - sudo killall -SIGKILL KahawaiTest || true - sudo killall -SIGKILL ffmpeg || true - sudo killall -SIGKILL RxTxApp || true - echo "Clenaup of the supporting processes..." - sudo killall -SIGINT pytest || true - sudo killall -SIGINT MtlManager || true - sudo killall -SIGINT phc2sys || true - sudo killall -SIGINT ptp4l || true - sudo killall -SIGINT netsniff-ng || true - sleep 2 \ No newline at end of file + run: bash "$GITHUB_ACTION_PATH/../../scripts/ci/cleanup.sh" \ No newline at end of file diff --git a/.github/actions/source-checksums/action.yml b/.github/actions/source-checksums/action.yml index 182da5686..7d823ca7d 100644 --- a/.github/actions/source-checksums/action.yml +++ b/.github/actions/source-checksums/action.yml @@ -1,8 +1,5 @@ name: Source Checksums -description: > - Compute deterministic SHA-256 hashes for each component group. - Waterfall: dpdk → mtl → {ffmpeg, gstreamer, plugins}. - A parent change forces all children to rehash automatically. +description: Compute deterministic SHA-256 hashes for CI dependency caches. outputs: dpdk: @@ -11,6 +8,12 @@ outputs: mtl: description: 'Hash for MTL sources (includes DPDK hash)' value: ${{ steps.compute.outputs.mtl }} + jpegxs: + description: 'Hash for JPEG XS bundle sources (includes MTL hash)' + value: ${{ steps.compute.outputs.jpegxs }} + ice: + description: 'Hash for patched ICE driver sources' + value: ${{ steps.compute.outputs.ice }} ffmpeg: description: 'Hash for FFmpeg plugin sources (includes MTL hash)' value: ${{ steps.compute.outputs.ffmpeg }} @@ -24,10 +27,8 @@ outputs: runs: using: composite steps: + - name: Install Task + uses: arduino/setup-task@b91d5d2c96a56797b48ac1e0e89220bf64044611 # v2.0.0 - id: compute shell: bash - run: | - set -euo pipefail - echo "::group::Source Checksums" - "${{ github.workspace }}/script/hash_sources.sh" -o "$GITHUB_OUTPUT" - echo "::endgroup::" + run: task ci:hash-dependencies -- -o "$GITHUB_OUTPUT" diff --git a/.github/actions/validate-host/action.yml b/.github/actions/validate-host/action.yml index 76f604064..0954d814a 100644 --- a/.github/actions/validate-host/action.yml +++ b/.github/actions/validate-host/action.yml @@ -1,14 +1,9 @@ name: Validate Host -description: > - Prepare a bare-metal test host: compute local source checksums, - restore build components from cache into .local_install/, - untar them, make binaries executable, and configure environment - so that subsequent steps can run MTL tests without system-wide installation. - Optionally sync .local_install/ to a shadow host via SSH. +description: Restore and validate cached dependencies, then idempotently align the host. inputs: shadow_host: - description: Path to a file defining IP and USER for the shadow host. If empty, shadow sync is skipped. + description: Path to a file defining IP and USER for the shadow host. required: false default: '' @@ -23,143 +18,120 @@ outputs: runs: using: composite steps: + - name: 'system: Check eBPF/XDP prerequisites' + uses: ./.github/actions/check-ebpf + with: + mode: all + strict: 'true' + - uses: ./.github/actions/cleanup - name: Compute local source checksums id: local-checksums uses: ./.github/actions/source-checksums + - name: Compute exact cache keys + id: keys + shell: bash + env: + HASH_DPDK: ${{ steps.local-checksums.outputs.dpdk }} + HASH_MTL: ${{ steps.local-checksums.outputs.mtl }} + HASH_JPEGXS: ${{ steps.local-checksums.outputs.jpegxs }} + HASH_ICE: ${{ steps.local-checksums.outputs.ice }} + HASH_FFMPEG: ${{ steps.local-checksums.outputs.ffmpeg }} + HASH_GSTREAMER: ${{ steps.local-checksums.outputs.gstreamer }} + HASH_PLUGINS: ${{ steps.local-checksums.outputs.plugins }} + run: task ci:cache-keys + - name: 'cache: Restore DPDK' - uses: actions/cache/restore@v4 + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: - key: stash-dpdk-${{ steps.local-checksums.outputs.dpdk }} + key: ${{ steps.keys.outputs.dpdk_key }} path: .local_install/dpdk fail-on-cache-miss: true - name: 'cache: Restore MTL' - uses: actions/cache/restore@v4 + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: - key: stash-mtl-${{ steps.local-checksums.outputs.mtl }} + key: ${{ steps.keys.outputs.mtl_key }} path: .local_install/mtl fail-on-cache-miss: true + - name: 'cache: Restore JPEG XS' + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + key: ${{ steps.keys.outputs.jpegxs_key }} + path: .local_install/jpegxs + fail-on-cache-miss: true + - name: 'cache: Restore FFmpeg' - uses: actions/cache/restore@v4 + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: - key: stash-ffmpeg-${{ steps.local-checksums.outputs.ffmpeg }} + key: ${{ steps.keys.outputs.ffmpeg_key }} path: .local_install/ffmpeg fail-on-cache-miss: true - name: 'cache: Restore GStreamer' - uses: actions/cache/restore@v4 + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: - key: stash-gstreamer-${{ steps.local-checksums.outputs.gstreamer }} + key: ${{ steps.keys.outputs.gstreamer_key }} path: .local_install/gstreamer fail-on-cache-miss: true - name: 'cache: Restore plugins' - uses: actions/cache/restore@v4 + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: - key: stash-plugins-${{ steps.local-checksums.outputs.plugins }} + key: ${{ steps.keys.outputs.plugins_key }} path: .local_install/plugins - # Non-fatal: a missing/failed st22 plugin build only disables the codec - # tests (they skip via require_encoder); it must not abort validation. - fail-on-cache-miss: false + fail-on-cache-miss: true - - name: Make artifacts executable - shell: bash - run: | - find .local_install -type f \( -name '*.so*' -o -path '*/bin/*' \) -exec chmod +x {} + 2>/dev/null || true + - name: 'cache: Restore ICE' + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + key: ${{ steps.keys.outputs.ice_key }} + path: .local_install/ice + fail-on-cache-miss: true - - name: 'kahawai: Generate CI plugin registry from cache' - shell: bash - run: | - # Build a CI-owned kahawai.json that points at the cached st22 avcodec - # plugin under .local_install/plugins -- no system install. MTL loads it - # via KAHAWAI_CFG_PATH (exported below). The plugin .so itself is built - # by the build workflow and restored from the `plugins` cache above. - # The template reproduces the runner plugin set: jpegxs/sample stay at - # their /usr/local install paths (a missing .so is a non-fatal dlopen - # warning), only avcodec is sourced from the cache. - set -euo pipefail - WS="${{ github.workspace }}" - template="${WS}/.github/workflows/kahawai_template.json" - cfg="${RUNNER_TEMP:-/tmp}/kahawai_ci.json" - plugin_dir="$(find "${WS}/.local_install/plugins" -name 'libst_plugin_st22_avcodec.so' -printf '%h\n' -quit 2>/dev/null || true)" - if [ -z "${plugin_dir}" ]; then - echo "::warning::st22 avcodec plugin not found in .local_install/plugins; H264 st22p tests will skip" - plugin_dir="${WS}/.local_install/plugins/lib/x86_64-linux-gnu" - fi - cp -f "${template}" "${cfg}" - sed -i "s+REPLACE_BY_CICD_PLUGIN_DIR+${plugin_dir}+g" "${cfg}" - echo "KAHAWAI_CFG_PATH=${cfg}" >> "$GITHUB_ENV" - echo "::notice::CI kahawai registry ${cfg} (avcodec from ${plugin_dir})" - - - name: 'installation: Auto-align host components (ICE driver & SVT-JPEG-XS)' + - name: Validate restored dependencies shell: bash env: - MTL_INSTALL_PREFIX: ${{ github.workspace }}/.local_install/mtl - PKG_CONFIG_PATH: ${{ github.workspace }}/.local_install/dpdk/lib/x86_64-linux-gnu/pkgconfig:${{ github.workspace }}/.local_install/mtl/lib/x86_64-linux-gnu/pkgconfig - LD_LIBRARY_PATH: ${{ github.workspace }}/.local_install/dpdk/lib/x86_64-linux-gnu:${{ github.workspace }}/.local_install/mtl/lib/x86_64-linux-gnu - run: | - export SETUP_BUILD_AND_INSTALL_ICE_DRIVER=1 - export PLUGIN_BUILD_AND_INSTALL_JPEGXS=1 - bash ${{ github.workspace }}/.github/scripts/setup_environment.sh - - - name: Configure LD_LIBRARY_PATH, PATH, GST_PLUGIN_PATH - shell: bash - run: | - WS="${{ github.workspace }}" - LI="${WS}/.local_install" - - # Shared libraries: DPDK + MTL + FFmpeg - echo "LD_LIBRARY_PATH=${LI}/dpdk/lib/x86_64-linux-gnu:${LI}/mtl/lib/x86_64-linux-gnu:${LI}/ffmpeg/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" >> "$GITHUB_ENV" + ICE_REQUIRE_SECURE_BOOT_PROBE: '1' + run: task ci:validate-dependencies - # Binaries on PATH - echo "${LI}/mtl/bin" >> "$GITHUB_PATH" - echo "${LI}/ffmpeg/bin" >> "$GITHUB_PATH" - echo "${LI}/dpdk/bin" >> "$GITHUB_PATH" + - name: Make cached artifacts executable + shell: bash + run: task ci:configure-host -- make-executable - # GStreamer plugins - echo "GST_PLUGIN_PATH=${LI}/gstreamer/gstreamer-1.0${GST_PLUGIN_PATH:+:$GST_PLUGIN_PATH}" >> "$GITHUB_ENV" + - name: Generate CI plugin registry + shell: bash + run: task ci:configure-host -- registry - # pkg-config for downstream tools - echo "PKG_CONFIG_PATH=${LI}/dpdk/lib/x86_64-linux-gnu/pkgconfig:${LI}/mtl/lib/x86_64-linux-gnu/pkgconfig" >> "$GITHUB_ENV" + - name: Preserve Task path for privileged activation + shell: bash + run: echo "TASK_BIN=$(command -v task)" >> "$GITHUB_ENV" - # Refresh linker cache (best-effort) - sudo ldconfig 2>/dev/null || true + - name: Activate ICE if required + shell: bash + env: + ICE_REQUIRE_SECURE_BOOT_PROBE: '1' + run: sudo -E "$TASK_BIN" ci:activate-ice - echo "::notice::Host environment configured for .local_install/{dpdk,mtl,ffmpeg,gstreamer,plugins}" + - name: Configure dependency environment + shell: bash + run: task ci:configure-host -- environment - name: Read shadow host credentials id: shadow-creds if: inputs.shadow_host != '' shell: bash - run: | - source "${{ inputs.shadow_host }}" - echo "ip=${IP}" >> "$GITHUB_OUTPUT" - echo "user=${USER}" >> "$GITHUB_OUTPUT" + env: + SHADOW_HOST_FILE: ${{ inputs.shadow_host }} + run: task ci:configure-host -- shadow-credentials - - name: Sync .local_install to shadow host + - name: Sync caches to shadow host if: steps.shadow-creds.outputs.ip != '' shell: bash - run: | - IP="${{ steps.shadow-creds.outputs.ip }}" - USER="${{ steps.shadow-creds.outputs.user }}" - SSH_OPTS=(-o StrictHostKeyChecking=accept-new -o BatchMode=yes -o ConnectTimeout=30) - echo "::group::Syncing .local_install/ to ${USER}@${IP}" - ssh "${SSH_OPTS[@]}" "${USER}@${IP}" "mkdir -p ${{ github.workspace }}/.local_install ${{ github.workspace }}/script ${{ github.workspace }}/tests" - rsync -az --delete \ - -e "ssh ${SSH_OPTS[*]}" \ - "${{ github.workspace }}/.local_install/" \ - "${USER}@${IP}:${{ github.workspace }}/.local_install/" - rsync -az --delete \ - -e "ssh ${SSH_OPTS[*]}" \ - "${{ github.workspace }}/script/" \ - "${USER}@${IP}:${{ github.workspace }}/script/" - rsync -az --delete \ - -e "ssh ${SSH_OPTS[*]}" \ - "${{ github.workspace }}/tests/" \ - "${USER}@${IP}:${{ github.workspace }}/tests/" - echo "::endgroup::" - echo "::notice::Synced .local_install/, script/, and tests/ to shadow host ${IP}" + env: + SHADOW_HOST_IP: ${{ steps.shadow-creds.outputs.ip }} + SHADOW_HOST_USER: ${{ steps.shadow-creds.outputs.user }} + run: task ci:configure-host -- shadow-sync diff --git a/.github/actions/wait-for-workflow/action.yml b/.github/actions/wait-for-workflow/action.yml index 5e7d78305..4c9d63af7 100644 --- a/.github/actions/wait-for-workflow/action.yml +++ b/.github/actions/wait-for-workflow/action.yml @@ -7,9 +7,9 @@ inputs: description: 'The check run name to wait for (e.g. "build", "Lint Code Base")' required: true timeout: - description: 'Maximum wait time in seconds' + description: 'Maximum in-progress time in seconds' required: false - default: '900' + default: '600' interval: description: 'Polling interval in seconds' required: false @@ -21,41 +21,9 @@ runs: - name: Wait for workflow check id: wait uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + env: + CHECK_NAME: ${{ inputs.check_name }} + WAIT_TIMEOUT: ${{ inputs.timeout }} + WAIT_INTERVAL: ${{ inputs.interval }} with: - script: | - const sha = context.payload.pull_request?.head?.sha || context.sha; - const checkName = '${{ inputs.check_name }}'; - const maxWait = parseInt('${{ inputs.timeout }}', 10); - const interval = parseInt('${{ inputs.interval }}', 10); - - core.info(`Waiting for "${checkName}" on SHA: ${sha}`); - let elapsed = 0; - - while (elapsed < maxWait) { - const { data } = await github.rest.checks.listForRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: sha, - check_name: checkName, - }); - - const runs = data.check_runs.filter(r => r.app?.slug === 'github-actions'); - const done = runs.find(r => r.status === 'completed' && r.conclusion === 'success'); - if (done) { - core.info(`"${checkName}" passed ✓`); - return; - } - const failed = runs.find(r => r.status === 'completed' && r.conclusion !== 'success'); - const pending = runs.some(r => r.status !== 'completed'); - if (failed && !pending) { - core.setFailed(`"${checkName}" finished with: ${failed.conclusion}`); - return; - } - - const status = pending ? 'in_progress' : 'not found'; - core.info(`${checkName}: ${status} — waiting… (${elapsed}s / ${maxWait}s)`); - await new Promise(r => setTimeout(r, interval * 1000)); - elapsed += interval; - } - - core.setFailed(`Timed out waiting for "${checkName}"`); + script: return require(`${{ github.action_path }}/wait.js`)({ github, context, core }) diff --git a/.github/actions/wait-for-workflow/wait.js b/.github/actions/wait-for-workflow/wait.js new file mode 100644 index 000000000..2e2a02218 --- /dev/null +++ b/.github/actions/wait-for-workflow/wait.js @@ -0,0 +1,34 @@ +'use strict'; + +module.exports = async ({ github, context, core }) => { + const sha = context.payload.pull_request?.head?.sha || context.sha; + const checkName = process.env.CHECK_NAME; + const maxWait = Number.parseInt(process.env.WAIT_TIMEOUT, 10); + const interval = Number.parseInt(process.env.WAIT_INTERVAL, 10); + core.info(`Waiting for "${checkName}" on SHA: ${sha}`); + + let activeElapsed = 0; + while (true) { + const { data } = await github.rest.checks.listForRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: sha, + check_name: checkName, + }); + const runs = data.check_runs.filter((run) => run.app?.slug === 'github-actions'); + if (runs.some((run) => run.status === 'completed' && run.conclusion === 'success')) return; + const failed = runs.find((run) => run.status === 'completed' && run.conclusion !== 'success'); + const pending = runs.some((run) => run.status !== 'completed'); + const inProgress = runs.some((run) => run.status === 'in_progress'); + if (failed && !pending) { + core.setFailed(`"${checkName}" finished with: ${failed.conclusion}`); + return; + } + const state = inProgress ? 'in_progress' : pending ? 'queued' : 'not found'; + core.info(`${checkName}: ${state}; active wait (${activeElapsed}s / ${maxWait}s)`); + if (inProgress && activeElapsed >= maxWait) break; + await new Promise((resolve) => setTimeout(resolve, interval * 1000)); + if (inProgress) activeElapsed += interval; + } + core.setFailed(`Timed out waiting for "${checkName}"`); +}; diff --git a/.github/ci-local/Dockerfile b/.github/ci-local/Dockerfile new file mode 100644 index 000000000..c6bb8485f --- /dev/null +++ b/.github/ci-local/Dockerfile @@ -0,0 +1,132 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2026 Intel Corporation +# +# Simulated GitHub Actions runner for the MTL workflows. +# +# The real `build` job runs on a self-hosted `e835` runner with GCC 13, a +# non-root user with passwordless sudo, and every apt dependency already +# present from earlier runs. A bare `ubuntu:24.04` differs from that in three +# ways that each produce a failure which has nothing to do with the change +# under test: +# +# * apt resolves the mirrors over IPv6, which many hosts cannot route +# * `sudo` resets the environment, dropping the proxy variables and +# DEBIAN_FRONTEND, so tzdata stops on an interactive prompt +# * every run re-downloads ~400 packages +# +# This image removes all three, so a failure inside it is a real failure. + +FROM public.ecr.aws/ubuntu/ubuntu:24.04@sha256:be20a0347f238b7d373edddc55923443b21dd9a60277bf8a93e43458cd0bf2fc + +# Matches the timezone setup_environment.sh applies under CICD_BUILD=1. +ARG TZ=Europe/Warsaw +# The container runs as the invoking host user so that files it creates in the +# bind-mounted workspace stay owned by that user. +ARG RUNNER_UID=1000 +ARG RUNNER_GID=1000 +ARG RUNNER_USER=runner +ARG http_proxy="" +ARG https_proxy="" +ARG no_proxy="" + +ENV DEBIAN_FRONTEND=noninteractive \ + TZ=${TZ} \ + HOME=/home/runner \ + PIP_BREAK_SYSTEM_PACKAGES=1 + +# apt: no IPv6, honour the build-time proxy, no interactive tzdata. +RUN set -eux; \ + echo 'Acquire::ForceIPv4 "true";' >/etc/apt/apt.conf.d/99force-ipv4; \ + if [ -n "${http_proxy}" ]; then \ + printf 'Acquire::http::Proxy "%s";\nAcquire::https::Proxy "%s";\n' \ + "${http_proxy}" "${https_proxy:-${http_proxy}}" >/etc/apt/apt.conf.d/99proxy; \ + fi; \ + ln -fs "/usr/share/zoneinfo/${TZ}" /etc/localtime; \ + echo "${TZ}" >/etc/timezone + +# Pre-warm the dependency set that setup_environment.sh installs. The script +# still runs apt at job time; anything already present is a no-op, so this list +# drifting behind the script costs a download, never correctness. +# +# It deliberately goes further than the script does, because the self-hosted +# runner does too: packages an earlier run installed under a flag that is off +# today are still on the machine. libelf-dev is the clearest case -- libdpdk.pc +# requires it, yet nothing installs it unless SETUP_BUILD_AND_INSTALL_EBPF_XDP +# is set, so a DPDK cache hit on a fresh machine fails where the runner passes. +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + clang \ + cmake \ + curl \ + doxygen \ + ethtool \ + gcc \ + gcc-13 \ + gcc-multilib \ + g++-13 \ + git \ + gstreamer1.0-libav \ + gstreamer1.0-plugins-base \ + gstreamer1.0-plugins-good \ + gstreamer1.0-tools \ + kmod \ + libcap-ng-dev \ + libcap2-bin \ + libelf-dev \ + libgstreamer-plugins-base1.0-dev \ + libgstreamer1.0-dev \ + libgtest-dev \ + libjson-c-dev \ + libnuma-dev \ + libpcap-dev \ + libsdl2-dev \ + libsdl2-ttf-dev \ + libssl-dev \ + libunwind-dev \ + linuxptp \ + llvm \ + locales \ + m4 \ + make \ + meson \ + nasm \ + netsniff-ng \ + ninja-build \ + patch \ + pkg-config \ + python3 \ + python3-dev \ + python3-pip \ + python3-pyelftools \ + python3-venv \ + sudo \ + systemtap-sdt-dev \ + tzdata \ + unzip \ + wget \ + yasm \ + zlib1g-dev; \ + dpkg-reconfigure -f noninteractive tzdata; \ + rm -rf /var/lib/apt/lists/* + +ENV CC=gcc-13 \ + CXX=g++-13 + +# A non-root user with passwordless sudo, like the GitHub runner. sudo's +# env_reset would otherwise strip the proxy and DEBIAN_FRONTEND that every apt +# call in setup_environment.sh depends on. +RUN set -eux; \ + groupadd --gid "${RUNNER_GID}" --non-unique "${RUNNER_USER}"; \ + useradd --uid "${RUNNER_UID}" --gid "${RUNNER_GID}" --non-unique \ + --home-dir "${HOME}" --create-home --shell /bin/bash "${RUNNER_USER}"; \ + effective_user="$(getent passwd "${RUNNER_UID}" | cut -d: -f1)"; \ + echo "${effective_user} ALL=(ALL) NOPASSWD:ALL" >/etc/sudoers.d/10-runner; \ + echo 'Defaults env_keep += "DEBIAN_FRONTEND http_proxy https_proxy no_proxy HTTP_PROXY HTTPS_PROXY NO_PROXY PIP_BREAK_SYSTEM_PACKAGES"' \ + >/etc/sudoers.d/20-env; \ + chmod 0440 /etc/sudoers.d/10-runner /etc/sudoers.d/20-env + +USER ${RUNNER_UID}:${RUNNER_GID} +WORKDIR /github/workspace diff --git a/.github/ci-local/Dockerfile.baremetal b/.github/ci-local/Dockerfile.baremetal new file mode 100644 index 000000000..d6763a03c --- /dev/null +++ b/.github/ci-local/Dockerfile.baremetal @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2026 Intel Corporation + +# The second kind of runner in this repository's CI: `runs-on: ${{ matrix.nic }}`, +# a bare-metal host that owns a NIC and consumes the artifacts the build runner +# produced. smoke-tests, gtest-bare-metal and the pytest workflows all land here +# through .github/actions/validate-host. +# +# It is the build image plus the tooling those jobs use, so the expensive apt +# layer is shared rather than duplicated. +# +# What a container cannot simulate, and the job scripts say so out loud: +# * VFs, PFs and PCI binding -- no devices are passed in; +# * the Kahawai ICE driver -- a container shares the host kernel and has no +# business replacing its modules; +# * hugepages and PTP. +# Everything before that point -- cache restore, artifact layout, library and +# plugin resolution, the environment the tests inherit -- is exactly what the +# real host does, and that is where these jobs actually break. + +ARG BASE_IMAGE=mtl-ci-local:22.04-1000 +FROM ${BASE_IMAGE} + +USER root + +ARG RUNNER_UID=1000 +ARG RUNNER_GID=1000 + +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + iproute2 \ + jq \ + kmod \ + openssh-client \ + pciutils \ + rsync \ + tcpdump; \ + rm -rf /var/lib/apt/lists/* + +USER ${RUNNER_UID}:${RUNNER_GID} +WORKDIR /github/workspace diff --git a/.github/ci-local/README.md b/.github/ci-local/README.md new file mode 100644 index 000000000..11cd21ac1 --- /dev/null +++ b/.github/ci-local/README.md @@ -0,0 +1,193 @@ + + + +# Local CI + +Runs the GitHub Actions jobs of this repository on your own machine, inside a +container that simulates the runner. The point is reproduction: a failure you +can only observe by pushing a commit and reading 40 MB of log takes minutes per +attempt, and the log rarely contains the state you actually need. + +```sh +# everything a pull request triggers, in the order GitHub runs it +.github/ci-local/test-pr-locally.sh + +# run the `build` job of .github/workflows/build.yml +.github/ci-local/run-job.sh build + +# run .github/actions/validate-host as one matrix NIC would +.github/ci-local/run-job.sh validate-host --nic e810 + +# rebuild one component even though it is cached +.github/ci-local/run-job.sh build --force mtl + +# throw away the working copy and take a fresh one +.github/ci-local/run-job.sh build --clean + +# get a shell on the prepared runner instead of running the job +.github/ci-local/run-job.sh build --shell +``` + +The common entry points are also [Taskfile](../../Taskfile.yml) tasks, so a +local run and a runner run invoke the same command: `task ci:test-pr`, +`task ci:build`, `task ci:validate-host NIC=e830`, `task ebpf:check`. + +An agent can drive local jobs and inspect production checks through the +`mtl-ci-local` MCP server (`.github/mcp/run_ci_server.sh`). Local tools include +`ci_test_pr`, `ci_run_job`, `ci_cache_status`, `ci_check_ebpf`, `ci_list_tasks`, +`ci_run_task`, `ci_last_log`, and `ci_diagnostics`. `ci_pr_checks` and +`ci_pr_failures` use the authenticated `gh` CLI to inspect a pushed PR. All +results are bounded; the production tools are read-only and never push, rerun, +comment, or merge. + +The MCP layer is only an adapter. `ci_test_pr` calls +`test-pr-locally.sh`, and `ci_run_task` calls the root Taskfile. +Changes to `.github/mcp/mtl_ci_mcp_server.py` must exercise each affected +wrapper through result formatting; `py_compile` alone cannot detect failures +such as using a summary variable before assignment or calling the shared +summary helper with the wrong signature. + +For pushed branches, call `ci_pr_checks` first. If it reports failures, call +`ci_pr_failures`; it prefers check-run annotations and falls back to a bounded, +prefix-stripped extract from failed-job logs. This avoids loading entire GitHub +Actions logs into the agent context. +Only failure-level annotations suppress the log fallback. Check rows and +failed-check sections, annotations, and diagnostic excerpts have hard caps, +with omitted counts included in the result. GitHub command failures never +include raw stderr because authentication diagnostics may contain credentials. +Generic failure annotations do not suppress failed-log extraction; they lack +the compiler or linter message needed to diagnose the failure. + +## What `test-pr-locally.sh` covers + +```text +Lint Code Base linter.yml (opt-in: --with-lint) + └─ build build.yml produces .local_install/* + └─ pr-gate pr-gate.yml would the test workflows run? + └─ validate-host per NIC smoke-tests.yml, gtest-bare-metal.yml +``` + +The tests themselves need a NIC, VFs, the Kahawai ICE driver and hugepages, so +they stay on real hardware. Cache restore, JPEG XS and ICE artifact validation, +library and plugin resolution, and ICE activation ordering run here. Activation +uses the production script's dry-run mode and never changes the host kernel. + +Results land in `.ci-local/` (git-ignored): + +| path | contents | +| --------------------------- | -------------------------------------------- | +| `.ci-local/logs/-*.log` | full job output, one file per run | +| `.ci-local/out/diagnostics.txt` | environment state captured after every run | +| `.ci-local/src/` | the checkout the job was built from | + +The checkout is taken from `git ls-files`, so it holds what a real checkout +holds and none of the host's build directories, which record absolute host +paths meson cannot relocate. It is updated in place rather than mirrored, +because the job downloads its own sources into the workspace — SVT-JPEG-XS, +the DPDK tarball, FFmpeg — and deleting those every run turns a 90 second +rerun into a 20 minute one. Use `--clean` when the copy needs to go. + +The install tree is `.local_install/` in the repository root — the same path +the workflow uses — so it survives between runs and unchanged components are +not rebuilt. + +## What maps to what + +`run-job.sh` plays the parts GitHub plays. It never re-implements a job step; +the steps live in `jobs/.sh`, one section per workflow step. + +| GitHub | here | +| ----------------------------- | ------------------------------------------- | +| `build` on `runs-on: e835` | `Dockerfile` | +| `runs-on: ${{ matrix.nic }}` | `Dockerfile.baremetal`, `run-job.sh --nic` | +| `actions/checkout` | `git ls-files` piped into `rsync`, into `.ci-local/src` | +| the `checksums` job | `script/hash_sources.sh`, the same helper CI calls | +| `actions/cache` restore | stamp files under `.local_install/.stamps/` | +| the job steps | `jobs/.sh` | +| `.github/actions/validate-host` | `jobs/validate-host.sh` | +| `actions/cache` post: save | stamps refreshed after the job, on success only | + +```mermaid +flowchart LR + subgraph host["host — run-job.sh"] + IMG["build runner image
Dockerfile"] + SYNC["sync working tree
.ci-local/src"] + HASH["hash_sources.sh
cache keys"] + REST["restore: stamp + tree check
HIT / MISS / STALE"] + SAVE["save: refresh stamps
only if job passed"] + end + subgraph cont["container — jobs/build.sh"] + EVAL["Evaluate cache results
CI_BUILD_*"] + BUILD["Build and validate
task ci:build-dependencies"] + DIAG["diagnostics.txt"] + end + IMG --> SYNC --> HASH --> REST --> EVAL --> BUILD --> DIAG --> SAVE +``` + +## Cache integrity + +The local harness and build workflow enforce the same two rules: + +1. **A hit must be usable, not merely present.** `tree_is_usable()` in + `run-job.sh` looks for the artifact that consumers actually resolve — + `libdpdk.pc`, `mtl.pc`, the JPEG XS manifest and plugin, `libavcodec.pc`, a + plugin `.so`, or the kernel-specific ICE module and metadata. A key match + with an unusable tree is reported `STALE` and rebuilt. +1. **Only a passing job may write the cache.** Stamps are refreshed after the + container exits, and only on success, so a broken tree can never become a + permanent hit. + +## Why the image exists + +A bare `ubuntu:24.04` fails for reasons that have nothing to do with the code +under test, and each one costs a debugging round trip: + +- apt resolves the Ubuntu mirrors over IPv6, which many hosts cannot route; +- `sudo` resets the environment, so the proxy variables and + `DEBIAN_FRONTEND` are dropped and `tzdata` stops on an interactive prompt — + `setup_environment.sh` installs everything through `sudo`; +- ~400 packages are downloaded on every single run. + +The image fixes all three, and runs as your own uid with passwordless sudo, so +files it writes into the bind-mounted workspace stay yours. + +It also installs packages the workflow itself never installs, because the +self-hosted runner already has them from earlier runs. See the second finding +below. + +## What running it locally found + +Reproducible with the commands above. + +1. **`libdpdk` needs `libelf-dev`, which nothing installs.** It arrives only + under `SETUP_BUILD_AND_INSTALL_EBPF_XDP`, which the `build` job leaves off. + The runner has it from an earlier XDP run, so the job passes there and + fails on any freshly provisioned machine whose DPDK cache is a hit — + `Package 'libelf', required by 'libdpdk', not found`. +1. **The second MTL build step is a silent no-op.** + `.github/scripts/setup_environment.sh` carries two `MTL_BUILD_AND_INSTALL` + blocks. The first runs `./build.sh`, which builds. The second runs + `./build.sh "${mtl_build_options}"`, and with fuzzing off that expands to + `./build.sh ''` — an empty positional argument, which `build.sh` rejects by + printing its usage and exiting **0**. `set -e` cannot catch a zero exit, so + the step is reported as having built MTL. Harmless only because the first + block already did. +1. **The GStreamer plugins were unloadable.** `validate-host` set + `GST_PLUGIN_PATH` to the cached plugin directory but did not put that + directory on `LD_LIBRARY_PATH`. The plugins there link against their own + `libgstmtl_common.so`, which lives beside them and is not installed + system-wide, so every one failed to load with + `libgstmtl_common.so: cannot open shared object file`. Fixed in the action; + `jobs/validate-host.sh` asserts `gst-inspect-1.0 mtl_st20p_tx` succeeds so it + cannot regress silently. +1. **libbpf installed successfully but was invisible to `pkg-config`.** Its + upstream default is `/usr/local/lib64`, which Ubuntu does not include in the + default pkg-config search path. `build_ebpf_xdp.sh` installs bundled libbpf + into `/usr/local/lib/$(cc -dumpmachine)` and refreshes `ldconfig`, matching + the multiarch path used by the rest of the job. + +## Adding a job + +Add `jobs/.sh`, mirroring the workflow steps in order and reading its +inputs from the `CI_LOCAL_*` variables `run-job.sh` exports. Then +`run-job.sh `. diff --git a/.github/ci-local/jobs/build.sh b/.github/ci-local/jobs/build.sh new file mode 100755 index 000000000..fb263d749 --- /dev/null +++ b/.github/ci-local/jobs/build.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash + +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2026 Intel Corporation +# +# The `build` job of .github/workflows/build.yml, as executed inside the +# simulated runner. Each section below is one step of that job, in order. +# +# Inputs come from .github/ci-local/run-job.sh, which has already played the +# part of the runner and of actions/cache: +# +# CI_LOCAL_WORKDIR where the checkout is mounted +# CI_LOCAL_OUT directory collected back to the host afterwards +# CI_LOCAL_MISS_ 1 when the component must be rebuilt +# CI_LOCAL_HASH_ the cache key the workflow would have used +# CI_LOCAL_SHELL 1 to stop before the build and hand over a shell + +set -uo pipefail + +WORKDIR="${CI_LOCAL_WORKDIR:-/github/workspace}" +OUT_DIR="${CI_LOCAL_OUT:-/github/out}" +LOCAL_INSTALL="${WORKDIR}/.local_install" +DIAG_FILE="${OUT_DIR}/diagnostics.txt" + +cd "${WORKDIR}" || exit 1 + +# ── step: system: Check eBPF/XDP prerequisites ────────────────────────────── +# Mirrors .github/actions/check-ebpf, which the build job runs first. +echo "::group::system: Check eBPF/XDP prerequisites" +if ! bash "${WORKDIR}/script/build_ebpf_xdp.sh" --check --mode build --strict; then + echo "::error::build job failed its eBPF/XDP prerequisite check" + exit 1 +fi +echo "::endgroup::" + +# ── step: Evaluate cache results ──────────────────────────────────────────── +echo "::group::Evaluate cache results" +export CI_BUILD_DPDK="${CI_LOCAL_MISS_DPDK:-1}" +export CI_BUILD_MTL="${CI_LOCAL_MISS_MTL:-1}" +export CI_BUILD_JPEGXS="${CI_LOCAL_MISS_JPEGXS:-1}" +export CI_BUILD_FFMPEG="${CI_LOCAL_MISS_FFMPEG:-1}" +export CI_BUILD_GSTREAMER="${CI_LOCAL_MISS_GSTREAMER:-1}" +export CI_BUILD_PLUGINS="${CI_LOCAL_MISS_PLUGINS:-1}" +export CI_BUILD_ICE="${CI_LOCAL_MISS_ICE:-1}" + +state() { [ "$1" = "1" ] && echo MISS || echo HIT; } +echo "::notice::DPDK=$(state "${CI_BUILD_DPDK}")" \ + "MTL=$(state "${CI_BUILD_MTL}")" \ + "JPEGXS=$(state "${CI_BUILD_JPEGXS}")" \ + "FFmpeg=$(state "${CI_BUILD_FFMPEG}")" \ + "GStreamer=$(state "${CI_BUILD_GSTREAMER}")" \ + "plugins=$(state "${CI_BUILD_PLUGINS}")" \ + "ICE=$(state "${CI_BUILD_ICE}")" + +any_miss=0 +for miss in "${CI_BUILD_DPDK}" "${CI_BUILD_MTL}" "${CI_BUILD_JPEGXS}" \ + "${CI_BUILD_FFMPEG}" "${CI_BUILD_GSTREAMER}" "${CI_BUILD_PLUGINS}" \ + "${CI_BUILD_ICE}"; do + [ "${miss}" = "1" ] && any_miss=1 +done +echo "::endgroup::" + +# ── step: Setup environment and build ─────────────────────────────────────── +# Environment copied verbatim from the workflow step of the same name. +# The workflow's runner is long-lived and already provisioned; a fresh +# container is not, so it takes the same path CI takes on a clean machine. +export SETUP_ENVIRONMENT=1 +export CICD_BUILD=1 + +dump_diagnostics() { + mkdir -p "${OUT_DIR}" + { + echo "=== when ===" + date -u +%Y-%m-%dT%H:%M:%SZ + echo + echo "=== os ===" + uname -a + # shellcheck disable=SC1091 + (. /etc/os-release && echo "${PRETTY_NAME}") + echo + echo "=== build environment ===" + env | grep -E '^(CI_BUILD_|MTL_|SETUP_|ECOSYSTEM_|PLUGIN_|TOOLS_|CICD_|PKG_CONFIG|LD_LIBRARY)' | sort + echo + echo "=== pkg-config: what the build can see ===" + pkg-config --list-all 2>/dev/null | grep -iE 'mtl|dpdk|jpeg' || echo "(nothing)" + echo + echo "=== pkg-config: the check that fails in CI ===" + pkg-config --print-errors --exists 'mtl >= 22.12.0' 2>&1 && + echo "mtl >= 22.12.0 OK" || echo "mtl >= 22.12.0 NOT FOUND" + echo + echo "=== .pc files under .local_install ===" + find "${LOCAL_INSTALL}" -name '*.pc' 2>/dev/null || echo "(none)" + echo + echo "=== .local_install tree (depth 3) ===" + find "${LOCAL_INSTALL}" -maxdepth 3 2>/dev/null || echo "(missing)" + echo + echo "=== cache stamps ===" + for f in "${LOCAL_INSTALL}"/.stamps/*; do + [ -e "$f" ] && echo "$(basename "$f")=$(cat "$f")" + done + echo + echo "=== disk ===" + df -h "${WORKDIR}" 2>/dev/null + } >"${DIAG_FILE}" 2>&1 + echo "diagnostics written to ${DIAG_FILE}" +} + +if [ "${CI_LOCAL_SHELL:-0}" = "1" ]; then + echo "Runner ready. The build environment is exported; run:" + echo " task ci:build-dependencies" + exec bash -i +fi + +if [ "${any_miss}" = "0" ]; then + echo "::notice::every component cached, nothing to build" + dump_diagnostics + exit 0 +fi + +echo "::group::Setup environment and build" +rc=0 +task ci:build-dependencies || rc=$? +if [ "$rc" -eq 0 ]; then + task ci:validate-dependencies || rc=$? +fi +echo "::endgroup::" + +dump_diagnostics + +if [ "${rc}" -ne 0 ]; then + echo "::error::build job failed with exit code ${rc}" +fi +exit "${rc}" diff --git a/.github/ci-local/jobs/validate-host.sh b/.github/ci-local/jobs/validate-host.sh new file mode 100755 index 000000000..906b90316 --- /dev/null +++ b/.github/ci-local/jobs/validate-host.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash + +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2026 Intel Corporation +# +# The .github/actions/validate-host composite action, as executed inside the +# simulated bare-metal runner. Each section below is one step of that action, +# in order. smoke-tests, gtest-bare-metal and the pytest workflows all begin +# by running it, and when it is wrong every one of them fails identically and +# unhelpfully -- so it is worth being able to run on its own. +# +# Inputs come from .github/ci-local/run-job.sh: +# +# CI_LOCAL_WORKDIR where the checkout is mounted +# CI_LOCAL_OUT directory collected back to the host afterwards +# CI_LOCAL_MISS_ 1 when the cache did not supply the component +# CI_LOCAL_NIC the matrix NIC being simulated +# PCI_DEVICE the PCI IDs that NIC maps to +# +# Two steps of the real action cannot run in a container and are reported as +# skipped rather than faked: the ICE driver alignment, which would replace a +# module in the host kernel, and the shadow-host rsync, which needs a second +# machine. + +set -uo pipefail + +WORKDIR="${CI_LOCAL_WORKDIR:-/github/workspace}" +OUT_DIR="${CI_LOCAL_OUT:-/github/out}" +NIC="${CI_LOCAL_NIC:-}" +LOCAL_INSTALL="${WORKDIR}/.local_install" +ENV_FILE="${OUT_DIR}/validate-host.env" +PATH_FILE="${OUT_DIR}/validate-host.path" +export GITHUB_ENV="$ENV_FILE" +export GITHUB_PATH="$PATH_FILE" + +cd "${WORKDIR}" || exit 1 +mkdir -p "${OUT_DIR}" +: >"${ENV_FILE}" +: >"${PATH_FILE}" + +rc=0 +failed_step="" +step_failed() { + rc=1 + failed_step="${failed_step}${failed_step:+, }$1" + echo "::error::$1" +} + +# ── step: system: Check eBPF/XDP prerequisites ────────────────────────────── +# The action runs this first and strict, so a host that cannot serve AF_XDP +# fails before any setup. Non-strict here: a container has no say over the +# kernel it was given, so enforcing kernel CONFIG_* would fail every local run. +echo "::group::system: Check eBPF/XDP prerequisites" +bash "${WORKDIR}/script/build_ebpf_xdp.sh" --check --mode all +echo "::endgroup::" + +# ── step: cache: Restore dependency artifacts ─────────────────────────────── +# The action restores each with fail-on-cache-miss: true, except plugins, +# whose absence only disables the codec tests. A miss here is the local +# equivalent of that hard failure -- it means the build job never published +# an artifact for these sources. +echo "::group::cache: Restore build artifacts" +for comp in dpdk mtl jpegxs ffmpeg gstreamer plugins ice; do + upper="${comp^^}" + miss_var="CI_LOCAL_MISS_${upper}" + if [ "${!miss_var:-1}" = "1" ]; then + step_failed "cache: ${comp} missing -- run the build job first (fail-on-cache-miss: true)" + else + echo " restored ${comp}" + fi +done +echo "::endgroup::" + +if [ "${rc}" -ne 0 ]; then + echo "::error::validate-host cannot continue without the build artifacts" + echo "failed steps: ${failed_step}" + exit 1 +fi + +# ── step: Make artifacts executable ───────────────────────────────────────── +task ci:validate-dependencies || step_failed "cache: structural validation failed" +task ci:configure-host -- make-executable || step_failed "cache: executable alignment failed" + +# ── step: kahawai: Generate CI plugin registry from cache ─────────────────── +echo "::group::kahawai: Generate CI plugin registry from cache" +RUNNER_TEMP="$OUT_DIR" task ci:configure-host -- registry || step_failed "kahawai: registry generation failed" +echo "::endgroup::" + +# ── step: activation: Model ICE ordering without host mutation ────────────── +echo "::group::activation: ICE dry run" +ICE_ACTIVATION_STAMP="${OUT_DIR}/ice.state" \ + ICE_COMMAND_LOG="${OUT_DIR}/ice-activation.log" \ + task ci:activate-ice -- --dry-run || step_failed "activation: ICE dry run failed" +echo "::endgroup::" + +# ── step: Configure LD_LIBRARY_PATH, PATH, GST_PLUGIN_PATH ────────────────── +# Written to a file, which is what $GITHUB_ENV and $GITHUB_PATH are. +echo "::group::Configure environment" +LI="${LOCAL_INSTALL}" +task ci:configure-host -- environment || step_failed "environment: configuration failed" +# shellcheck disable=SC1090 # generated above +set -a && . "${ENV_FILE}" && set +a +PATH="$(paste -sd: "$PATH_FILE"):${PATH}" +export PATH +echo "::notice::Host environment configured from validated caches" +echo "::endgroup::" + +# ── verification ──────────────────────────────────────────────────────────── +# Not a step of the action: the action configures an environment and never +# checks that it works. Every consumer then fails on its own symptom. These +# are the resolutions the test jobs depend on, asserted once, here. +echo "::group::verify: the environment the tests will inherit" + +check() { + # check