Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions .github/config/ubsan_suppressions.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# UBSan/IntSan suppression file for known false positives in libstdc++ headers.
# These are intentional unsigned arithmetic patterns in the C++ standard library
# that cannot be fixed in project code.

# std::string::npos is size_t(-1); comparisons against it use intentional unsigned wrap
unsigned-integer-overflow:*bits/basic_string.h*

# Mersenne Twister PRNG uses deliberate unsigned modular arithmetic
unsigned-integer-overflow:*bits/random.tcc*

# uniform_int_distribution rejection sampling uses unsigned negation (-range)
unsigned-integer-overflow:*bits/uniform_int_dist.h*
implicit-integer-sign-change:*bits/uniform_int_dist.h*
233 changes: 233 additions & 0 deletions .github/workflows/sanitizer_tests.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,233 @@
name: Sanitizer Tests

on:
schedule:
- cron: '0 18 * * *' # Daily 18:00 UTC (20:00 CEST / 19:00 CET)
workflow_dispatch:
push:
branches: ['ajanczew-sanitizers'] # TODO: remove before merge

defaults:
run:
shell: bash

permissions:
contents: read

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
JOBS_NUM: 16

jobs:
# ── Address / Thread / Undefined / Integer sanitizers ──────────────────────
# Release build with clang; runs unit tests (parrallelUT.sh) + conformance
# (ParallelAllTests.sh) non-halting, then gates once via the summary step.
sanitizer-tests:
name: "Sanitizer: ${{ matrix.sanitizer }}"
runs-on: ['self-hosted', 'linux', 'x64', 'docker', 'kubernetes', 'valgrind', 'jpeg-perf']
timeout-minutes: 180
strategy:
fail-fast: false
matrix:
include:
- sanitizer: address
- sanitizer: thread
- sanitizer: undefined
- sanitizer: integer
steps:
- name: 'Harden Runner'
uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
with:
egress-policy: audit

- name: 'Setup: Cleanup workspace'
run: |
rm -rf ./* || true
rm -rf ./.??* || true

- name: 'Checkout repository'
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

- name: 'Setup: Install dependencies'
run: |
sudo apt-get update
sudo apt-get -y install cmake nasm clang llvm make

- name: 'Fix workspace permissions'
if: always()
run: |
sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE" || true
chmod -R u+rwX "$GITHUB_WORKSPACE" || true

- name: 'Build: release with ${{ matrix.sanitizer }} sanitizer'
working-directory: Build/linux
run: |
./build.sh release test sanitizer=${{ matrix.sanitizer }} cc=clang cxx=clang++ jobs=$(nproc)

Comment thread
ajanczew marked this conversation as resolved.
- name: 'Unit tests: ${{ matrix.sanitizer }} sanitizer (release)'
# continue-on-error: collect all findings; the summary step is the single gate
continue-on-error: true
env:
# Non-halting: one pass collects every finding; the summary step gates.
ASAN_OPTIONS: "halt_on_error=0:exitcode=0:print_stacktrace=1"
TSAN_OPTIONS: "halt_on_error=0:exitcode=0:history_size=4"
UBSAN_OPTIONS: "suppressions=${{ github.workspace }}/.github/config/ubsan_suppressions.txt:print_stacktrace=1:halt_on_error=0:exitcode=0"
run: |
cd Bin/Release/
chmod +x ${{ github.workspace }}/tests/scripts/parrallelUT.sh
chmod +x ./*
export LD_LIBRARY_PATH=$(pwd)
mkdir -p "${{ github.workspace }}/sanitizer-logs"
# pipefail so a failing shard is not masked by tee
set -o pipefail
${{ github.workspace }}/tests/scripts/parrallelUT.sh ./SvtJpegxsUnitTests ${{ env.JOBS_NUM }} \
2>&1 | tee "${{ github.workspace }}/sanitizer-logs/unit-tests-${{ matrix.sanitizer }}.log"

- name: 'Conformance tests: ${{ matrix.sanitizer }} sanitizer (release)'
# continue-on-error: collect all findings; the summary step is the single gate
continue-on-error: true
working-directory: ${{ github.workspace }}/tests/scripts
env:
# Non-halting: one pass collects every finding; the summary step gates.
ASAN_OPTIONS: "halt_on_error=0:exitcode=0:print_stacktrace=1"
TSAN_OPTIONS: "halt_on_error=0:exitcode=0:history_size=4"
UBSAN_OPTIONS: "suppressions=${{ github.workspace }}/.github/config/ubsan_suppressions.txt:print_stacktrace=1:halt_on_error=0:exitcode=0"
DEC_BIN_DIR: ${{ github.workspace }}/Bin/Release
INPUT_FILES_PATH: /opt/samples
run: |
mkdir -p "${{ github.workspace }}/sanitizer-logs"
chmod +x ./*.sh
chmod +x "$DEC_BIN_DIR"/*
export LD_LIBRARY_PATH="$DEC_BIN_DIR"
# pipefail so a failing conformance shard is not masked by tee
set -o pipefail
./ParallelAllTests.sh ${{ env.JOBS_NUM }} "$INPUT_FILES_PATH" "$DEC_BIN_DIR/SvtJpegxsDecApp" \
2>&1 | tee "${{ github.workspace }}/sanitizer-logs/conformance-tests-${{ matrix.sanitizer }}.log"

- name: 'Summarize: ${{ matrix.sanitizer }} sanitizer findings'
if: always()
working-directory: ${{ github.workspace }}
run: |
chmod +x tests/scripts/sanitizer_summary.sh
# integer is report-only (does not gate on findings); every other sanitizer fails on findings.
# A non-zero test-harness exit (broken run) fails the job for ALL sanitizers, integer included.
tests/scripts/sanitizer_summary.sh ${{ matrix.sanitizer == 'integer' && '--no-gate ' || '' }}"${{ matrix.sanitizer }}" \
sanitizer-logs/unit-tests-${{ matrix.sanitizer }}.log \
sanitizer-logs/conformance-tests-${{ matrix.sanitizer }}.log

- name: 'Archive: upload binary and logs on failure'
if: failure()
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: sanitizer-${{ matrix.sanitizer }}-failure-${{ github.run_number }}
retention-days: 7
path: |
Bin/Release/SvtJpegxsUnitTests
Bin/Release/SvtJpegxsEncApp
Bin/Release/SvtJpegxsDecApp
sanitizer-logs/unit-tests-${{ matrix.sanitizer }}.log
sanitizer-logs/conformance-tests-${{ matrix.sanitizer }}.log

# Integer sanitizer runs report-only (halt_on_error=0), so the job stays
# green; always archive its logs so the findings are not lost.
- name: 'Archive: upload integer sanitizer report'
if: always() && matrix.sanitizer == 'integer'
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: sanitizer-integer-report-${{ github.run_number }}
retention-days: 7
path: |
sanitizer-logs/unit-tests-integer.log
sanitizer-logs/conformance-tests-integer.log

- name: 'Cleanup: Remove workspace artifacts'
if: always()
run: |
rm -rf Bin/ Build/linux/Release/ || true

# ── Memory sanitizer ────────────────────────────────────────────────────────
# Must be Debug (release does not build with MSan).
# Unit tests can't run under MSan: gtest aborts during global init.
# Use the C encoder/decoder apps instead.
sanitizer-memory:
name: "Sanitizer: memory"
runs-on: ['self-hosted', 'linux', 'x64', 'docker', 'kubernetes', 'valgrind', 'jpeg-perf']
timeout-minutes: 180
steps:
- name: 'Harden Runner'
uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
with:
egress-policy: audit

- name: 'Setup: Cleanup workspace'
run: |
rm -rf ./* || true
rm -rf ./.??* || true

- name: 'Checkout repository'
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

- name: 'Setup: Install dependencies'
run: |
sudo apt-get update
sudo apt-get -y install cmake nasm clang llvm make

- name: 'Fix workspace permissions'
if: always()
run: |
sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE" || true
chmod -R u+rwX "$GITHUB_WORKSPACE" || true

- name: 'Build: debug with memory sanitizer'
working-directory: Build/linux
run: |
./build.sh debug sanitizer=memory cc=clang cxx=clang++ jobs=$(nproc)

- name: 'Conformance tests: memory sanitizer (debug)'
# continue-on-error: collect all findings; the summary step is the single gate
continue-on-error: true
working-directory: ${{ github.workspace }}/tests/scripts
env:
DEC_BIN_DIR: ${{ github.workspace }}/Bin/Debug
INPUT_FILES_PATH: /opt/samples
# Non-halting: collect findings in one pass; the summary step gates.
MSAN_OPTIONS: "halt_on_error=0:exitcode=0"
# avx2: AVX-512 paths produce MSan false positives
SANITIZER_ASM: avx2
run: |
mkdir -p "${{ github.workspace }}/sanitizer-logs"
chmod +x ./*.sh
chmod +x "$DEC_BIN_DIR"/*
export LD_LIBRARY_PATH="$DEC_BIN_DIR"
# pipefail so a failing conformance shard is not masked by tee
set -o pipefail
# fast: reduce the matrix (MSan is very slow); SANITIZER_ASM pins avx2
./ParallelAllTests.sh ${{ env.JOBS_NUM }} "$INPUT_FILES_PATH" "$DEC_BIN_DIR/SvtJpegxsDecApp" fast \
2>&1 | tee "${{ github.workspace }}/sanitizer-logs/memory-conformance-tests.log"

- name: 'Summarize: memory sanitizer findings'
if: always()
working-directory: ${{ github.workspace }}
run: |
chmod +x tests/scripts/sanitizer_summary.sh
tests/scripts/sanitizer_summary.sh "memory" \
sanitizer-logs/memory-conformance-tests.log

- name: 'Archive: upload binaries and logs on failure'
if: failure()
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: sanitizer-memory-failure-${{ github.run_number }}
retention-days: 7
path: |
Bin/Debug/SvtJpegxsEncApp
Bin/Debug/SvtJpegxsDecApp
sanitizer-logs/memory-conformance-tests.log

- name: 'Cleanup: Remove workspace artifacts'
if: always()
run: |
rm -rf Bin/ Build/linux/Debug/ || true
11 changes: 6 additions & 5 deletions tests/UnitTests/UnitTest_AVX512/CodeDeprecated-avx512.cc
Original file line number Diff line number Diff line change
Expand Up @@ -237,17 +237,18 @@ int dwt_horizontal_depricated_avx512(int32_t* out_lf, int32_t* out_hf, const int
}

if (remaining > 1 && simd_batch > 0) {
dst_lf_row[0] = src_row[0] + ((dst_hf_row[-1] + dst_hf_row[0] + 2) >> 2);
// Widen to int64 so the intermediate sums cannot overflow int32 (UBSan).
dst_lf_row[0] = (int32_t)((int64_t)src_row[0] + (((int64_t)dst_hf_row[-1] + dst_hf_row[0] + 2) >> 2));
}
else if (remaining > 1) {
// Fix: When simd_batch==0, dst_hf_row[-1] is before the allocated buffer.
// Use the boundary formula instead; this value gets overwritten by
// "correct the first for lf" anyway.
dst_lf_row[0] = src_row[0] + ((dst_hf_row[0] + 1) >> 1);
dst_lf_row[0] = (int32_t)((int64_t)src_row[0] + (((int64_t)dst_hf_row[0] + 1) >> 1));
}

for (uint32_t i = 1; i < (remaining / 2); i++) {
dst_lf_row[i] = src_row[2 * i] + ((dst_hf_row[i - 1] + dst_hf_row[i] + 2) >> 2);
dst_lf_row[i] = (int32_t)((int64_t)src_row[2 * i] + (((int64_t)dst_hf_row[i - 1] + dst_hf_row[i] + 2) >> 2));
}
dst_lf_row += remaining / 2;
dst_hf_row += remaining / 2;
Expand All @@ -256,15 +257,15 @@ int dwt_horizontal_depricated_avx512(int32_t* out_lf, int32_t* out_hf, const int

// correct the last for hf and lf
if (width % 2) {
dst_lf_row[0] = src_row[-1] + ((dst_hf_row[-1] + 1) >> 1);
dst_lf_row[0] = (int32_t)((int64_t)src_row[-1] + (((int64_t)dst_hf_row[-1] + 1) >> 1));
}
else {
dst_hf_row[-1] = src_row[-1] - src_row[-2];
// Fix: For width==2 there is only 1 HF element, so dst_hf_row[-2] is before
// the buffer. Skip the LF correction here; "correct the first for lf" below
// will write the correct value for LF[0].
if (width >= 4) {
dst_lf_row[-1] = src_row[-2] + ((dst_hf_row[-2] + dst_hf_row[-1] + 2) >> 2);
dst_lf_row[-1] = (int32_t)((int64_t)src_row[-2] + (((int64_t)dst_hf_row[-2] + dst_hf_row[-1] + 2) >> 2));
}
}

Expand Down
4 changes: 2 additions & 2 deletions tests/scripts/DecoderConformanceTest.sh
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ function test_dec {
return
fi

cmd="$valgrind$exec_dec --find-bitstream-header -i $path_bitstreams/test_bitsreams/$name.jxs -o ./$tmp_dir/$yuv_name --lp $PARAM_LP_NUM --asm $PARAM_ASM --packetization-mode $PARAM_PACKETIZATION"
cmd="$valgrind$exec_dec --find-bitstream-header -i $path_bitstreams/test_bitsreams/$name.jxs -o ./$tmp_dir/$yuv_name --lp $PARAM_LP_NUM --asm ${SANITIZER_ASM:-$PARAM_ASM} --packetization-mode $PARAM_PACKETIZATION"
echo "run command: $cmd"
${cmd}

Expand Down Expand Up @@ -87,7 +87,7 @@ function test_msb_aligned_output {
return
fi

cmd="$valgrind$exec_dec -i $bitstream_msb_prep -o $yuv_tmp --lp $PARAM_LP_NUM --asm $PARAM_ASM --packetization-mode $PARAM_PACKETIZATION "$params
cmd="$valgrind$exec_dec -i $bitstream_msb_prep -o $yuv_tmp --lp $PARAM_LP_NUM --asm ${SANITIZER_ASM:-$PARAM_ASM} --packetization-mode $PARAM_PACKETIZATION "$params
echo "run command: $cmd"
${cmd}

Expand Down
6 changes: 5 additions & 1 deletion tests/scripts/DecoderMultiFramesTest.sh
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ function test_dec {
return
fi

cmd="$valgrind$exec_dec -i $bin_name -o $yuv_tmp --lp $PARAM_LP_NUM --asm $PARAM_ASM --packetization-mode $PARAM_PACKETIZATION "$params
cmd="$valgrind$exec_dec -i $bin_name -o $yuv_tmp --lp $PARAM_LP_NUM --asm ${SANITIZER_ASM:-$PARAM_ASM} --packetization-mode $PARAM_PACKETIZATION "$params
echo "run command: $cmd"
${cmd}

Expand Down Expand Up @@ -142,6 +142,10 @@ function test_all_broken {
PARAM_LP_NUM=$2
PARAM_PACKETIZATION="0"

# broken_* fixtures live alongside the correct ones; set explicitly so this
# function is order-independent (in 'fast' mode test_all_correct is skipped).
path_use=$path_correct

# TEST WITH IGNORE SOME FRAMES
#a. broken_decomh_Daylight_1280x720_8b_422_20fx1fx20f.jxs
#1. 20frames 8bit yuv422 1280x720 v1 h5
Expand Down
12 changes: 6 additions & 6 deletions tests/scripts/EncoderTest.sh
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ mkdir $tmp_dir

#RUN different RC parameters Release/Debug/ ASM_C/ASM_MAX compare Parameters (1:asm) (2:lp number)
function test_rate_control {
asm=$1
asm=${SANITIZER_ASM:-$1}
lp=$2
cpu_profile=$3
packetization_mode=$4
Expand Down Expand Up @@ -306,7 +306,7 @@ function test_rate_control {

#RUN different RC parameters Release/Debug/ ASM_C/ASM_MAX compare Parameters (1:asm) (2:lp number)
function test_rate_control_signs {
asm=$1
asm=${SANITIZER_ASM:-$1}
lp=$2
cpu_profile=$3
packetization_mode=$4
Expand All @@ -332,7 +332,7 @@ function test_rate_control_signs {

#RUN input-msb-aligned Parameters (1:asm) (2:lp number)
function test_msb_aligned {
asm=$1
asm=${SANITIZER_ASM:-$1}
lp=$2
cpu_profile=$3
packetization_mode=$4
Expand All @@ -350,7 +350,7 @@ function test_msb_aligned {
}

function test_uncommon_resolution {
asm=$1
asm=${SANITIZER_ASM:-$1}
lp=$2
cpu_profile=$3
packetization_mode=$4
Expand Down Expand Up @@ -634,7 +634,7 @@ function test_uncommon_resolution {
}

function test_handle_errors {
asm=$1
asm=${SANITIZER_ASM:-$1}
lp=$2
cpu_profile=$3
packetization_mode=$4
Expand Down Expand Up @@ -721,7 +721,7 @@ function test_handle_errors {

#RUN different RC parameters Release/Debug/ ASM_C/ASM_MAX compare Parameters (1:asm) (2:lp number)
function test_invalid_yuv {
asm=$1
asm=${SANITIZER_ASM:-$1}
lp=$2
cpu_profile=$3
packetization_mode=$4
Expand Down
Loading
Loading