Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 41 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
version: 2
updates:
- package-ecosystem: npm
directory: /
schedule:
interval: weekly
cooldown:
default-days: 7
open-pull-requests-limit: 5
labels: []
commit-message:
prefix: chore(deps)
groups:
npm:
update-types: [minor, patch]
- package-ecosystem: pip
directory: /analysis
schedule:
interval: weekly
cooldown:
default-days: 7
open-pull-requests-limit: 5
labels: []
commit-message:
prefix: chore(deps)
groups:
python:
update-types: [minor, patch]
- package-ecosystem: github-actions
directory: /
schedule:
interval: weekly
cooldown:
default-days: 7
open-pull-requests-limit: 5
labels: []
commit-message:
prefix: ci(deps)
groups:
github-actions:
update-types: [minor, patch]
77 changes: 53 additions & 24 deletions .github/workflows/benchmark-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,9 @@ jobs:
fail-fast: false
matrix: ${{ fromJson(needs.discover.outputs.matrix) }}
env:
BENCHMARK_UUID: ${{ matrix.benchmark_uuid }}
RUN_ID: ${{ matrix.run_id }}
TIMEOUT_HOURS: ${{ matrix.timeout_hours }}
AWS_REGION: ${{ secrets.AWS_REGION != '' && secrets.AWS_REGION || 'us-east-1' }}
SCFUZZBENCH_BUCKET: ${{ secrets.SCFUZZBENCH_BUCKET }}
steps:
Expand Down Expand Up @@ -211,13 +214,23 @@ jobs:
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ env.AWS_REGION }}

- name: Validate release identifiers
run: |
set -euo pipefail
for identifier in "${BENCHMARK_UUID}" "${RUN_ID}"; do
if [[ ! "${identifier}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$ ]]; then
echo "Invalid benchmark UUID or run ID" >&2
exit 1
fi
done

- name: Check for existing release
id: release_check
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
tag="scfuzzbench-${{ matrix.benchmark_uuid }}-${{ matrix.run_id }}"
tag="scfuzzbench-${BENCHMARK_UUID}-${RUN_ID}"
if gh release view "${tag}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then
echo "exists=true" >> "${GITHUB_OUTPUT}"
else
Expand All @@ -227,7 +240,7 @@ jobs:

- name: Setup Python
if: ${{ steps.release_check.outputs.exists != 'true' || inputs.force_reanalyze == 'true' }}
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.11"

Expand All @@ -242,19 +255,19 @@ jobs:
REPORT_BUDGET_HOURS: ${{ matrix.report_budget_hours }}
run: |
set -euo pipefail
dest_root="${GITHUB_WORKSPACE}/analysis-artifacts/${{ matrix.run_id }}"
dest_root="${GITHUB_WORKSPACE}/analysis-artifacts/${RUN_ID}"
analysis_dir="${dest_root}/analysis"
mkdir -p "${analysis_dir}"
logs_manifest_key="logs/${{ matrix.run_id }}/${{ matrix.benchmark_uuid }}/manifest.json"
runs_manifest_key="runs/${{ matrix.run_id }}/${{ matrix.benchmark_uuid }}/manifest.json"
logs_manifest_key="logs/${RUN_ID}/${BENCHMARK_UUID}/manifest.json"
runs_manifest_key="runs/${RUN_ID}/${BENCHMARK_UUID}/manifest.json"
if aws s3 cp "s3://${SCFUZZBENCH_BUCKET}/${logs_manifest_key}" "${analysis_dir}/manifest.json"; then
echo "Fetched manifest from ${logs_manifest_key}"
else
aws s3 cp "s3://${SCFUZZBENCH_BUCKET}/${runs_manifest_key}" "${analysis_dir}/manifest.json"
echo "Fetched manifest from ${runs_manifest_key}"
fi

timeout_hours="${{ matrix.timeout_hours }}"
timeout_hours="${TIMEOUT_HOURS}"
if [[ -z "${timeout_hours}" || "${timeout_hours}" == "null" ]]; then
timeout_hours=$(ANALYSIS_DIR="${analysis_dir}" python3 - <<'PY'
import json
Expand Down Expand Up @@ -298,10 +311,13 @@ jobs:

- name: Collect analysis artifacts
if: ${{ steps.release_check.outputs.exists != 'true' || inputs.force_reanalyze == 'true' }}
env:
DEST_ROOT: ${{ steps.manifest.outputs.dest_root }}
ANALYSIS_DIR: ${{ steps.manifest.outputs.analysis_dir }}
run: |
set -euo pipefail
dest_root="${{ steps.manifest.outputs.dest_root }}"
analysis_dir="${{ steps.manifest.outputs.analysis_dir }}"
dest_root="${DEST_ROOT}"
analysis_dir="${ANALYSIS_DIR}"

# Website + release notes expect these at the top-level.
cp "${dest_root}/data/REPORT.md" "${analysis_dir}/REPORT.md"
Expand Down Expand Up @@ -362,10 +378,13 @@ jobs:
- name: Build release bundles
if: ${{ steps.release_check.outputs.exists != 'true' || inputs.force_reanalyze == 'true' }}
id: bundles
env:
DEST_ROOT: ${{ steps.manifest.outputs.dest_root }}
ANALYSIS_DIR: ${{ steps.manifest.outputs.analysis_dir }}
run: |
set -euo pipefail
dest_root="${{ steps.manifest.outputs.dest_root }}"
analysis_dir="${{ steps.manifest.outputs.analysis_dir }}"
dest_root="${DEST_ROOT}"
analysis_dir="${ANALYSIS_DIR}"
release_dir="${dest_root}/release"
mkdir -p "${release_dir}"

Expand All @@ -389,11 +408,16 @@ jobs:

- name: Upload analysis artifacts to S3
if: ${{ steps.release_check.outputs.exists != 'true' || inputs.force_reanalyze == 'true' }}
env:
ANALYSIS_DIR: ${{ steps.manifest.outputs.analysis_dir }}
RELEASE_DIR: ${{ steps.bundles.outputs.release_dir }}
HAS_LOGS_ZIP: ${{ steps.bundles.outputs.has_logs_zip }}
HAS_CORPUS_ZIP: ${{ steps.bundles.outputs.has_corpus_zip }}
run: |
set -euo pipefail
analysis_dir="${{ steps.manifest.outputs.analysis_dir }}"
release_dir="${{ steps.bundles.outputs.release_dir }}"
s3_prefix="analysis/${{ matrix.benchmark_uuid }}/${{ matrix.run_id }}"
analysis_dir="${ANALYSIS_DIR}"
release_dir="${RELEASE_DIR}"
s3_prefix="analysis/${BENCHMARK_UUID}/${RUN_ID}"

aws s3 cp "${analysis_dir}/REPORT.md" "s3://${SCFUZZBENCH_BUCKET}/${s3_prefix}/REPORT.md" --content-type text/markdown
aws s3 cp "${analysis_dir}/bugs_over_time.png" "s3://${SCFUZZBENCH_BUCKET}/${s3_prefix}/bugs_over_time.png" --content-type image/png
Expand Down Expand Up @@ -447,27 +471,31 @@ jobs:
aws s3 cp "${analysis_dir}/manifest.json" "s3://${SCFUZZBENCH_BUCKET}/${s3_prefix}/manifest.json" --content-type application/json

aws s3 cp "${release_dir}/analysis.zip" "s3://${SCFUZZBENCH_BUCKET}/${s3_prefix}/bundles/analysis.zip" --content-type application/zip
if [[ "${{ steps.bundles.outputs.has_logs_zip }}" == "true" ]]; then
if [[ "${HAS_LOGS_ZIP}" == "true" ]]; then
aws s3 cp "${release_dir}/logs.zip" "s3://${SCFUZZBENCH_BUCKET}/${s3_prefix}/bundles/logs.zip" --content-type application/zip
fi
if [[ "${{ steps.bundles.outputs.has_corpus_zip }}" == "true" ]]; then
if [[ "${HAS_CORPUS_ZIP}" == "true" ]]; then
aws s3 cp "${release_dir}/corpus.zip" "s3://${SCFUZZBENCH_BUCKET}/${s3_prefix}/bundles/corpus.zip" --content-type application/zip
fi

- name: Compose release body
if: ${{ steps.release_check.outputs.exists != 'true' }}
id: body
env:
ANALYSIS_DIR: ${{ steps.manifest.outputs.analysis_dir }}
HAS_LOGS_ZIP: ${{ steps.bundles.outputs.has_logs_zip }}
HAS_CORPUS_ZIP: ${{ steps.bundles.outputs.has_corpus_zip }}
run: |
set -euo pipefail
analysis_dir="${{ steps.manifest.outputs.analysis_dir }}"
analysis_dir="${ANALYSIS_DIR}"
report_path="${analysis_dir}/REPORT.md"
release_body="${analysis_dir}/RELEASE.md"
cp "${report_path}" "${release_body}"

base_url="https://${SCFUZZBENCH_BUCKET}.s3.${AWS_REGION}.amazonaws.com"
analysis_base="${base_url}/analysis/${{ matrix.benchmark_uuid }}/${{ matrix.run_id }}"
logs_base="${base_url}/logs/${{ matrix.run_id }}/${{ matrix.benchmark_uuid }}"
corpus_base="${base_url}/corpus/${{ matrix.run_id }}/${{ matrix.benchmark_uuid }}"
analysis_base="${base_url}/analysis/${BENCHMARK_UUID}/${RUN_ID}"
logs_base="${base_url}/logs/${RUN_ID}/${BENCHMARK_UUID}"
corpus_base="${base_url}/corpus/${RUN_ID}/${BENCHMARK_UUID}"
bundles_base="${analysis_base}/bundles"

{
Expand Down Expand Up @@ -528,14 +556,14 @@ jobs:
echo "- Runner resource usage (Markdown): ${analysis_base}/runner_resource_usage.md"
echo "- Runner resource summary (CSV): ${analysis_base}/runner_resource_summary.csv"
echo "- Runner resource timeseries (CSV): ${analysis_base}/runner_resource_timeseries.csv"
if [[ "${{ steps.bundles.outputs.has_logs_zip }}" == "true" ]]; then
if [[ "${HAS_LOGS_ZIP}" == "true" ]]; then
echo "- Logs bundle: ${bundles_base}/logs.zip"
fi
if [[ "${{ steps.bundles.outputs.has_corpus_zip }}" == "true" ]]; then
if [[ "${HAS_CORPUS_ZIP}" == "true" ]]; then
echo "- Corpus bundle: ${bundles_base}/corpus.zip"
fi
echo "- Raw logs prefix: ${logs_base}/"
if [[ "${{ steps.bundles.outputs.has_corpus_zip }}" == "true" ]]; then
if [[ "${HAS_CORPUS_ZIP}" == "true" ]]; then
echo "- Raw corpus prefix: ${corpus_base}/"
fi
} >> "${release_body}"
Expand All @@ -546,10 +574,11 @@ jobs:
if: ${{ steps.release_check.outputs.exists != 'true' }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TAG: ${{ steps.release_check.outputs.tag }}
RELEASE_BODY: ${{ steps.body.outputs.release_body }}
run: |
set -euo pipefail
tag="${{ steps.release_check.outputs.tag }}"
gh release create "${tag}" --repo "${GITHUB_REPOSITORY}" --title "${tag}" --notes-file "${{ steps.body.outputs.release_body }}"
gh release create "${TAG}" --repo "${GITHUB_REPOSITORY}" --title "${TAG}" --notes-file "${RELEASE_BODY}"

refresh_docs:
needs: [discover, release]
Expand Down
21 changes: 15 additions & 6 deletions .github/workflows/benchmark-request.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ jobs:
uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6
- name: Parse and validate benchmark request
id: prepare
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const owner = context.repo.owner;
Expand Down Expand Up @@ -462,7 +462,12 @@ jobs:
contents: read
id-token: write
uses: ./.github/workflows/benchmark-run.yml
secrets: inherit
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
SCFUZZBENCH_BUCKET: ${{ secrets.SCFUZZBENCH_BUCKET }}
TF_BACKEND_CONFIG: ${{ secrets.TF_BACKEND_CONFIG }}
AWS_REGION: ${{ secrets.AWS_REGION }}
with:
target_repo_url: ${{ needs.prepare.outputs.target_repo_url }}
target_commit: ${{ needs.prepare.outputs.target_commit }}
Expand Down Expand Up @@ -492,7 +497,11 @@ jobs:
- name: Secure runner
uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6
- name: Comment result (and close on success)
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
BENCHMARK_RESULT: ${{ needs['benchmark-run'].result }}
RUN_ID: ${{ needs['benchmark-run'].outputs.run_id }}
BENCHMARK_UUID: ${{ needs['benchmark-run'].outputs.benchmark_uuid }}
with:
script: |
const owner = context.repo.owner;
Expand Down Expand Up @@ -538,7 +547,7 @@ jobs:
await github.rest.issues.setLabels({ owner, repo, issue_number, labels: want });
}

const result = "${{ needs['benchmark-run'].result }}";
const result = process.env.BENCHMARK_RESULT;
if (result !== "success") {
await setStatusLabel(LABEL_VALIDATED);
await upsertBotComment(
Expand All @@ -553,8 +562,8 @@ jobs:
return;
}

const run_id = "${{ needs['benchmark-run'].outputs.run_id }}";
const benchmark_uuid = "${{ needs['benchmark-run'].outputs.benchmark_uuid }}";
const run_id = process.env.RUN_ID;
const benchmark_uuid = process.env.BENCHMARK_UUID;
const docsUrl = `https://scfuzzbench.com/runs/${run_id}/${benchmark_uuid}/`;

await upsertBotComment(
Expand Down
12 changes: 8 additions & 4 deletions .github/workflows/benchmark-run.yml
Original file line number Diff line number Diff line change
Expand Up @@ -546,14 +546,18 @@ jobs:

- name: Upload run metadata
if: ${{ !inputs.dry_run }}
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: run-metadata
path: run_metadata.json

- name: Summary
env:
DRY_RUN: ${{ inputs.dry_run }}
RUN_ID: ${{ steps.tf_outputs.outputs.run_id }}
BENCHMARK_UUID: ${{ steps.tf_outputs.outputs.benchmark_uuid }}
run: |
if [[ "${{ inputs.dry_run }}" == "true" ]]; then
if [[ "${DRY_RUN}" == "true" ]]; then
{
echo "Benchmark run dry-run validation passed."
echo ""
Expand All @@ -564,7 +568,7 @@ jobs:
{
echo "Benchmark run started."
echo ""
echo "- Run ID: ${{ steps.tf_outputs.outputs.run_id }}"
echo "- Benchmark UUID: ${{ steps.tf_outputs.outputs.benchmark_uuid }}"
echo "- Run ID: ${RUN_ID}"
echo "- Benchmark UUID: ${BENCHMARK_UUID}"
} >> "${GITHUB_STEP_SUMMARY}"
fi
13 changes: 7 additions & 6 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,13 @@ jobs:
rm -rf "${topdir}" repo.tar.gz

- name: Setup Go
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version: "1.22.x"
go-version: "1.27.1"
cache: false

- name: Install actionlint
run: go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.10
run: go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.12

- name: Lint GitHub Actions workflows
run: |
Expand Down Expand Up @@ -106,7 +107,7 @@ jobs:
rm -rf "${topdir}" repo.tar.gz

- name: Setup Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: '3.11'

Expand Down Expand Up @@ -137,9 +138,9 @@ jobs:
rm -rf "${topdir}" repo.tar.gz

- name: Setup Node
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: "22"
node-version: "24"
cache: "npm"

- name: Install Node deps
Expand Down
10 changes: 5 additions & 5 deletions .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,14 +62,14 @@ jobs:
aws-region: ${{ env.AWS_REGION }}

- name: Setup Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.11"

- name: Setup Node
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: "22"
node-version: "24"
cache: "npm"

- name: Install Node deps
Expand All @@ -91,7 +91,7 @@ jobs:
run: npm run docs:build

- name: Upload Pages artifact
uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1
uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0
with:
path: docs/.vitepress/dist

Expand All @@ -108,4 +108,4 @@ jobs:
- name: Secure runner
uses: tempoxyz/gh-actions/actions/secure-runner@ee2960a6cc25d99bda45614135bfe1197d879bac # 2026-09-26T06-46-03Z-ee2960a6
- id: deployment
uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5
uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1
Loading
Loading